<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package libkb

import (
	"fmt"
	"time"

	keybase1 "github.com/keybase/client/go/protocol"
	jsonw "github.com/keybase/go-jsonw"
)

// Can be a ProofLinkWithState, one of the identities listed in a
// tracking statement, or a PGP Fingerprint!
type TrackIDComponent interface {
	ToIDString() string
	ToKeyValuePair() (string, string)
	GetProofState() keybase1.ProofState
	LastWriterWins() bool
}

type TrackSet struct {
	ids      map[string]TrackIDComponent
	services map[string]bool
}

func NewTrackSet() *TrackSet {
	return &TrackSet{
		ids:      make(map[string]TrackIDComponent),
		services: make(map[string]bool),
	}
}

func (ts TrackSet) Add(t TrackIDComponent) {
	ts.ids[t.ToIDString()] = t
	if t.LastWriterWins() {
		k, _ := t.ToKeyValuePair()
		ts.services[k] = true
	}
}

func (ts TrackSet) GetProofState(id string) keybase1.ProofState {
	ret := keybase1.ProofState_NONE
	if obj := ts.ids[id]; obj != nil {
		ret = obj.GetProofState()
	}
	return ret
}

func (ts TrackSet) Subtract(b TrackSet) (out []TrackIDComponent) {
	for _, c := range ts.ids {
		if !b.HasMember(c) {
			out = append(out, c)
		}
	}
	return
}

func (ts TrackSet) HasMember(t TrackIDComponent) bool {
	var found bool

	// For LastWriterWins like social networks, then it just matters
	// that there is some proof for the service.  For non-last-writer-wins,
	// like HTTPS and DNS, then the full proof needs to show up in A.
	if t.LastWriterWins() {
		k, _ := t.ToKeyValuePair()
		_, found = ts.services[k]
	} else {
		_, found = ts.ids[t.ToIDString()]
	}
	return found
}

func (ts TrackSet) LenEq(b TrackSet) bool {
	return len(ts.ids) == len(b.ids)
}

//=====================================================================

type TrackInstructions struct {
	Local  bool
	Remote bool
}

//=====================================================================

type TrackSummary struct {
	time     time.Time
	isRemote bool
	username string
}

func (s TrackSummary) IsRemote() bool      { return s.isRemote }
func (s TrackSummary) GetCTime() time.Time { return s.time }
func (s TrackSummary) Username() string    { return s.username }

//=====================================================================

type TrackLookup struct {
	link         *TrackChainLink     // The original chain link that I signed
	set          *TrackSet           // The total set of tracked identities
	ids          map[string][]string // A http -> [foo.com, boo.com] lookup
	trackerSeqno Seqno               // The seqno in the tracker's sighcain
}

func (l TrackLookup) ToSummary() TrackSummary {
	return TrackSummary{
		time:     l.GetCTime(),
		isRemote: l.IsRemote(),
	}
}

func (l TrackLookup) GetProofState(id string) keybase1.ProofState {
	return l.set.GetProofState(id)
}

func (l TrackLookup) GetTrackerSeqno() Seqno {
	return l.trackerSeqno
}

func (l TrackLookup) GetTrackedKeys() []TrackedKey {
	ret, err := l.link.GetTrackedKeys()
	if err != nil {
		G.Log.Warning("Error in lookup of tracked PGP fingerprints: %s", err)
	}
	return ret
}

func (l TrackLookup) GetEldestKID() keybase1.KID {
	ret, err := l.link.GetEldestKID()
	if err != nil {
		G.Log.Warning("Error in lookup of eldest KID: %s", err)
	}
	return ret
}

func (l TrackLookup) IsRemote() bool {
	return l.link.IsRemote()
}

type TrackDiff interface {
	BreaksTracking() bool
	ToDisplayString() string
	ToDisplayMarkup() *Markup
	IsSameAsTracked() bool
	GetTrackDiffType() keybase1.TrackDiffType
}

type TrackDiffUpgraded struct {
	prev, curr string
}

func (t TrackDiffUpgraded) IsSameAsTracked() bool {
	return false
}

func (t TrackDiffUpgraded) BreaksTracking() bool {
	return false
}
func (t TrackDiffUpgraded) ToDisplayString() string {
	return "Upgraded from " + t.prev + " to " + t.curr
}
func (t TrackDiffUpgraded) GetPrev() string { return t.prev }
func (t TrackDiffUpgraded) GetCurr() string { return t.curr }
func (t TrackDiffUpgraded) ToDisplayMarkup() *Markup {
	return NewMarkup(t.ToDisplayString())
}
func (t TrackDiffUpgraded) GetTrackDiffType() keybase1.TrackDiffType {
	return keybase1.TrackDiffType_UPGRADED
}

type TrackDiffNone struct{}

func (t TrackDiffNone) BreaksTracking() bool {
	return false
}
func (t TrackDiffNone) IsSameAsTracked() bool {
	return true
}

func (t TrackDiffNone) ToDisplayString() string {
	return "tracked"
}
func (t TrackDiffNone) ToDisplayMarkup() *Markup {
	return NewMarkup(t.ToDisplayString())
}
func (t TrackDiffNone) GetTrackDiffType() keybase1.TrackDiffType {
	return keybase1.TrackDiffType_NONE
}

type TrackDiffNew struct{}

func (t TrackDiffNew) BreaksTracking() bool {
	return false
}
func (t TrackDiffNew) IsSameAsTracked() bool {
	return false
}

type TrackDiffClash struct {
	observed, expected string
}

func (t TrackDiffNew) ToDisplayString() string {
	return "new"
}
func (t TrackDiffNew) ToDisplayMarkup() *Markup {
	return NewMarkup(t.ToDisplayString())
}
func (t TrackDiffNew) GetTrackDiffType() keybase1.TrackDiffType {
	return keybase1.TrackDiffType_NEW
}

func (t TrackDiffClash) BreaksTracking() bool {
	return true
}

func (t TrackDiffClash) ToDisplayString() string {
	return "CHANGED from \"" + t.expected + "\""
}
func (t TrackDiffClash) IsSameAsTracked() bool {
	return false
}
func (t TrackDiffClash) ToDisplayMarkup() *Markup {
	return NewMarkup(t.ToDisplayString())
}
func (t TrackDiffClash) GetTrackDiffType() keybase1.TrackDiffType {
	return keybase1.TrackDiffType_CLASH
}

type TrackDiffRevoked struct {
	idc TrackIDComponent
}

func (t TrackDiffRevoked) BreaksTracking() bool {
	return true
}
func (t TrackDiffRevoked) ToDisplayString() string {
	return "Deleted proof: " + t.idc.ToIDString()
}
func (t TrackDiffRevoked) IsSameAsTracked() bool {
	return false
}
func (t TrackDiffRevoked) ToDisplayMarkup() *Markup {
	return NewMarkup(t.ToDisplayString())
}
func (t TrackDiffRevoked) GetTrackDiffType() keybase1.TrackDiffType {
	return keybase1.TrackDiffType_REVOKED
}

type TrackDiffRemoteFail struct {
	observed keybase1.ProofState
}

func (t TrackDiffRemoteFail) BreaksTracking() bool {
	return true
}
func (t TrackDiffRemoteFail) ToDisplayString() string {
	return "remote failed"
}
func (t TrackDiffRemoteFail) ToDisplayMarkup() *Markup {
	return NewMarkup(t.ToDisplayString())
}
func (t TrackDiffRemoteFail) GetTrackDiffType() keybase1.TrackDiffType {
	return keybase1.TrackDiffType_REMOTE_FAIL
}
func (t TrackDiffRemoteFail) IsSameAsTracked() bool {
	return false
}

type TrackDiffRemoteWorking struct {
	tracked keybase1.ProofState
}

func (t TrackDiffRemoteWorking) BreaksTracking() bool {
	return false
}
func (t TrackDiffRemoteWorking) ToDisplayString() string {
	return "working"
}
func (t TrackDiffRemoteWorking) ToDisplayMarkup() *Markup {
	return NewMarkup(t.ToDisplayString())
}
func (t TrackDiffRemoteWorking) GetTrackDiffType() keybase1.TrackDiffType {
	return keybase1.TrackDiffType_REMOTE_WORKING
}
func (t TrackDiffRemoteWorking) IsSameAsTracked() bool {
	return false
}

type TrackDiffRemoteChanged struct {
	tracked, observed keybase1.ProofState
}

func (t TrackDiffRemoteChanged) BreaksTracking() bool {
	return false
}
func (t TrackDiffRemoteChanged) ToDisplayString() string {
	return "changed"
}
func (t TrackDiffRemoteChanged) ToDisplayMarkup() *Markup {
	return NewMarkup(t.ToDisplayString())
}
func (t TrackDiffRemoteChanged) GetTrackDiffType() keybase1.TrackDiffType {
	return keybase1.TrackDiffType_REMOTE_CHANGED
}
func (t TrackDiffRemoteChanged) IsSameAsTracked() bool {
	return false
}

type TrackDiffNewEldest struct {
	tracked  keybase1.KID
	observed keybase1.KID
}

func (t TrackDiffNewEldest) BreaksTracking() bool {
	return true
}
func (t TrackDiffNewEldest) IsSameAsTracked() bool {
	return false
}
func (t TrackDiffNewEldest) GetTrackDiffType() keybase1.TrackDiffType {
	return keybase1.TrackDiffType_NEW_ELDEST
}
func (t TrackDiffNewEldest) ToDisplayString() string {
	return fmt.Sprintf("Eldest key changed from %s to %s", t.tracked, t.observed)
}
func (t TrackDiffNewEldest) ToDisplayMarkup() *Markup {
	return NewMarkup(t.ToDisplayString())
}

func NewTrackLookup(link *TrackChainLink) *TrackLookup {
	sbs := link.ToServiceBlocks()
	set := NewTrackSet()
	ids := make(map[string][]string)
	for _, sb := range sbs {
		set.Add(sb)
		k, v := sb.ToKeyValuePair()
		ids[k] = append(ids[k], v)
	}
	ret := &TrackLookup{link: link, set: set, ids: ids, trackerSeqno: link.GetSeqno()}
	return ret
}

func (l *TrackLookup) GetCTime() time.Time {
	return l.link.GetCTime()
}

//=====================================================================

func LocalTrackDBKey(tracker, trackee keybase1.UID) DbKey {
	return DbKey{Typ: DBLocalTrack, Key: fmt.Sprintf("%s-%s", tracker, trackee)}
}

//=====================================================================

func LocalTrackChainLinkFor(tracker, trackee keybase1.UID, g *GlobalContext) (ret *TrackChainLink, err error) {
	g.Log.Debug("+ GetLocalTrack(%s,%s)", tracker, trackee)
	defer g.Log.Debug("- GetLocalTrack(%s,%s) -> (%v, %s)", tracker, trackee, ret, ErrToOk(err))

	var obj *jsonw.Wrapper
	obj, err = g.LocalDb.Get(LocalTrackDBKey(tracker, trackee))
	if err != nil {
		g.Log.Debug("| DB lookup failed")
		return
	}
	if obj == nil {
		g.Log.Debug("| No local track found")
		return
	}

	cl := &ChainLink{payloadJSON: obj, unsigned: true}
	if err = cl.UnpackLocal(); err != nil {
		g.Log.Debug("| unpack failed -> %s", err)
		return
	}
	base := GenericChainLink{cl}
	ret, err = ParseTrackChainLink(base)
	if ret != nil && err == nil {
		ret.local = true
	}

	return
}

func StoreLocalTrack(tracker keybase1.UID, trackee keybase1.UID, statement *jsonw.Wrapper, g *GlobalContext) error {
	g.Log.Debug("| StoreLocalTrack")
	return g.LocalDb.Put(LocalTrackDBKey(tracker, trackee), nil, statement)
}

func RemoveLocalTrack(tracker keybase1.UID, trackee keybase1.UID, g *GlobalContext) error {
	g.Log.Debug("| RemoveLocalTrack")
	return g.LocalDb.Delete(LocalTrackDBKey(tracker, trackee))
}
<commit_msg>notes from review [ci skip]<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package libkb

import (
	"fmt"
	"time"

	keybase1 "github.com/keybase/client/go/protocol"
	jsonw "github.com/keybase/go-jsonw"
)

// Can be a ProofLinkWithState, one of the identities listed in a
// tracking statement, or a PGP Fingerprint!
type TrackIDComponent interface {
	ToIDString() string
	ToKeyValuePair() (string, string)
	GetProofState() keybase1.ProofState
	LastWriterWins() bool
}

type TrackSet struct {
	ids      map[string]TrackIDComponent
	services map[string]bool
}

func NewTrackSet() *TrackSet {
	return &TrackSet{
		ids:      make(map[string]TrackIDComponent),
		services: make(map[string]bool),
	}
}

func (ts TrackSet) Add(t TrackIDComponent) {
	ts.ids[t.ToIDString()] = t
	if t.LastWriterWins() {
		k, _ := t.ToKeyValuePair()
		ts.services[k] = true
	}
}

func (ts TrackSet) GetProofState(id string) keybase1.ProofState {
	ret := keybase1.ProofState_NONE
	if obj := ts.ids[id]; obj != nil {
		ret = obj.GetProofState()
	}
	return ret
}

func (ts TrackSet) Subtract(b TrackSet) (out []TrackIDComponent) {
	for _, c := range ts.ids {
		if !b.HasMember(c) {
			out = append(out, c)
		}
	}
	return
}

func (ts TrackSet) HasMember(t TrackIDComponent) bool {
	var found bool

	// For LastWriterWins like social networks, then it just matters
	// that there is some proof for the service.  For non-last-writer-wins,
	// like HTTPS and DNS, then the full proof needs to show up in A.
	if t.LastWriterWins() {
		k, _ := t.ToKeyValuePair()
		_, found = ts.services[k]
	} else {
		_, found = ts.ids[t.ToIDString()]
	}
	return found
}

func (ts TrackSet) LenEq(b TrackSet) bool {
	return len(ts.ids) == len(b.ids)
}

//=====================================================================

type TrackInstructions struct {
	Local  bool
	Remote bool
}

//=====================================================================

type TrackSummary struct {
	time     time.Time
	isRemote bool
	username string
}

func (s TrackSummary) IsRemote() bool      { return s.isRemote }
func (s TrackSummary) GetCTime() time.Time { return s.time }
func (s TrackSummary) Username() string    { return s.username }

//=====================================================================

type TrackLookup struct {
	link         *TrackChainLink     // The original chain link that I signed
	set          *TrackSet           // The total set of tracked identities
	ids          map[string][]string // A http -> [foo.com, boo.com] lookup
	trackerSeqno Seqno               // The seqno in the tracker's sighcain
}

func (l TrackLookup) ToSummary() TrackSummary {
	return TrackSummary{
		time:     l.GetCTime(),
		isRemote: l.IsRemote(),
	}
}

func (l TrackLookup) GetProofState(id string) keybase1.ProofState {
	return l.set.GetProofState(id)
}

func (l TrackLookup) GetTrackerSeqno() Seqno {
	return l.trackerSeqno
}

func (l TrackLookup) GetTrackedKeys() []TrackedKey {
	ret, err := l.link.GetTrackedKeys()
	if err != nil {
		G.Log.Warning("Error in lookup of tracked PGP fingerprints: %s", err)
	}
	return ret
}

func (l TrackLookup) GetEldestKID() keybase1.KID {
	ret, err := l.link.GetEldestKID()
	if err != nil {
		G.Log.Warning("Error in lookup of eldest KID: %s", err)
	}
	return ret
}

func (l TrackLookup) IsRemote() bool {
	return l.link.IsRemote()
}

type TrackDiff interface {
	BreaksTracking() bool
	ToDisplayString() string
	ToDisplayMarkup() *Markup
	IsSameAsTracked() bool
	GetTrackDiffType() keybase1.TrackDiffType
}

type TrackDiffUpgraded struct {
	prev, curr string
}

func (t TrackDiffUpgraded) IsSameAsTracked() bool {
	return false
}

func (t TrackDiffUpgraded) BreaksTracking() bool {
	return false
}
func (t TrackDiffUpgraded) ToDisplayString() string {
	return "Upgraded from " + t.prev + " to " + t.curr
}
func (t TrackDiffUpgraded) GetPrev() string { return t.prev }
func (t TrackDiffUpgraded) GetCurr() string { return t.curr }
func (t TrackDiffUpgraded) ToDisplayMarkup() *Markup {
	return NewMarkup(t.ToDisplayString())
}
func (t TrackDiffUpgraded) GetTrackDiffType() keybase1.TrackDiffType {
	return keybase1.TrackDiffType_UPGRADED
}

type TrackDiffNone struct{}

func (t TrackDiffNone) BreaksTracking() bool {
	return false
}
func (t TrackDiffNone) IsSameAsTracked() bool {
	return true
}

func (t TrackDiffNone) ToDisplayString() string {
	return "tracked"
}
func (t TrackDiffNone) ToDisplayMarkup() *Markup {
	return NewMarkup(t.ToDisplayString())
}
func (t TrackDiffNone) GetTrackDiffType() keybase1.TrackDiffType {
	return keybase1.TrackDiffType_NONE
}

type TrackDiffNew struct{}

func (t TrackDiffNew) BreaksTracking() bool {
	return false
}
func (t TrackDiffNew) IsSameAsTracked() bool {
	return false
}

type TrackDiffClash struct {
	observed, expected string
}

func (t TrackDiffNew) ToDisplayString() string {
	return "new"
}
func (t TrackDiffNew) ToDisplayMarkup() *Markup {
	return NewMarkup(t.ToDisplayString())
}
func (t TrackDiffNew) GetTrackDiffType() keybase1.TrackDiffType {
	return keybase1.TrackDiffType_NEW
}

func (t TrackDiffClash) BreaksTracking() bool {
	return true
}

func (t TrackDiffClash) ToDisplayString() string {
	return "CHANGED from \"" + t.expected + "\""
}
func (t TrackDiffClash) IsSameAsTracked() bool {
	return false
}
func (t TrackDiffClash) ToDisplayMarkup() *Markup {
	return NewMarkup(t.ToDisplayString())
}
func (t TrackDiffClash) GetTrackDiffType() keybase1.TrackDiffType {
	return keybase1.TrackDiffType_CLASH
}

type TrackDiffRevoked struct {
	idc TrackIDComponent
}

func (t TrackDiffRevoked) BreaksTracking() bool {
	return true
}
func (t TrackDiffRevoked) ToDisplayString() string {
	return "Deleted proof: " + t.idc.ToIDString()
}
func (t TrackDiffRevoked) IsSameAsTracked() bool {
	return false
}
func (t TrackDiffRevoked) ToDisplayMarkup() *Markup {
	return NewMarkup(t.ToDisplayString())
}
func (t TrackDiffRevoked) GetTrackDiffType() keybase1.TrackDiffType {
	return keybase1.TrackDiffType_REVOKED
}

type TrackDiffRemoteFail struct {
	observed keybase1.ProofState
}

func (t TrackDiffRemoteFail) BreaksTracking() bool {
	return true
}
func (t TrackDiffRemoteFail) ToDisplayString() string {
	return "remote failed"
}
func (t TrackDiffRemoteFail) ToDisplayMarkup() *Markup {
	return NewMarkup(t.ToDisplayString())
}
func (t TrackDiffRemoteFail) GetTrackDiffType() keybase1.TrackDiffType {
	return keybase1.TrackDiffType_REMOTE_FAIL
}
func (t TrackDiffRemoteFail) IsSameAsTracked() bool {
	return false
}

type TrackDiffRemoteWorking struct {
	tracked keybase1.ProofState
}

func (t TrackDiffRemoteWorking) BreaksTracking() bool {
	return false
}
func (t TrackDiffRemoteWorking) ToDisplayString() string {
	return "working"
}
func (t TrackDiffRemoteWorking) ToDisplayMarkup() *Markup {
	return NewMarkup(t.ToDisplayString())
}
func (t TrackDiffRemoteWorking) GetTrackDiffType() keybase1.TrackDiffType {
	return keybase1.TrackDiffType_REMOTE_WORKING
}
func (t TrackDiffRemoteWorking) IsSameAsTracked() bool {
	return false
}

type TrackDiffRemoteChanged struct {
	tracked, observed keybase1.ProofState
}

func (t TrackDiffRemoteChanged) BreaksTracking() bool {
	return false
}
func (t TrackDiffRemoteChanged) ToDisplayString() string {
	return "changed"
}
func (t TrackDiffRemoteChanged) ToDisplayMarkup() *Markup {
	return NewMarkup(t.ToDisplayString())
}
func (t TrackDiffRemoteChanged) GetTrackDiffType() keybase1.TrackDiffType {
	return keybase1.TrackDiffType_REMOTE_CHANGED
}
func (t TrackDiffRemoteChanged) IsSameAsTracked() bool {
	return false
}

type TrackDiffNewEldest struct {
	tracked  keybase1.KID
	observed keybase1.KID
}

func (t TrackDiffNewEldest) BreaksTracking() bool {
	return true
}
func (t TrackDiffNewEldest) IsSameAsTracked() bool {
	return false
}
func (t TrackDiffNewEldest) GetTrackDiffType() keybase1.TrackDiffType {
	return keybase1.TrackDiffType_NEW_ELDEST
}
func (t TrackDiffNewEldest) ToDisplayString() string {
	return fmt.Sprintf("Account reset! Old key was %s; new key is %s", t.tracked, t.observed)
}
func (t TrackDiffNewEldest) ToDisplayMarkup() *Markup {
	return NewMarkup(t.ToDisplayString())
}

func NewTrackLookup(link *TrackChainLink) *TrackLookup {
	sbs := link.ToServiceBlocks()
	set := NewTrackSet()
	ids := make(map[string][]string)
	for _, sb := range sbs {
		set.Add(sb)
		k, v := sb.ToKeyValuePair()
		ids[k] = append(ids[k], v)
	}
	ret := &TrackLookup{link: link, set: set, ids: ids, trackerSeqno: link.GetSeqno()}
	return ret
}

func (l *TrackLookup) GetCTime() time.Time {
	return l.link.GetCTime()
}

//=====================================================================

func LocalTrackDBKey(tracker, trackee keybase1.UID) DbKey {
	return DbKey{Typ: DBLocalTrack, Key: fmt.Sprintf("%s-%s", tracker, trackee)}
}

//=====================================================================

func LocalTrackChainLinkFor(tracker, trackee keybase1.UID, g *GlobalContext) (ret *TrackChainLink, err error) {
	g.Log.Debug("+ GetLocalTrack(%s,%s)", tracker, trackee)
	defer g.Log.Debug("- GetLocalTrack(%s,%s) -> (%v, %s)", tracker, trackee, ret, ErrToOk(err))

	var obj *jsonw.Wrapper
	obj, err = g.LocalDb.Get(LocalTrackDBKey(tracker, trackee))
	if err != nil {
		g.Log.Debug("| DB lookup failed")
		return
	}
	if obj == nil {
		g.Log.Debug("| No local track found")
		return
	}

	cl := &ChainLink{payloadJSON: obj, unsigned: true}
	if err = cl.UnpackLocal(); err != nil {
		g.Log.Debug("| unpack failed -> %s", err)
		return
	}
	base := GenericChainLink{cl}
	ret, err = ParseTrackChainLink(base)
	if ret != nil && err == nil {
		ret.local = true
	}

	return
}

func StoreLocalTrack(tracker keybase1.UID, trackee keybase1.UID, statement *jsonw.Wrapper, g *GlobalContext) error {
	g.Log.Debug("| StoreLocalTrack")
	return g.LocalDb.Put(LocalTrackDBKey(tracker, trackee), nil, statement)
}

func RemoveLocalTrack(tracker keybase1.UID, trackee keybase1.UID, g *GlobalContext) error {
	g.Log.Debug("| RemoveLocalTrack")
	return g.LocalDb.Delete(LocalTrackDBKey(tracker, trackee))
}
<|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/weed-fs/go/glog"
	"code.google.com/p/weed-fs/go/util"
	"code.google.com/p/weed-fs/go/weed/weed_server"
	"net/http"
	"os"
	"runtime"
	"strconv"
	"strings"
	"time"
)

func init() {
	cmdVolume.Run = runVolume // break init cycle
}

var cmdVolume = &Command{
	UsageLine: "volume -port=8080 -dir=/tmp -max=5 -ip=server_name -mserver=localhost:9333",
	Short:     "start a volume server",
	Long: `start a volume server to provide storage spaces

  `,
}

var (
	vport                 = cmdVolume.Flag.Int("port", 8080, "http listen port")
	volumeFolders         = cmdVolume.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
	maxVolumeCounts       = cmdVolume.Flag.String("max", "7", "maximum numbers of volumes, count[,count]...")
	ip                    = cmdVolume.Flag.String("ip", "localhost", "ip or server name")
	publicUrl             = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible <ip|server_name>:<port>")
	masterNode            = cmdVolume.Flag.String("mserver", "localhost:9333", "master server location")
	vpulse                = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than the master's setting")
	vReadTimeout          = cmdVolume.Flag.Int("readTimeout", 3, "connection read timeout in seconds. Increase this if uploading large files.")
	vMaxCpu               = cmdVolume.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
	dataCenter            = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name")
	rack                  = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
	volumeWhiteListOption = cmdVolume.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")

	volumeWhiteList []string
)

func runVolume(cmd *Command, args []string) bool {
	if *vMaxCpu < 1 {
		*vMaxCpu = runtime.NumCPU()
	}
	runtime.GOMAXPROCS(*vMaxCpu)
	folders := strings.Split(*volumeFolders, ",")
	maxCountStrings := strings.Split(*maxVolumeCounts, ",")
	maxCounts := make([]int, 0)
	for _, maxString := range maxCountStrings {
		if max, e := strconv.Atoi(maxString); e == nil {
			maxCounts = append(maxCounts, max)
		} else {
			glog.Fatalf("The max specified in -max not a valid number %s", max)
		}
	}
	if len(folders) != len(maxCounts) {
		glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(folders), len(maxCounts))
	}
	for _, folder := range folders {
		if err := util.TestFolderWritable(folder); err != nil {
			glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", folder, err)
		}
	}

	if *publicUrl == "" {
		*publicUrl = *ip + ":" + strconv.Itoa(*vport)
	}
	if *volumeWhiteListOption != "" {
		volumeWhiteList = strings.Split(*volumeWhiteListOption, ",")
	}

	r := http.NewServeMux()

	weed_server.NewVolumeServer(r, VERSION, *ip, *vport, *publicUrl, folders, maxCounts,
		*masterNode, *vpulse, *dataCenter, *rack, volumeWhiteList,
	)

	glog.V(0).Infoln("Start Weed volume server", VERSION, "at http://"+*ip+":"+strconv.Itoa(*vport))
	srv := &http.Server{
		Addr:        *ip + ":" + strconv.Itoa(*vport),
		Handler:     r,
		ReadTimeout: (time.Duration(*vReadTimeout) * time.Second),
	}
	e := srv.ListenAndServe()
	if e != nil {
		glog.Fatalf("Fail to start:%s", e.Error())
	}
	return true
}
<commit_msg>help message adjustment<commit_after>package main

import (
	"code.google.com/p/weed-fs/go/glog"
	"code.google.com/p/weed-fs/go/util"
	"code.google.com/p/weed-fs/go/weed/weed_server"
	"net/http"
	"os"
	"runtime"
	"strconv"
	"strings"
	"time"
)

func init() {
	cmdVolume.Run = runVolume // break init cycle
}

var cmdVolume = &Command{
	UsageLine: "volume -port=8080 -dir=/tmp -max=5 -ip=server_name -mserver=localhost:9333",
	Short:     "start a volume server",
	Long: `start a volume server to provide storage spaces

  `,
}

var (
	vport                 = cmdVolume.Flag.Int("port", 8080, "http listen port")
	volumeFolders         = cmdVolume.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
	maxVolumeCounts       = cmdVolume.Flag.String("max", "7", "maximum numbers of volumes, count[,count]...")
	ip                    = cmdVolume.Flag.String("ip", "localhost", "ip or server name")
	publicUrl             = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible <ip|server_name>:<port>")
	masterNode            = cmdVolume.Flag.String("mserver", "localhost:9333", "master server location")
	vpulse                = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting")
	vReadTimeout          = cmdVolume.Flag.Int("readTimeout", 3, "connection read timeout in seconds. Increase this if uploading large files.")
	vMaxCpu               = cmdVolume.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
	dataCenter            = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name")
	rack                  = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
	volumeWhiteListOption = cmdVolume.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")

	volumeWhiteList []string
)

func runVolume(cmd *Command, args []string) bool {
	if *vMaxCpu < 1 {
		*vMaxCpu = runtime.NumCPU()
	}
	runtime.GOMAXPROCS(*vMaxCpu)
	folders := strings.Split(*volumeFolders, ",")
	maxCountStrings := strings.Split(*maxVolumeCounts, ",")
	maxCounts := make([]int, 0)
	for _, maxString := range maxCountStrings {
		if max, e := strconv.Atoi(maxString); e == nil {
			maxCounts = append(maxCounts, max)
		} else {
			glog.Fatalf("The max specified in -max not a valid number %s", max)
		}
	}
	if len(folders) != len(maxCounts) {
		glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(folders), len(maxCounts))
	}
	for _, folder := range folders {
		if err := util.TestFolderWritable(folder); err != nil {
			glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", folder, err)
		}
	}

	if *publicUrl == "" {
		*publicUrl = *ip + ":" + strconv.Itoa(*vport)
	}
	if *volumeWhiteListOption != "" {
		volumeWhiteList = strings.Split(*volumeWhiteListOption, ",")
	}

	r := http.NewServeMux()

	weed_server.NewVolumeServer(r, VERSION, *ip, *vport, *publicUrl, folders, maxCounts,
		*masterNode, *vpulse, *dataCenter, *rack, volumeWhiteList,
	)

	glog.V(0).Infoln("Start Weed volume server", VERSION, "at http://"+*ip+":"+strconv.Itoa(*vport))
	srv := &http.Server{
		Addr:        *ip + ":" + strconv.Itoa(*vport),
		Handler:     r,
		ReadTimeout: (time.Duration(*vReadTimeout) * time.Second),
	}
	e := srv.ListenAndServe()
	if e != nil {
		glog.Fatalf("Fail to start:%s", e.Error())
	}
	return true
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"log"
	"os"

	"github.com/xikug/gsocket"
)

type demoServer struct{}

// OnConnect 客户端连接事件
func (server demoServer) OnConnect(c *gsocket.Connection) {
	log.Printf("CONNECTED: %s\n", c.RemoteAddr())
}

// OnDisconnect 客户端断开连接事件
func (server demoServer) OnDisconnect(c *gsocket.Connection) {
	log.Printf("DISCONNECTED: %s\n", c.RemoteAddr())
}

// OnRecv 收到客户端发来的数据
func (server demoServer) OnRecv(c *gsocket.Connection, data []byte) {
	log.Printf("DATA RECVED: %s %d - %v\n", c.RemoteAddr(), len(data), data)
	session.Send(data)
}

// OnError 有错误发生
func (server demoServer) OnError(c *gsocket.Connection, err error) {
	log.Printf("ERROR: %s - %s\n", c.RemoteAddr(), err.Error())
}

func main() {
	demoServer := &demoServer{}
	//CreateTCPServer 的handler可以传nil
	server := gsocket.CreateTCPServer("0.0.0.0", 9595,
		demoServer.OnConnect, demoServer.OnDisconnect, demoServer.OnRecv, demoServer.OnError)

	err := server.Start()
	if err != nil {
		log.Printf("Start Server Error: %s\n", err.Error())
		return
	}

	log.Printf("Listening %s...\n", server.Addr())

	pause()
}

func pause() {
	println("按回车键退出...\n")
	r := bufio.NewReader(os.Stdin)
	r.ReadByte()
}
<commit_msg>修正server.go的BUG<commit_after>package main

import (
	"bufio"
	"log"
	"os"

	"github.com/xikug/gsocket"
)

type demoServer struct{}

// OnConnect 客户端连接事件
func (server demoServer) OnConnect(c *gsocket.Connection) {
	log.Printf("CONNECTED: %s\n", c.RemoteAddr())
}

// OnDisconnect 客户端断开连接事件
func (server demoServer) OnDisconnect(c *gsocket.Connection) {
	log.Printf("DISCONNECTED: %s\n", c.RemoteAddr())
}

// OnRecv 收到客户端发来的数据
func (server demoServer) OnRecv(c *gsocket.Connection, data []byte) {
	log.Printf("DATA RECVED: %s %d - %v\n", c.RemoteAddr(), len(data), data)
	c.Send(data)
}

// OnError 有错误发生
func (server demoServer) OnError(c *gsocket.Connection, err error) {
	log.Printf("ERROR: %s - %s\n", c.RemoteAddr(), err.Error())
}

func main() {
	demoServer := &demoServer{}
	//CreateTCPServer 的handler可以传nil
	server := gsocket.CreateTCPServer("0.0.0.0", 9595,
		demoServer.OnConnect, demoServer.OnDisconnect, demoServer.OnRecv, demoServer.OnError)

	err := server.Start()
	if err != nil {
		log.Printf("Start Server Error: %s\n", err.Error())
		return
	}

	log.Printf("Listening %s...\n", server.Addr())

	pause()
}

func pause() {
	println("按回车键退出...\n")
	r := bufio.NewReader(os.Stdin)
	r.ReadByte()
}
<|endoftext|>


<commit_before>// Copyright 2015 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// upgrade upgrades corpus from an old format to a new format.
// Upgrade is not fully automatic. You need to update prog.Serialize.
// Run the tool. Then update prog.Deserialize. And run the tool again that
// the corpus is not changed this time.
package main

import (
	"bytes"
	"crypto/sha1"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"

	"github.com/google/syzkaller/pkg/osutil"
	"github.com/google/syzkaller/prog"
	_ "github.com/google/syzkaller/sys"
)

func main() {
	if len(os.Args) != 2 {
		fatalf("usage: syz-upgrage corpus_dir")
	}
	files, err := ioutil.ReadDir(os.Args[1])
	if err != nil {
		fatalf("failed to read corpus dir: %v", err)
	}
	target, err := prog.GetTarget(runtime.GOOS, runtime.GOARCH)
	if err != nil {
		fatalf("%v", err)
	}
	for _, f := range files {
		fname := filepath.Join(os.Args[1], f.Name())
		data, err := ioutil.ReadFile(fname)
		if err != nil {
			fatalf("failed to read program: %v", err)
		}
		p, err := target.Deserialize(data)
		if err != nil {
			fatalf("failed to deserialize program: %v", err)
		}
		data1 := p.Serialize()
		if bytes.Equal(data, data1) {
			continue
		}
		fmt.Printf("upgrading:\n%s\nto:\n%s\n\n", data, data1)
		hash := sha1.Sum(data1)
		fname1 := filepath.Join(os.Args[1], hex.EncodeToString(hash[:]))
		if err := osutil.WriteFile(fname1, data1); err != nil {
			fatalf("failed to write program: %v", err)
		}
		if err := os.Remove(fname); err != nil {
			fatalf("failed to remove program: %v", err)
		}
	}
}

func fatalf(msg string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, msg+"\n", args...)
	os.Exit(1)
}
<commit_msg>syz-upgrade: fix typo in usage message<commit_after>// Copyright 2015 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// upgrade upgrades corpus from an old format to a new format.
// Upgrade is not fully automatic. You need to update prog.Serialize.
// Run the tool. Then update prog.Deserialize. And run the tool again that
// the corpus is not changed this time.
package main

import (
	"bytes"
	"crypto/sha1"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"

	"github.com/google/syzkaller/pkg/osutil"
	"github.com/google/syzkaller/prog"
	_ "github.com/google/syzkaller/sys"
)

func main() {
	if len(os.Args) != 2 {
		fatalf("usage: syz-upgrade corpus_dir")
	}
	files, err := ioutil.ReadDir(os.Args[1])
	if err != nil {
		fatalf("failed to read corpus dir: %v", err)
	}
	target, err := prog.GetTarget(runtime.GOOS, runtime.GOARCH)
	if err != nil {
		fatalf("%v", err)
	}
	for _, f := range files {
		fname := filepath.Join(os.Args[1], f.Name())
		data, err := ioutil.ReadFile(fname)
		if err != nil {
			fatalf("failed to read program: %v", err)
		}
		p, err := target.Deserialize(data)
		if err != nil {
			fatalf("failed to deserialize program: %v", err)
		}
		data1 := p.Serialize()
		if bytes.Equal(data, data1) {
			continue
		}
		fmt.Printf("upgrading:\n%s\nto:\n%s\n\n", data, data1)
		hash := sha1.Sum(data1)
		fname1 := filepath.Join(os.Args[1], hex.EncodeToString(hash[:]))
		if err := osutil.WriteFile(fname1, data1); err != nil {
			fatalf("failed to write program: %v", err)
		}
		if err := os.Remove(fname); err != nil {
			fatalf("failed to remove program: %v", err)
		}
	}
}

func fatalf(msg string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, msg+"\n", args...)
	os.Exit(1)
}
<|endoftext|>


<commit_before>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package executor

import (
	"fmt"
	"math"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/juju/errors"
	"github.com/pingcap/tidb/ast"
	"github.com/pingcap/tidb/config"
	"github.com/pingcap/tidb/context"
	"github.com/pingcap/tidb/infoschema"
	"github.com/pingcap/tidb/kv"
	"github.com/pingcap/tidb/model"
	"github.com/pingcap/tidb/mysql"
	"github.com/pingcap/tidb/plan"
	"github.com/pingcap/tidb/sessionctx/variable"
	"github.com/pingcap/tidb/terror"
	"github.com/pingcap/tidb/util/logutil"
)

type processinfoSetter interface {
	SetProcessInfo(string)
}

// recordSet wraps an executor, implements ast.RecordSet interface
type recordSet struct {
	fields      []*ast.ResultField
	executor    Executor
	stmt        *ExecStmt
	processinfo processinfoSetter
	lastErr     error
}

func (a *recordSet) Fields() []*ast.ResultField {
	if len(a.fields) == 0 {
		for _, col := range a.executor.Schema().Columns {
			dbName := col.DBName.O
			if dbName == "" && col.TblName.L != "" {
				dbName = a.stmt.ctx.GetSessionVars().CurrentDB
			}
			rf := &ast.ResultField{
				ColumnAsName: col.ColName,
				TableAsName:  col.TblName,
				DBName:       model.NewCIStr(dbName),
				Table:        &model.TableInfo{Name: col.OrigTblName},
				Column: &model.ColumnInfo{
					FieldType: *col.RetType,
					Name:      col.ColName,
				},
			}
			a.fields = append(a.fields, rf)
		}
	}
	return a.fields
}

func (a *recordSet) Next() (*ast.Row, error) {
	row, err := a.executor.Next()
	if err != nil {
		a.lastErr = err
		return nil, errors.Trace(err)
	}
	if row == nil {
		if a.stmt != nil {
			a.stmt.ctx.GetSessionVars().LastFoundRows = a.stmt.ctx.GetSessionVars().StmtCtx.FoundRows()
		}
		return nil, nil
	}

	if a.stmt != nil {
		a.stmt.ctx.GetSessionVars().StmtCtx.AddFoundRows(1)
	}
	return &ast.Row{Data: row}, nil
}

func (a *recordSet) Close() error {
	err := a.executor.Close()
	a.stmt.logSlowQuery(a.lastErr == nil)
	if a.processinfo != nil {
		a.processinfo.SetProcessInfo("")
	}
	return errors.Trace(err)
}

// ExecStmt implements the ast.Statement interface, it builds a plan.Plan to an ast.Statement.
type ExecStmt struct {
	// InfoSchema stores a reference to the schema information.
	InfoSchema infoschema.InfoSchema
	// Plan stores a reference to the final physical plan.
	Plan plan.Plan
	// Expensive represents whether this query is an expensive one.
	Expensive bool
	// Cacheable represents whether the physical plan can be cached.
	Cacheable bool
	// Text represents the origin query text.
	Text string

	ctx            context.Context
	startTime      time.Time
	isPreparedStmt bool

	// ReadOnly represents the statement is read-only.
	ReadOnly bool
}

// OriginText implements ast.Statement interface.
func (a *ExecStmt) OriginText() string {
	return a.Text
}

// IsPrepared implements ast.Statement interface.
func (a *ExecStmt) IsPrepared() bool {
	return a.isPreparedStmt
}

// IsReadOnly implements ast.Statement interface.
func (a *ExecStmt) IsReadOnly() bool {
	return a.ReadOnly
}

// Exec implements the ast.Statement Exec interface.
// This function builds an Executor from a plan. If the Executor doesn't return result,
// like the INSERT, UPDATE statements, it executes in this function, if the Executor returns
// result, execution is done after this function returns, in the returned ast.RecordSet Next method.
func (a *ExecStmt) Exec(ctx context.Context) (ast.RecordSet, error) {
	a.startTime = time.Now()
	a.ctx = ctx

	if _, ok := a.Plan.(*plan.Analyze); ok && ctx.GetSessionVars().InRestrictedSQL {
		oriStats := ctx.GetSessionVars().Systems[variable.TiDBBuildStatsConcurrency]
		oriScan := ctx.GetSessionVars().DistSQLScanConcurrency
		oriIndex := ctx.GetSessionVars().IndexSerialScanConcurrency
		oriIso := ctx.GetSessionVars().Systems[variable.TxnIsolation]
		ctx.GetSessionVars().Systems[variable.TiDBBuildStatsConcurrency] = "1"
		ctx.GetSessionVars().DistSQLScanConcurrency = 1
		ctx.GetSessionVars().IndexSerialScanConcurrency = 1
		ctx.GetSessionVars().Systems[variable.TxnIsolation] = ast.ReadCommitted
		defer func() {
			ctx.GetSessionVars().Systems[variable.TiDBBuildStatsConcurrency] = oriStats
			ctx.GetSessionVars().DistSQLScanConcurrency = oriScan
			ctx.GetSessionVars().IndexSerialScanConcurrency = oriIndex
			ctx.GetSessionVars().Systems[variable.TxnIsolation] = oriIso
		}()
	}

	e, err := a.buildExecutor(ctx)
	if err != nil {
		return nil, errors.Trace(err)
	}

	if err := e.Open(); err != nil {
		return nil, errors.Trace(err)
	}

	var pi processinfoSetter
	if raw, ok := ctx.(processinfoSetter); ok {
		pi = raw
		sql := a.OriginText()
		if simple, ok := a.Plan.(*plan.Simple); ok && simple.Statement != nil {
			if ss, ok := simple.Statement.(ast.SensitiveStmtNode); ok {
				// Use SecureText to avoid leak password information.
				sql = ss.SecureText()
			}
		}
		// Update processinfo, ShowProcess() will use it.
		pi.SetProcessInfo(sql)
	}
	// Fields or Schema are only used for statements that return result set.
	if e.Schema().Len() == 0 {
		return a.handleNoDelayExecutor(e, ctx, pi)
	}

	return &recordSet{
		executor:    e,
		stmt:        a,
		processinfo: pi,
	}, nil
}

func (a *ExecStmt) handleNoDelayExecutor(e Executor, ctx context.Context, pi processinfoSetter) (ast.RecordSet, error) {
	// Check if "tidb_snapshot" is set for the write executors.
	// In history read mode, we can not do write operations.
	switch e.(type) {
	case *DeleteExec, *InsertExec, *UpdateExec, *ReplaceExec, *LoadData, *DDLExec:
		snapshotTS := ctx.GetSessionVars().SnapshotTS
		if snapshotTS != 0 {
			return nil, errors.New("can not execute write statement when 'tidb_snapshot' is set")
		}
	}

	var err error
	defer func() {
		if pi != nil {
			pi.SetProcessInfo("")
		}
		terror.Log(errors.Trace(e.Close()))
		a.logSlowQuery(err == nil)
	}()
	for {
		var row Row
		row, err = e.Next()
		if err != nil {
			return nil, errors.Trace(err)
		}
		// Even though there isn't any result set, the row is still used to indicate if there is
		// more work to do.
		// For example, the UPDATE statement updates a single row on a Next call, we keep calling Next until
		// There is no more rows to update.
		if row == nil {
			return nil, nil
		}
	}
}

// buildExecutor build a executor from plan, prepared statement may need additional procedure.
func (a *ExecStmt) buildExecutor(ctx context.Context) (Executor, error) {
	priority := kv.PriorityNormal
	if _, ok := a.Plan.(*plan.Execute); !ok {
		// Do not sync transaction for Execute statement, because the real optimization work is done in
		// "ExecuteExec.Build".
		var err error
		isPointGet := IsPointGetWithPKOrUniqueKeyByAutoCommit(ctx, a.Plan)
		if isPointGet {
			log.Debugf("[%d][InitTxnWithStartTS] %s", ctx.GetSessionVars().ConnectionID, a.Text)
			err = ctx.InitTxnWithStartTS(math.MaxUint64)
		} else {
			log.Debugf("[%d][ActivePendingTxn] %s", ctx.GetSessionVars().ConnectionID, a.Text)
			err = ctx.ActivePendingTxn()
		}
		if err != nil {
			return nil, errors.Trace(err)
		}

		if stmtPri := ctx.GetSessionVars().StmtCtx.Priority; stmtPri != mysql.NoPriority {
			priority = int(stmtPri)
		} else {
			switch {
			case isPointGet:
				priority = kv.PriorityHigh
			case a.Expensive:
				priority = kv.PriorityLow
			}
		}
	}
	if _, ok := a.Plan.(*plan.Analyze); ok && ctx.GetSessionVars().InRestrictedSQL {
		priority = kv.PriorityLow
	}

	b := newExecutorBuilder(ctx, a.InfoSchema, priority)
	e := b.build(a.Plan)
	if b.err != nil {
		return nil, errors.Trace(b.err)
	}

	// ExecuteExec is not a real Executor, we only use it to build another Executor from a prepared statement.
	if executorExec, ok := e.(*ExecuteExec); ok {
		err := executorExec.Build()
		if err != nil {
			return nil, errors.Trace(err)
		}
		a.Text = executorExec.Stmt.Text()
		a.isPreparedStmt = true
		a.Plan = executorExec.Plan
		e = executorExec.StmtExec
	}
	return e, nil
}

func (a *ExecStmt) logSlowQuery(succ bool) {
	cfg := config.GetGlobalConfig()
	costTime := time.Since(a.startTime)
	sql := a.Text
	if len(sql) > cfg.Log.QueryLogMaxLen {
		sql = fmt.Sprintf("%.*q(len:%d)", cfg.Log.QueryLogMaxLen, sql, len(a.Text))
	}
	connID := a.ctx.GetSessionVars().ConnectionID
	logEntry := log.NewEntry(logutil.SlowQueryLogger)
	logEntry.Data = log.Fields{
		"connectionId": connID,
		"costTime":     costTime,
		"sql":          sql,
	}
	if costTime < time.Duration(cfg.Log.SlowThreshold)*time.Millisecond {
		logEntry.WithField("type", "query").WithField("succ", succ).Debugf("query")
	} else {
		logEntry.WithField("type", "slow-query").WithField("succ", succ).Warnf("slow-query")
	}
}

// IsPointGetWithPKOrUniqueKeyByAutoCommit returns true when meets following conditions:
//  1. ctx is auto commit tagged
//  2. txn is nil
//  2. plan is point get by pk or unique key
func IsPointGetWithPKOrUniqueKeyByAutoCommit(ctx context.Context, p plan.Plan) bool {
	// check auto commit
	if !ctx.GetSessionVars().IsAutocommit() {
		return false
	}

	// check txn
	if ctx.Txn() != nil {
		return false
	}

	// check plan
	if proj, ok := p.(*plan.Projection); ok {
		if len(proj.Children()) != 1 {
			return false
		}
		p = proj.Children()[0]
	}

	switch v := p.(type) {
	case *plan.PhysicalIndexScan:
		return v.IsPointGetByUniqueKey(ctx.GetSessionVars().StmtCtx)
	case *plan.PhysicalIndexReader:
		indexScan := v.IndexPlans[0].(*plan.PhysicalIndexScan)
		return indexScan.IsPointGetByUniqueKey(ctx.GetSessionVars().StmtCtx)
	case *plan.PhysicalIndexLookUpReader:
		indexScan := v.IndexPlans[0].(*plan.PhysicalIndexScan)
		return indexScan.IsPointGetByUniqueKey(ctx.GetSessionVars().StmtCtx)
	case *plan.PhysicalTableScan:
		return len(v.Ranges) == 1 && v.Ranges[0].IsPoint()
	case *plan.PhysicalTableReader:
		tableScan := v.TablePlans[0].(*plan.PhysicalTableScan)
		return len(tableScan.Ranges) == 1 && tableScan.Ranges[0].IsPoint()
	default:
		return false
	}
}
<commit_msg>executor: add default database to query log (#5078)<commit_after>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package executor

import (
	"fmt"
	"math"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/juju/errors"
	"github.com/pingcap/tidb/ast"
	"github.com/pingcap/tidb/config"
	"github.com/pingcap/tidb/context"
	"github.com/pingcap/tidb/infoschema"
	"github.com/pingcap/tidb/kv"
	"github.com/pingcap/tidb/model"
	"github.com/pingcap/tidb/mysql"
	"github.com/pingcap/tidb/plan"
	"github.com/pingcap/tidb/sessionctx/variable"
	"github.com/pingcap/tidb/terror"
	"github.com/pingcap/tidb/util/logutil"
)

type processinfoSetter interface {
	SetProcessInfo(string)
}

// recordSet wraps an executor, implements ast.RecordSet interface
type recordSet struct {
	fields      []*ast.ResultField
	executor    Executor
	stmt        *ExecStmt
	processinfo processinfoSetter
	lastErr     error
}

func (a *recordSet) Fields() []*ast.ResultField {
	if len(a.fields) == 0 {
		for _, col := range a.executor.Schema().Columns {
			dbName := col.DBName.O
			if dbName == "" && col.TblName.L != "" {
				dbName = a.stmt.ctx.GetSessionVars().CurrentDB
			}
			rf := &ast.ResultField{
				ColumnAsName: col.ColName,
				TableAsName:  col.TblName,
				DBName:       model.NewCIStr(dbName),
				Table:        &model.TableInfo{Name: col.OrigTblName},
				Column: &model.ColumnInfo{
					FieldType: *col.RetType,
					Name:      col.ColName,
				},
			}
			a.fields = append(a.fields, rf)
		}
	}
	return a.fields
}

func (a *recordSet) Next() (*ast.Row, error) {
	row, err := a.executor.Next()
	if err != nil {
		a.lastErr = err
		return nil, errors.Trace(err)
	}
	if row == nil {
		if a.stmt != nil {
			a.stmt.ctx.GetSessionVars().LastFoundRows = a.stmt.ctx.GetSessionVars().StmtCtx.FoundRows()
		}
		return nil, nil
	}

	if a.stmt != nil {
		a.stmt.ctx.GetSessionVars().StmtCtx.AddFoundRows(1)
	}
	return &ast.Row{Data: row}, nil
}

func (a *recordSet) Close() error {
	err := a.executor.Close()
	a.stmt.logSlowQuery(a.lastErr == nil)
	if a.processinfo != nil {
		a.processinfo.SetProcessInfo("")
	}
	return errors.Trace(err)
}

// ExecStmt implements the ast.Statement interface, it builds a plan.Plan to an ast.Statement.
type ExecStmt struct {
	// InfoSchema stores a reference to the schema information.
	InfoSchema infoschema.InfoSchema
	// Plan stores a reference to the final physical plan.
	Plan plan.Plan
	// Expensive represents whether this query is an expensive one.
	Expensive bool
	// Cacheable represents whether the physical plan can be cached.
	Cacheable bool
	// Text represents the origin query text.
	Text string

	ctx            context.Context
	startTime      time.Time
	isPreparedStmt bool

	// ReadOnly represents the statement is read-only.
	ReadOnly bool
}

// OriginText implements ast.Statement interface.
func (a *ExecStmt) OriginText() string {
	return a.Text
}

// IsPrepared implements ast.Statement interface.
func (a *ExecStmt) IsPrepared() bool {
	return a.isPreparedStmt
}

// IsReadOnly implements ast.Statement interface.
func (a *ExecStmt) IsReadOnly() bool {
	return a.ReadOnly
}

// Exec implements the ast.Statement Exec interface.
// This function builds an Executor from a plan. If the Executor doesn't return result,
// like the INSERT, UPDATE statements, it executes in this function, if the Executor returns
// result, execution is done after this function returns, in the returned ast.RecordSet Next method.
func (a *ExecStmt) Exec(ctx context.Context) (ast.RecordSet, error) {
	a.startTime = time.Now()
	a.ctx = ctx

	if _, ok := a.Plan.(*plan.Analyze); ok && ctx.GetSessionVars().InRestrictedSQL {
		oriStats := ctx.GetSessionVars().Systems[variable.TiDBBuildStatsConcurrency]
		oriScan := ctx.GetSessionVars().DistSQLScanConcurrency
		oriIndex := ctx.GetSessionVars().IndexSerialScanConcurrency
		oriIso := ctx.GetSessionVars().Systems[variable.TxnIsolation]
		ctx.GetSessionVars().Systems[variable.TiDBBuildStatsConcurrency] = "1"
		ctx.GetSessionVars().DistSQLScanConcurrency = 1
		ctx.GetSessionVars().IndexSerialScanConcurrency = 1
		ctx.GetSessionVars().Systems[variable.TxnIsolation] = ast.ReadCommitted
		defer func() {
			ctx.GetSessionVars().Systems[variable.TiDBBuildStatsConcurrency] = oriStats
			ctx.GetSessionVars().DistSQLScanConcurrency = oriScan
			ctx.GetSessionVars().IndexSerialScanConcurrency = oriIndex
			ctx.GetSessionVars().Systems[variable.TxnIsolation] = oriIso
		}()
	}

	e, err := a.buildExecutor(ctx)
	if err != nil {
		return nil, errors.Trace(err)
	}

	if err := e.Open(); err != nil {
		return nil, errors.Trace(err)
	}

	var pi processinfoSetter
	if raw, ok := ctx.(processinfoSetter); ok {
		pi = raw
		sql := a.OriginText()
		if simple, ok := a.Plan.(*plan.Simple); ok && simple.Statement != nil {
			if ss, ok := simple.Statement.(ast.SensitiveStmtNode); ok {
				// Use SecureText to avoid leak password information.
				sql = ss.SecureText()
			}
		}
		// Update processinfo, ShowProcess() will use it.
		pi.SetProcessInfo(sql)
	}
	// Fields or Schema are only used for statements that return result set.
	if e.Schema().Len() == 0 {
		return a.handleNoDelayExecutor(e, ctx, pi)
	}

	return &recordSet{
		executor:    e,
		stmt:        a,
		processinfo: pi,
	}, nil
}

func (a *ExecStmt) handleNoDelayExecutor(e Executor, ctx context.Context, pi processinfoSetter) (ast.RecordSet, error) {
	// Check if "tidb_snapshot" is set for the write executors.
	// In history read mode, we can not do write operations.
	switch e.(type) {
	case *DeleteExec, *InsertExec, *UpdateExec, *ReplaceExec, *LoadData, *DDLExec:
		snapshotTS := ctx.GetSessionVars().SnapshotTS
		if snapshotTS != 0 {
			return nil, errors.New("can not execute write statement when 'tidb_snapshot' is set")
		}
	}

	var err error
	defer func() {
		if pi != nil {
			pi.SetProcessInfo("")
		}
		terror.Log(errors.Trace(e.Close()))
		a.logSlowQuery(err == nil)
	}()
	for {
		var row Row
		row, err = e.Next()
		if err != nil {
			return nil, errors.Trace(err)
		}
		// Even though there isn't any result set, the row is still used to indicate if there is
		// more work to do.
		// For example, the UPDATE statement updates a single row on a Next call, we keep calling Next until
		// There is no more rows to update.
		if row == nil {
			return nil, nil
		}
	}
}

// buildExecutor build a executor from plan, prepared statement may need additional procedure.
func (a *ExecStmt) buildExecutor(ctx context.Context) (Executor, error) {
	priority := kv.PriorityNormal
	if _, ok := a.Plan.(*plan.Execute); !ok {
		// Do not sync transaction for Execute statement, because the real optimization work is done in
		// "ExecuteExec.Build".
		var err error
		isPointGet := IsPointGetWithPKOrUniqueKeyByAutoCommit(ctx, a.Plan)
		if isPointGet {
			log.Debugf("[%d][InitTxnWithStartTS] %s", ctx.GetSessionVars().ConnectionID, a.Text)
			err = ctx.InitTxnWithStartTS(math.MaxUint64)
		} else {
			log.Debugf("[%d][ActivePendingTxn] %s", ctx.GetSessionVars().ConnectionID, a.Text)
			err = ctx.ActivePendingTxn()
		}
		if err != nil {
			return nil, errors.Trace(err)
		}

		if stmtPri := ctx.GetSessionVars().StmtCtx.Priority; stmtPri != mysql.NoPriority {
			priority = int(stmtPri)
		} else {
			switch {
			case isPointGet:
				priority = kv.PriorityHigh
			case a.Expensive:
				priority = kv.PriorityLow
			}
		}
	}
	if _, ok := a.Plan.(*plan.Analyze); ok && ctx.GetSessionVars().InRestrictedSQL {
		priority = kv.PriorityLow
	}

	b := newExecutorBuilder(ctx, a.InfoSchema, priority)
	e := b.build(a.Plan)
	if b.err != nil {
		return nil, errors.Trace(b.err)
	}

	// ExecuteExec is not a real Executor, we only use it to build another Executor from a prepared statement.
	if executorExec, ok := e.(*ExecuteExec); ok {
		err := executorExec.Build()
		if err != nil {
			return nil, errors.Trace(err)
		}
		a.Text = executorExec.Stmt.Text()
		a.isPreparedStmt = true
		a.Plan = executorExec.Plan
		e = executorExec.StmtExec
	}
	return e, nil
}

func (a *ExecStmt) logSlowQuery(succ bool) {
	cfg := config.GetGlobalConfig()
	costTime := time.Since(a.startTime)
	sql := a.Text
	if len(sql) > cfg.Log.QueryLogMaxLen {
		sql = fmt.Sprintf("%.*q(len:%d)", cfg.Log.QueryLogMaxLen, sql, len(a.Text))
	}
	connID := a.ctx.GetSessionVars().ConnectionID
	currentDB := a.ctx.GetSessionVars().CurrentDB
	logEntry := log.NewEntry(logutil.SlowQueryLogger)
	logEntry.Data = log.Fields{
		"connectionId": connID,
		"costTime":     costTime,
		"database":     currentDB,
		"sql":          sql,
	}
	if costTime < time.Duration(cfg.Log.SlowThreshold)*time.Millisecond {
		logEntry.WithField("type", "query").WithField("succ", succ).Debugf("query")
	} else {
		logEntry.WithField("type", "slow-query").WithField("succ", succ).Warnf("slow-query")
	}
}

// IsPointGetWithPKOrUniqueKeyByAutoCommit returns true when meets following conditions:
//  1. ctx is auto commit tagged
//  2. txn is nil
//  2. plan is point get by pk or unique key
func IsPointGetWithPKOrUniqueKeyByAutoCommit(ctx context.Context, p plan.Plan) bool {
	// check auto commit
	if !ctx.GetSessionVars().IsAutocommit() {
		return false
	}

	// check txn
	if ctx.Txn() != nil {
		return false
	}

	// check plan
	if proj, ok := p.(*plan.Projection); ok {
		if len(proj.Children()) != 1 {
			return false
		}
		p = proj.Children()[0]
	}

	switch v := p.(type) {
	case *plan.PhysicalIndexScan:
		return v.IsPointGetByUniqueKey(ctx.GetSessionVars().StmtCtx)
	case *plan.PhysicalIndexReader:
		indexScan := v.IndexPlans[0].(*plan.PhysicalIndexScan)
		return indexScan.IsPointGetByUniqueKey(ctx.GetSessionVars().StmtCtx)
	case *plan.PhysicalIndexLookUpReader:
		indexScan := v.IndexPlans[0].(*plan.PhysicalIndexScan)
		return indexScan.IsPointGetByUniqueKey(ctx.GetSessionVars().StmtCtx)
	case *plan.PhysicalTableScan:
		return len(v.Ranges) == 1 && v.Ranges[0].IsPoint()
	case *plan.PhysicalTableReader:
		tableScan := v.TablePlans[0].(*plan.PhysicalTableScan)
		return len(tableScan.Ranges) == 1 && tableScan.Ranges[0].IsPoint()
	default:
		return false
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"net/http"
	_ "net/http/pprof"

	hpe "github.com/appscode/haproxy_exporter/exporter"
	"github.com/appscode/pat"
	"github.com/appscode/voyager/api"
	"github.com/orcaman/concurrent-map"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/prometheus/common/log"
	"github.com/prometheus/common/version"
	kerr "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	PathParamAPIGroup  = ":apiGroup"
	PathParamNamespace = ":namespace"
	PathParamName      = ":name"
	QueryParamPodIP    = "pod"
)

var (
	selectedServerMetrics map[int]*prometheus.GaugeVec

	registerers = cmap.New() // URL.path => *prometheus.Registry
)

func DeleteRegistry(w http.ResponseWriter, r *http.Request) {
	registerers.Remove(r.URL.Path)
	w.WriteHeader(http.StatusOK)
}

func ExportMetrics(w http.ResponseWriter, r *http.Request) {
	params, found := pat.FromContext(r.Context())
	if !found {
		http.Error(w, "Missing parameters", http.StatusBadRequest)
		return
	}
	apiGroup := params.Get(PathParamAPIGroup)
	if apiGroup == "" {
		http.Error(w, "Missing parameter:"+PathParamAPIGroup, http.StatusBadRequest)
		return
	}
	namespace := params.Get(PathParamNamespace)
	if namespace == "" {
		http.Error(w, "Missing parameter:"+PathParamNamespace, http.StatusBadRequest)
		return
	}
	name := params.Get(PathParamName)
	if name == "" {
		http.Error(w, "Missing parameter:"+PathParamName, http.StatusBadRequest)
		return
	}
	podIP := r.URL.Query().Get(QueryParamPodIP)
	if podIP == "" {
		podIP = "127.0.0.1"
		return
	}

	switch apiGroup {
	case "extensions":
		var reg *prometheus.Registry
		if val, ok := registerers.Get(r.URL.Path); ok {
			reg = val.(*prometheus.Registry)
		} else {
			reg = prometheus.NewRegistry()
			if absent := registerers.SetIfAbsent(r.URL.Path, reg); !absent {
				r2, _ := registerers.Get(r.URL.Path)
				reg = r2.(*prometheus.Registry)
			} else {
				log.Infof("Configuring exporter for standard ingress %s in namespace %s", name, namespace)
				ingress, err := kubeClient.ExtensionsV1beta1().Ingresses(namespace).Get(name, metav1.GetOptions{})
				if kerr.IsNotFound(err) {
					http.NotFound(w, r)
					return
				} else if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
					return
				}
				engress, err := api.NewEngressFromIngress(ingress)
				if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
					return
				}
				scrapeURL, err := getScrapeURL(engress, podIP)
				if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
					return
				}
				exporter, err := hpe.NewExporter(scrapeURL, selectedServerMetrics, haProxyTimeout)
				if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
					return
				}
				reg.MustRegister(exporter)
				reg.MustRegister(version.NewCollector("haproxy_exporter"))
			}
		}
		promhttp.HandlerFor(reg, promhttp.HandlerOpts{}).ServeHTTP(w, r)
		return
	case api.GroupName:
		var reg *prometheus.Registry
		if val, ok := registerers.Get(r.URL.Path); ok {
			reg = val.(*prometheus.Registry)
		} else {
			reg = prometheus.NewRegistry()
			if absent := registerers.SetIfAbsent(r.URL.Path, reg); !absent {
				r2, _ := registerers.Get(r.URL.Path)
				reg = r2.(*prometheus.Registry)
			} else {
				log.Infof("Configuring exporter for appscode ingress %s in namespace %s", name, namespace)
				engress, err := extClient.Ingress(namespace).Get(name)
				if kerr.IsNotFound(err) {
					http.NotFound(w, r)
					return
				} else if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
					return
				}
				scrapeURL, err := getScrapeURL(engress, podIP)
				if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
					return
				}
				exporter, err := hpe.NewExporter(scrapeURL, selectedServerMetrics, haProxyTimeout)
				if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
					return
				}
				reg.MustRegister(exporter)
				reg.MustRegister(version.NewCollector("haproxy_exporter"))
			}
		}
		promhttp.HandlerFor(reg, promhttp.HandlerOpts{}).ServeHTTP(w, r)
		return
	}
	http.NotFound(w, r)
}

func getScrapeURL(r *api.Ingress, podIP string) (string, error) {
	if !r.Stats() {
		return "", errors.New("Stats not exposed")
	}
	if r.StatsSecretName() != "" {
		return fmt.Sprintf("http://%s:%d?stats;csv", podIP, r.StatsPort()), nil
	}
	secret, err := kubeClient.CoreV1().Secrets(r.Namespace).Get(r.StatsSecretName(), metav1.GetOptions{})
	if err != nil {
		return "", err
	}
	userName := string(secret.Data["username"])
	passWord := string(secret.Data["password"])
	return fmt.Sprintf("http://%s:%s@%s:%d?stats;csv", userName, passWord, podIP, r.StatsPort()), nil
}
<commit_msg>Update ***Getter interfaces match form (#231)<commit_after>package main

import (
	"errors"
	"fmt"
	"net/http"
	_ "net/http/pprof"

	hpe "github.com/appscode/haproxy_exporter/exporter"
	"github.com/appscode/pat"
	"github.com/appscode/voyager/api"
	"github.com/orcaman/concurrent-map"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/prometheus/common/log"
	"github.com/prometheus/common/version"
	kerr "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	PathParamAPIGroup  = ":apiGroup"
	PathParamNamespace = ":namespace"
	PathParamName      = ":name"
	QueryParamPodIP    = "pod"
)

var (
	selectedServerMetrics map[int]*prometheus.GaugeVec

	registerers = cmap.New() // URL.path => *prometheus.Registry
)

func DeleteRegistry(w http.ResponseWriter, r *http.Request) {
	registerers.Remove(r.URL.Path)
	w.WriteHeader(http.StatusOK)
}

func ExportMetrics(w http.ResponseWriter, r *http.Request) {
	params, found := pat.FromContext(r.Context())
	if !found {
		http.Error(w, "Missing parameters", http.StatusBadRequest)
		return
	}
	apiGroup := params.Get(PathParamAPIGroup)
	if apiGroup == "" {
		http.Error(w, "Missing parameter:"+PathParamAPIGroup, http.StatusBadRequest)
		return
	}
	namespace := params.Get(PathParamNamespace)
	if namespace == "" {
		http.Error(w, "Missing parameter:"+PathParamNamespace, http.StatusBadRequest)
		return
	}
	name := params.Get(PathParamName)
	if name == "" {
		http.Error(w, "Missing parameter:"+PathParamName, http.StatusBadRequest)
		return
	}
	podIP := r.URL.Query().Get(QueryParamPodIP)
	if podIP == "" {
		podIP = "127.0.0.1"
		return
	}

	switch apiGroup {
	case "extensions":
		var reg *prometheus.Registry
		if val, ok := registerers.Get(r.URL.Path); ok {
			reg = val.(*prometheus.Registry)
		} else {
			reg = prometheus.NewRegistry()
			if absent := registerers.SetIfAbsent(r.URL.Path, reg); !absent {
				r2, _ := registerers.Get(r.URL.Path)
				reg = r2.(*prometheus.Registry)
			} else {
				log.Infof("Configuring exporter for standard ingress %s in namespace %s", name, namespace)
				ingress, err := kubeClient.ExtensionsV1beta1().Ingresses(namespace).Get(name, metav1.GetOptions{})
				if kerr.IsNotFound(err) {
					http.NotFound(w, r)
					return
				} else if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
					return
				}
				engress, err := api.NewEngressFromIngress(ingress)
				if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
					return
				}
				scrapeURL, err := getScrapeURL(engress, podIP)
				if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
					return
				}
				exporter, err := hpe.NewExporter(scrapeURL, selectedServerMetrics, haProxyTimeout)
				if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
					return
				}
				reg.MustRegister(exporter)
				reg.MustRegister(version.NewCollector("haproxy_exporter"))
			}
		}
		promhttp.HandlerFor(reg, promhttp.HandlerOpts{}).ServeHTTP(w, r)
		return
	case api.GroupName:
		var reg *prometheus.Registry
		if val, ok := registerers.Get(r.URL.Path); ok {
			reg = val.(*prometheus.Registry)
		} else {
			reg = prometheus.NewRegistry()
			if absent := registerers.SetIfAbsent(r.URL.Path, reg); !absent {
				r2, _ := registerers.Get(r.URL.Path)
				reg = r2.(*prometheus.Registry)
			} else {
				log.Infof("Configuring exporter for appscode ingress %s in namespace %s", name, namespace)
				engress, err := extClient.Ingresses(namespace).Get(name)
				if kerr.IsNotFound(err) {
					http.NotFound(w, r)
					return
				} else if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
					return
				}
				scrapeURL, err := getScrapeURL(engress, podIP)
				if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
					return
				}
				exporter, err := hpe.NewExporter(scrapeURL, selectedServerMetrics, haProxyTimeout)
				if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
					return
				}
				reg.MustRegister(exporter)
				reg.MustRegister(version.NewCollector("haproxy_exporter"))
			}
		}
		promhttp.HandlerFor(reg, promhttp.HandlerOpts{}).ServeHTTP(w, r)
		return
	}
	http.NotFound(w, r)
}

func getScrapeURL(r *api.Ingress, podIP string) (string, error) {
	if !r.Stats() {
		return "", errors.New("Stats not exposed")
	}
	if r.StatsSecretName() != "" {
		return fmt.Sprintf("http://%s:%d?stats;csv", podIP, r.StatsPort()), nil
	}
	secret, err := kubeClient.CoreV1().Secrets(r.Namespace).Get(r.StatsSecretName(), metav1.GetOptions{})
	if err != nil {
		return "", err
	}
	userName := string(secret.Data["username"])
	passWord := string(secret.Data["password"])
	return fmt.Sprintf("http://%s:%s@%s:%d?stats;csv", userName, passWord, podIP, r.StatsPort()), nil
}
<|endoftext|>


<commit_before>package exec

import (
	u "github.com/araddon/gou"
	"github.com/araddon/qlbridge/datasource"
	"github.com/araddon/qlbridge/expr"
	"github.com/araddon/qlbridge/value"
	"github.com/araddon/qlbridge/vm"
)

type Projection struct {
	*TaskBase
	sql *expr.SqlSelect
}

func NewProjection(sqlSelect *expr.SqlSelect) *Projection {
	s := &Projection{
		TaskBase: NewTaskBase("Projection"),
		sql:      sqlSelect,
	}
	s.Handler = projectionEvaluator(sqlSelect, s)
	return s
}

// Create handler function for evaluation (ie, field selection from tuples)
func projectionEvaluator(sql *expr.SqlSelect, task TaskRunner) MessageHandler {
	out := task.MessageOut()
	//evaluator := vm.Evaluator(where)
	return func(ctx *Context, msg datasource.Message) bool {
		defer func() {
			if r := recover(); r != nil {
				u.Errorf("crap, %v", r)
			}
		}()

		u.Infof("got projection message: %#v", msg.Body())
		var outMsg datasource.Message
		switch mt := msg.(type) {
		case *datasource.SqlDriverMessageMap:
			// readContext := datasource.NewContextUrlValues(uv)
			// use our custom write context for example purposes
			writeContext := datasource.NewContextSimple()
			outMsg = writeContext
			//u.Infof("about to project: colsct%v %#v", len(sql.Columns), outMsg)
			for _, from := range sql.From {
				for _, col := range from.Columns {
					//u.Debugf("col:   %#v", col)
					if col.Guard != nil {
						ifColValue, ok := vm.Eval(mt, col.Guard)
						if !ok {
							u.Errorf("Could not evaluate if:   %v", col.Guard.StringAST())
							//return fmt.Errorf("Could not evaluate if clause: %v", col.Guard.String())
						}
						//u.Debugf("if eval val:  %T:%v", ifColValue, ifColValue)
						switch ifColVal := ifColValue.(type) {
						case value.BoolValue:
							if ifColVal.Val() == false {
								//u.Debugf("Filtering out col")
								continue
							}
						}
					}
					if col.Star {
						for k, v := range mt.Vals {
							writeContext.Put(&expr.Column{As: k}, nil, value.NewValue(v))
						}
					} else {
						//u.Debugf("tree.Root: as?%v %v", col.As, col.Expr.String())
						v, ok := vm.Eval(mt, col.Expr)
						//u.Debugf("evaled: ok?%v key=%v  val=%v", ok, col.Key(), v)
						if ok {
							writeContext.Put(col, mt, v)
						}
					}
				}
			}

		case *datasource.ContextUrlValues:
			// readContext := datasource.NewContextUrlValues(uv)
			// use our custom write context for example purposes
			writeContext := datasource.NewContextSimple()
			outMsg = writeContext
			//u.Infof("about to project: colsct%v %#v", len(sql.Columns), outMsg)
			for _, col := range sql.Columns {
				//u.Debugf("col:   %#v", col)
				if col.Guard != nil {
					ifColValue, ok := vm.Eval(mt, col.Guard)
					if !ok {
						u.Errorf("Could not evaluate if:   %v", col.Guard.StringAST())
						//return fmt.Errorf("Could not evaluate if clause: %v", col.Guard.String())
					}
					//u.Debugf("if eval val:  %T:%v", ifColValue, ifColValue)
					switch ifColVal := ifColValue.(type) {
					case value.BoolValue:
						if ifColVal.Val() == false {
							//u.Debugf("Filtering out col")
							continue
						}
					}
				}
				if col.Star {
					for k, v := range mt.Row() {
						writeContext.Put(&expr.Column{As: k}, nil, v)
					}
				} else {
					//u.Debugf("tree.Root: as?%v %#v", col.As, col.Expr)
					v, ok := vm.Eval(mt, col.Expr)
					//u.Debugf("evaled: ok?%v key=%v  val=%v", ok, col.Key(), v)
					if ok {
						writeContext.Put(col, mt, v)
					}
				}

			}
		default:
			u.Errorf("could not project msg:  %T", msg)
		}

		//u.Debugf("completed projection for: %p %#v", out, outMsg)
		select {
		case out <- outMsg:
			return true
		case <-task.SigChan():
			return false
		}
	}
}
<commit_msg>turn down log spam<commit_after>package exec

import (
	u "github.com/araddon/gou"
	"github.com/araddon/qlbridge/datasource"
	"github.com/araddon/qlbridge/expr"
	"github.com/araddon/qlbridge/value"
	"github.com/araddon/qlbridge/vm"
)

type Projection struct {
	*TaskBase
	sql *expr.SqlSelect
}

func NewProjection(sqlSelect *expr.SqlSelect) *Projection {
	s := &Projection{
		TaskBase: NewTaskBase("Projection"),
		sql:      sqlSelect,
	}
	s.Handler = projectionEvaluator(sqlSelect, s)
	return s
}

// Create handler function for evaluation (ie, field selection from tuples)
func projectionEvaluator(sql *expr.SqlSelect, task TaskRunner) MessageHandler {
	out := task.MessageOut()
	//evaluator := vm.Evaluator(where)
	return func(ctx *Context, msg datasource.Message) bool {
		defer func() {
			if r := recover(); r != nil {
				u.Errorf("crap, %v", r)
			}
		}()

		//u.Infof("got projection message: %#v", msg.Body())
		var outMsg datasource.Message
		switch mt := msg.(type) {
		case *datasource.SqlDriverMessageMap:
			// readContext := datasource.NewContextUrlValues(uv)
			// use our custom write context for example purposes
			writeContext := datasource.NewContextSimple()
			outMsg = writeContext
			//u.Infof("about to project: colsct%v %#v", len(sql.Columns), outMsg)
			for _, from := range sql.From {
				for _, col := range from.Columns {
					//u.Debugf("col:   %#v", col)
					if col.Guard != nil {
						ifColValue, ok := vm.Eval(mt, col.Guard)
						if !ok {
							u.Errorf("Could not evaluate if:   %v", col.Guard.StringAST())
							//return fmt.Errorf("Could not evaluate if clause: %v", col.Guard.String())
						}
						//u.Debugf("if eval val:  %T:%v", ifColValue, ifColValue)
						switch ifColVal := ifColValue.(type) {
						case value.BoolValue:
							if ifColVal.Val() == false {
								//u.Debugf("Filtering out col")
								continue
							}
						}
					}
					if col.Star {
						for k, v := range mt.Vals {
							writeContext.Put(&expr.Column{As: k}, nil, value.NewValue(v))
						}
					} else {
						//u.Debugf("tree.Root: as?%v %v", col.As, col.Expr.String())
						v, ok := vm.Eval(mt, col.Expr)
						//u.Debugf("evaled: ok?%v key=%v  val=%v", ok, col.Key(), v)
						if ok {
							writeContext.Put(col, mt, v)
						}
					}
				}
			}

		case *datasource.ContextUrlValues:
			// readContext := datasource.NewContextUrlValues(uv)
			// use our custom write context for example purposes
			writeContext := datasource.NewContextSimple()
			outMsg = writeContext
			//u.Infof("about to project: colsct%v %#v", len(sql.Columns), outMsg)
			for _, col := range sql.Columns {
				//u.Debugf("col:   %#v", col)
				if col.Guard != nil {
					ifColValue, ok := vm.Eval(mt, col.Guard)
					if !ok {
						u.Errorf("Could not evaluate if:   %v", col.Guard.StringAST())
						//return fmt.Errorf("Could not evaluate if clause: %v", col.Guard.String())
					}
					//u.Debugf("if eval val:  %T:%v", ifColValue, ifColValue)
					switch ifColVal := ifColValue.(type) {
					case value.BoolValue:
						if ifColVal.Val() == false {
							//u.Debugf("Filtering out col")
							continue
						}
					}
				}
				if col.Star {
					for k, v := range mt.Row() {
						writeContext.Put(&expr.Column{As: k}, nil, v)
					}
				} else {
					//u.Debugf("tree.Root: as?%v %#v", col.As, col.Expr)
					v, ok := vm.Eval(mt, col.Expr)
					//u.Debugf("evaled: ok?%v key=%v  val=%v", ok, col.Key(), v)
					if ok {
						writeContext.Put(col, mt, v)
					}
				}

			}
		default:
			u.Errorf("could not project msg:  %T", msg)
		}

		//u.Debugf("completed projection for: %p %#v", out, outMsg)
		select {
		case out <- outMsg:
			return true
		case <-task.SigChan():
			return false
		}
	}
}
<|endoftext|>


<commit_before>package queueinformer

import (
	"fmt"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	v1 "k8s.io/api/core/v1"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/client-go/tools/cache"

	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorclient"
)

// An Operator is a collection of QueueInformers
// OpClient is used to establish the connection to kubernetes
type Operator struct {
	queueInformers []*QueueInformer
	queueIndexers  []*QueueIndexer
	informers      []cache.SharedIndexInformer
	OpClient       operatorclient.ClientInterface
	Log            *logrus.Logger
	syncCh         chan error
}

// NewOperator creates a new Operator configured to manage the cluster defined in kubeconfig.
func NewOperator(kubeconfig string, logger *logrus.Logger, queueInformers ...*QueueInformer) (*Operator, error) {
	opClient := operatorclient.NewClientFromConfig(kubeconfig, logger)
	if queueInformers == nil {
		queueInformers = []*QueueInformer{}
	}
	operator := &Operator{
		OpClient:       opClient,
		queueInformers: queueInformers,
		Log:            logger,
	}
	return operator, nil
}

func NewOperatorFromClient(opClient operatorclient.ClientInterface, logger *logrus.Logger, queueInformers ...*QueueInformer) (*Operator, error) {
	if queueInformers == nil {
		queueInformers = []*QueueInformer{}
	}
	operator := &Operator{
		OpClient:       opClient,
		queueInformers: queueInformers,
		Log:            logger,
	}
	return operator, nil
}

// RegisterQueueInformer adds a QueueInformer to this operator
func (o *Operator) RegisterQueueInformer(queueInformer *QueueInformer) {
	if o.queueInformers == nil {
		o.queueInformers = []*QueueInformer{}
	}
	o.queueInformers = append(o.queueInformers, queueInformer)
}

// RegisterInformer adds an Informer to this operator
func (o *Operator) RegisterInformer(informer cache.SharedIndexInformer) {
	if o.informers == nil {
		o.informers = []cache.SharedIndexInformer{}
	}
	o.informers = append(o.informers, informer)
}

// RegisterQueueIndexer adds a QueueIndexer to this operator
func (o *Operator) RegisterQueueIndexer(indexer *QueueIndexer) {
	if o.queueIndexers == nil {
		o.queueIndexers = []*QueueIndexer{}
	}
	o.queueIndexers = append(o.queueIndexers, indexer)
}

// Run starts the operator's control loops
func (o *Operator) Run(stopc <-chan struct{}) (ready, done chan struct{}, atLevel chan error) {
	ready = make(chan struct{})
	atLevel = make(chan error, 25)
	done = make(chan struct{})

	o.syncCh = atLevel

	go func() {
		defer func() {
			close(ready)
			close(atLevel)
			close(done)
		}()

		for _, queueInformer := range o.queueInformers {
			defer queueInformer.queue.ShutDown()
		}

		errChan := make(chan error)
		go func() {
			v, err := o.OpClient.KubernetesInterface().Discovery().ServerVersion()
			if err != nil {
				errChan <- errors.Wrap(err, "communicating with server failed")
				return
			}
			o.Log.Infof("connection established. cluster-version: %v", v)
			errChan <- nil
		}()

		var hasSyncedCheckFns []cache.InformerSynced
		for _, queueInformer := range o.queueInformers {
			hasSyncedCheckFns = append(hasSyncedCheckFns, queueInformer.informer.HasSynced)
		}
		for _, informer := range o.informers {
			hasSyncedCheckFns = append(hasSyncedCheckFns, informer.HasSynced)
		}

		select {
		case err := <-errChan:
			if err != nil {
				o.Log.Infof("operator not ready: %s", err.Error())
				return
			}
			o.Log.Info("operator ready")
		case <-stopc:
			return
		}

		o.Log.Info("starting informers...")
		for _, queueInformer := range o.queueInformers {
			go queueInformer.informer.Run(stopc)
		}

		for _, informer := range o.informers {
			go informer.Run(stopc)
		}

		o.Log.Info("waiting for caches to sync...")
		if ok := cache.WaitForCacheSync(stopc, hasSyncedCheckFns...); !ok {
			o.Log.Info("failed to wait for caches to sync")
			return
		}

		o.Log.Info("starting workers...")
		for _, queueInformer := range o.queueInformers {
			go o.worker(queueInformer)
			go o.worker(queueInformer)
		}

		for _, queueIndexer := range o.queueIndexers {
			go o.indexerWorker(queueIndexer)
			go o.indexerWorker(queueIndexer)
		}
		ready <- struct{}{}
		<-stopc
	}()

	return
}

// worker runs a worker thread that just dequeues items, processes them, and marks them done.
// It enforces that the syncHandler is never invoked concurrently with the same key.
func (o *Operator) worker(loop *QueueInformer) {
	for o.processNextWorkItem(loop) {
	}
}

func (o *Operator) processNextWorkItem(loop *QueueInformer) bool {
	queue := loop.queue
	key, quit := queue.Get()

	if quit {
		return false
	}
	defer queue.Done(key)

	// requeue five times on error
	err := o.sync(loop, key.(string))
	if err != nil && queue.NumRequeues(key.(string)) < 5 {
		o.Log.Infof("retrying %s", key)
		utilruntime.HandleError(errors.Wrap(err, fmt.Sprintf("Sync %q failed", key)))
		queue.AddRateLimited(key)
		return true
	}
	queue.Forget(key)

	select {
	case o.syncCh <- err:
	default:
	}

	if err := loop.HandleMetrics(); err != nil {
		o.Log.Error(err)
	}
	return true
}

func (o *Operator) sync(loop *QueueInformer, key string) error {
	logger := o.Log.WithField("queue", loop.name).WithField("key", key)
	obj, exists, err := loop.informer.GetIndexer().GetByKey(key)
	if err != nil {
		return err
	}
	if !exists {
		// For now, we ignore the case where an object used to exist but no longer does
		logger.Info("couldn't get from queue")
		logger.Debugf("have keys: %v", loop.informer.GetIndexer().ListKeys())
		return nil
	}
	return loop.syncHandler(obj)
}

// This provides the same function as above, but for queues that are not auto-fed by informers.
// indexerWorker runs a worker thread that just dequeues items, processes them, and marks them done.
// It enforces that the syncHandler is never invoked concurrently with the same key.
func (o *Operator) indexerWorker(loop *QueueIndexer) {
	for o.processNextIndexerWorkItem(loop) {
	}
}

func (o *Operator) processNextIndexerWorkItem(loop *QueueIndexer) bool {
	queue := loop.queue
	key, quit := queue.Get()

	if quit {
		return false
	}
	defer queue.Done(key)

	// requeue five times on error
	if err := o.syncIndexer(loop, key.(string)); err != nil && queue.NumRequeues(key.(string)) < 5 {
		o.Log.Infof("retrying %s", key)
		utilruntime.HandleError(errors.Wrap(err, fmt.Sprintf("Sync %q failed", key)))
		queue.AddRateLimited(key)
		return true
	}
	queue.Forget(key)
	if err := loop.HandleMetrics(); err != nil {
		o.Log.Error(err)
	}
	return true
}

func (o *Operator) syncIndexer(loop *QueueIndexer, key string) error {
	logger := o.Log.WithField("queue", loop.name).WithField("key", key)
	namespace, _, err := cache.SplitMetaNamespaceKey(key)
	if err != nil {
		return err
	}

	indexer, ok := loop.indexers[namespace]
	if !ok {
		if indexer, ok = loop.indexers[v1.NamespaceAll]; !ok {
			return fmt.Errorf("no indexer found for %s, have %v", namespace, loop.indexers)
		}
	}
	obj, exists, err := indexer.GetByKey(key)
	if err != nil {
		return err
	}
	if !exists {
		// For now, we ignore the case where an object used to exist but no longer does
		logger.Info("couldn't get from queue")
		logger.Debugf("have keys: %v", indexer.ListKeys())
		return nil
	}
	return loop.syncHandler(obj)
}
<commit_msg>refactor(queueinformer): prevent > 1 run calls for duplicate informers<commit_after>package queueinformer

import (
	"fmt"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	v1 "k8s.io/api/core/v1"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/client-go/tools/cache"

	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorclient"
)

// An Operator is a collection of QueueInformers
// OpClient is used to establish the connection to kubernetes
type Operator struct {
	queueInformers []*QueueInformer
	queueIndexers  []*QueueIndexer
	informers      []cache.SharedIndexInformer
	OpClient       operatorclient.ClientInterface
	Log            *logrus.Logger
	syncCh         chan error
}

// NewOperator creates a new Operator configured to manage the cluster defined in kubeconfig.
func NewOperator(kubeconfig string, logger *logrus.Logger, queueInformers ...*QueueInformer) (*Operator, error) {
	opClient := operatorclient.NewClientFromConfig(kubeconfig, logger)
	if queueInformers == nil {
		queueInformers = []*QueueInformer{}
	}
	operator := &Operator{
		OpClient:       opClient,
		queueInformers: queueInformers,
		Log:            logger,
	}
	return operator, nil
}

func NewOperatorFromClient(opClient operatorclient.ClientInterface, logger *logrus.Logger, queueInformers ...*QueueInformer) (*Operator, error) {
	if queueInformers == nil {
		queueInformers = []*QueueInformer{}
	}
	operator := &Operator{
		OpClient:       opClient,
		queueInformers: queueInformers,
		Log:            logger,
	}
	return operator, nil
}

// RegisterQueueInformer adds a QueueInformer to this operator
func (o *Operator) RegisterQueueInformer(queueInformer *QueueInformer) {
	if o.queueInformers == nil {
		o.queueInformers = []*QueueInformer{}
	}
	o.queueInformers = append(o.queueInformers, queueInformer)
}

// RegisterInformer adds an Informer to this operator
func (o *Operator) RegisterInformer(informer cache.SharedIndexInformer) {
	if o.informers == nil {
		o.informers = []cache.SharedIndexInformer{}
	}
	o.informers = append(o.informers, informer)
}

// RegisterQueueIndexer adds a QueueIndexer to this operator
func (o *Operator) RegisterQueueIndexer(indexer *QueueIndexer) {
	if o.queueIndexers == nil {
		o.queueIndexers = []*QueueIndexer{}
	}
	o.queueIndexers = append(o.queueIndexers, indexer)
}

// Run starts the operator's control loops
func (o *Operator) Run(stopc <-chan struct{}) (ready, done chan struct{}, atLevel chan error) {
	ready = make(chan struct{})
	atLevel = make(chan error, 25)
	done = make(chan struct{})

	o.syncCh = atLevel

	go func() {
		defer func() {
			close(ready)
			close(atLevel)
			close(done)
		}()

		for _, queueInformer := range o.queueInformers {
			defer queueInformer.queue.ShutDown()
		}

		errChan := make(chan error)
		go func() {
			v, err := o.OpClient.KubernetesInterface().Discovery().ServerVersion()
			if err != nil {
				errChan <- errors.Wrap(err, "communicating with server failed")
				return
			}
			o.Log.Infof("connection established. cluster-version: %v", v)
			errChan <- nil
		}()

		var hasSyncedCheckFns []cache.InformerSynced
		for _, queueInformer := range o.queueInformers {
			hasSyncedCheckFns = append(hasSyncedCheckFns, queueInformer.informer.HasSynced)
		}
		for _, informer := range o.informers {
			hasSyncedCheckFns = append(hasSyncedCheckFns, informer.HasSynced)
		}

		select {
		case err := <-errChan:
			if err != nil {
				o.Log.Infof("operator not ready: %s", err.Error())
				return
			}
			o.Log.Info("operator ready")
		case <-stopc:
			return
		}

		o.Log.Info("starting informers...")
		{
			started := make(map[cache.SharedIndexInformer]struct{})
			for _, queueInformer := range o.queueInformers {
				if _, ok := started[queueInformer.informer]; !ok {
					go queueInformer.informer.Run(stopc)
				}
				started[queueInformer.informer] = struct{}{}
			}

			for _, informer := range o.informers {
				if _, ok := started[informer]; !ok {
					go informer.Run(stopc)
				}
				started[informer] = struct{}{}
			}
		}

		o.Log.Info("waiting for caches to sync...")
		if ok := cache.WaitForCacheSync(stopc, hasSyncedCheckFns...); !ok {
			o.Log.Info("failed to wait for caches to sync")
			return
		}

		o.Log.Info("starting workers...")
		for _, queueInformer := range o.queueInformers {
			go o.worker(queueInformer)
			go o.worker(queueInformer)
		}

		for _, queueIndexer := range o.queueIndexers {
			go o.indexerWorker(queueIndexer)
			go o.indexerWorker(queueIndexer)
		}
		ready <- struct{}{}
		<-stopc
	}()

	return
}

// worker runs a worker thread that just dequeues items, processes them, and marks them done.
// It enforces that the syncHandler is never invoked concurrently with the same key.
func (o *Operator) worker(loop *QueueInformer) {
	for o.processNextWorkItem(loop) {
	}
}

func (o *Operator) processNextWorkItem(loop *QueueInformer) bool {
	queue := loop.queue
	key, quit := queue.Get()

	if quit {
		return false
	}
	defer queue.Done(key)

	// requeue five times on error
	err := o.sync(loop, key.(string))
	if err != nil && queue.NumRequeues(key.(string)) < 5 {
		o.Log.Infof("retrying %s", key)
		utilruntime.HandleError(errors.Wrap(err, fmt.Sprintf("Sync %q failed", key)))
		queue.AddRateLimited(key)
		return true
	}
	queue.Forget(key)

	select {
	case o.syncCh <- err:
	default:
	}

	if err := loop.HandleMetrics(); err != nil {
		o.Log.Error(err)
	}
	return true
}

func (o *Operator) sync(loop *QueueInformer, key string) error {
	logger := o.Log.WithField("queue", loop.name).WithField("key", key)
	obj, exists, err := loop.informer.GetIndexer().GetByKey(key)
	if err != nil {
		return err
	}
	if !exists {
		// For now, we ignore the case where an object used to exist but no longer does
		logger.Info("couldn't get from queue")
		logger.Debugf("have keys: %v", loop.informer.GetIndexer().ListKeys())
		return nil
	}
	return loop.syncHandler(obj)
}

// This provides the same function as above, but for queues that are not auto-fed by informers.
// indexerWorker runs a worker thread that just dequeues items, processes them, and marks them done.
// It enforces that the syncHandler is never invoked concurrently with the same key.
func (o *Operator) indexerWorker(loop *QueueIndexer) {
	for o.processNextIndexerWorkItem(loop) {
	}
}

func (o *Operator) processNextIndexerWorkItem(loop *QueueIndexer) bool {
	queue := loop.queue
	key, quit := queue.Get()

	if quit {
		return false
	}
	defer queue.Done(key)

	// requeue five times on error
	if err := o.syncIndexer(loop, key.(string)); err != nil && queue.NumRequeues(key.(string)) < 5 {
		o.Log.Infof("retrying %s", key)
		utilruntime.HandleError(errors.Wrap(err, fmt.Sprintf("Sync %q failed", key)))
		queue.AddRateLimited(key)
		return true
	}
	queue.Forget(key)
	if err := loop.HandleMetrics(); err != nil {
		o.Log.Error(err)
	}
	return true
}

func (o *Operator) syncIndexer(loop *QueueIndexer, key string) error {
	logger := o.Log.WithField("queue", loop.name).WithField("key", key)
	namespace, _, err := cache.SplitMetaNamespaceKey(key)
	if err != nil {
		return err
	}

	indexer, ok := loop.indexers[namespace]
	if !ok {
		if indexer, ok = loop.indexers[v1.NamespaceAll]; !ok {
			return fmt.Errorf("no indexer found for %s, have %v", namespace, loop.indexers)
		}
	}
	obj, exists, err := indexer.GetByKey(key)
	if err != nil {
		return err
	}
	if !exists {
		// For now, we ignore the case where an object used to exist but no longer does
		logger.Info("couldn't get from queue")
		logger.Debugf("have keys: %v", indexer.ListKeys())
		return nil
	}
	return loop.syncHandler(obj)
}
<|endoftext|>


<commit_before>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main

import (
	"bytes"
	"context"
	"flag"
	"fmt"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	log "github.com/golang/glog"
	"github.com/google/kne/deploy"
	kexec "github.com/google/kne/os/exec"
	cpb "github.com/google/kne/proto/controller"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/credentials/alts"
	"google.golang.org/grpc/status"
	"k8s.io/client-go/util/homedir"
)

var (
	logOut = &logger{}
	execer = kexec.NewExecer(logOut, logOut)

	defaultKubeCfg            = ""
	defaultMetallbManifestDir = ""
	defaultMeshnetManifestDir = ""
	// Flags.
	port = flag.Int("port", 50051, "Controller server port")
)

func init() {
	if home := homedir.HomeDir(); home != "" {
		defaultKubeCfg = filepath.Join(home, ".kube", "config")
		defaultMeshnetManifestDir = filepath.Join(home, "kne", "manifests", "meshnet", "base")
		defaultMetallbManifestDir = filepath.Join(home, "kne", "manifests", "metallb")
	}
}

type logger struct{}

func (l *logger) Write(p []byte) (int, error) {
	log.Info(string(p))
	return len(p), nil
}

type server struct {
	cpb.UnimplementedTopologyManagerServer
}

func newDeployment(req *cpb.CreateClusterRequest) (*deploy.Deployment, error) {
	d := &deploy.Deployment{}
	switch kind := req.ClusterSpec.(type) {
	case *cpb.CreateClusterRequest_Kind:
		d.Cluster = &deploy.KindSpec{
			Name:                     req.GetKind().Name,
			Recycle:                  req.GetKind().Recycle,
			Version:                  req.GetKind().Version,
			Image:                    req.GetKind().Image,
			Retain:                   req.GetKind().Retain,
			GoogleArtifactRegistries: req.GetKind().GoogleArtifactRegistries,
			ContainerImages:          req.GetKind().ContainerImages,
		}
	default:
		return nil, fmt.Errorf("cluster type not supported: %T", kind)
	}
	switch metallb := req.IngressSpec.(type) {
	case *cpb.CreateClusterRequest_Metallb:
		l := &deploy.MetalLBSpec{}
		var path string
		path = defaultMetallbManifestDir
		if req.GetMetallb().ManifestDir != "" {
			path = req.GetMetallb().ManifestDir
		}
		p, err := validatePath(path)
		if err != nil {
			return nil, fmt.Errorf("failed to validate path %q", path)
		}
		l.ManifestDir = p
		l.IPCount = int(req.GetMetallb().IpCount)
		l.Version = req.GetMetallb().Version
		d.Ingress = l
	default:
		return nil, fmt.Errorf("ingress spec not supported: %T", metallb)
	}
	switch meshnet := req.CniSpec.(type) {
	case *cpb.CreateClusterRequest_Meshnet:
		m := &deploy.MeshnetSpec{}
		var path string
		path = defaultMeshnetManifestDir
		if req.GetMeshnet().ManifestDir != "" {
			path = req.GetMeshnet().ManifestDir
		}
		p, err := validatePath(path)
		if err != nil {
			return nil, fmt.Errorf("failed to validate path %q", path)
		}
		m.Image = req.GetMeshnet().Image
		m.ManifestDir = p
		d.CNI = m
	default:
		return nil, fmt.Errorf("cni type not supported: %T", meshnet)
	}
	return d, nil
}

func (s *server) CreateCluster(ctx context.Context, req *cpb.CreateClusterRequest) (*cpb.CreateClusterResponse, error) {
	log.Infof("Received CreateCluster request: %+v", req)
	d, err := newDeployment(req)
	if err != nil {
		return nil, status.Errorf(codes.InvalidArgument, "unable to parse request: %v", err)
	}
	log.Infof("Parsed request into deployment: %v", d)
	if err := d.Deploy(ctx, defaultKubeCfg); err != nil {
		resp := &cpb.CreateClusterResponse{
			Name:  req.GetKind().Name,
			State: cpb.ClusterState_CLUSTER_STATE_ERROR,
		}
		return resp, status.Errorf(codes.Internal, "failed to deploy cluster: %v", err)
	}
	log.Infof("Cluster %q deployed and ready for topology", req.GetKind().Name)
	resp := &cpb.CreateClusterResponse{
		Name:  req.GetKind().Name,
		State: cpb.ClusterState_CLUSTER_STATE_RUNNING,
	}
	return resp, nil
}

func (s *server) DeleteCluster(ctx context.Context, req *cpb.DeleteClusterRequest) (*cpb.DeleteClusterResponse, error) {
	log.Infof("Received DeleteCluster request: %+v", req)
	if _, err := exec.LookPath("kind"); err != nil {
		return nil, status.Errorf(codes.Internal, "kind cli not installed on host")
	}
	var b bytes.Buffer
	execer.SetStdout(&b)
	if err := execer.Exec("kind", "get", "clusters"); err != nil {
		return nil, status.Errorf(codes.NotFound, "cannot check for existence of kind cluster")
	}
	execer.SetStdout(logOut)
	clusters := strings.Split(b.String(), "\n")
	found := false
	for _, c := range clusters {
		if c == req.GetName() {
			found = true
			break
		}
	}
	if !found {
		return nil, status.Errorf(codes.NotFound, "cluster does not exist, or is not a kind cluster")
	}
	args := []string{"delete", "cluster"}
	if req.GetName() != "" {
		args = append(args, "--name", req.GetName())
	}
	if err := execer.Exec("kind", args...); err != nil {
		return nil, status.Errorf(codes.Internal, "failed to delete cluster using cli")
	}
	log.Infof("Deleted kind cluster %q", req.GetName())
	return &cpb.DeleteClusterResponse{}, nil
}

func validatePath(path string) (string, error) {
	path, err := filepath.Abs(path)
	if err != nil {
		return "", fmt.Errorf("failed to evaluate absolute path: %q", path)
	}
	if _, err := os.Stat(path); err != nil {
		return "", fmt.Errorf("path %q does not exist", path)
	}
	return path, nil
}

func main() {
	flag.Parse()
	addr := fmt.Sprintf(":%d", *port)
	lis, err := net.Listen("tcp6", addr)
	if err != nil {
		log.Fatalf("failed to listen: %v", err)
	}
	creds := alts.NewServerCreds(alts.DefaultServerOptions())
	s := grpc.NewServer(grpc.Creds(creds))
	cpb.RegisterTopologyManagerServer(s, &server{})
	log.Infof("Controller server listening at %v", lis.Addr())
	if err := s.Serve(lis); err != nil {
		log.Fatalf("failed to serve: %v", err)
	}
}
<commit_msg>add logging<commit_after>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main

import (
	"bytes"
	"context"
	"flag"
	"fmt"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	log "github.com/golang/glog"
	"github.com/google/kne/deploy"
	kexec "github.com/google/kne/os/exec"
	cpb "github.com/google/kne/proto/controller"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/credentials/alts"
	"google.golang.org/grpc/status"
	"k8s.io/client-go/util/homedir"
)

var (
	logOut = &logger{}
	execer = kexec.NewExecer(logOut, logOut)

	defaultKubeCfg            = ""
	defaultMetallbManifestDir = ""
	defaultMeshnetManifestDir = ""
	// Flags.
	port = flag.Int("port", 50051, "Controller server port")
)

func init() {
	if home := homedir.HomeDir(); home != "" {
		defaultKubeCfg = filepath.Join(home, ".kube", "config")
		defaultMeshnetManifestDir = filepath.Join(home, "kne", "manifests", "meshnet", "base")
		defaultMetallbManifestDir = filepath.Join(home, "kne", "manifests", "metallb")
	}
}

type logger struct{}

func (l *logger) Write(p []byte) (int, error) {
	log.Info(string(p))
	return len(p), nil
}

type server struct {
	cpb.UnimplementedTopologyManagerServer
}

func newDeployment(req *cpb.CreateClusterRequest) (*deploy.Deployment, error) {
	d := &deploy.Deployment{}
	switch kind := req.ClusterSpec.(type) {
	case *cpb.CreateClusterRequest_Kind:
		d.Cluster = &deploy.KindSpec{
			Name:                     req.GetKind().Name,
			Recycle:                  req.GetKind().Recycle,
			Version:                  req.GetKind().Version,
			Image:                    req.GetKind().Image,
			Retain:                   req.GetKind().Retain,
			GoogleArtifactRegistries: req.GetKind().GoogleArtifactRegistries,
			ContainerImages:          req.GetKind().ContainerImages,
		}
	default:
		return nil, fmt.Errorf("cluster type not supported: %T", kind)
	}
	switch metallb := req.IngressSpec.(type) {
	case *cpb.CreateClusterRequest_Metallb:
		l := &deploy.MetalLBSpec{}
		var path string
		path = defaultMetallbManifestDir
		if req.GetMetallb().ManifestDir != "" {
			path = req.GetMetallb().ManifestDir
		}
		p, err := validatePath(path)
		if err != nil {
			return nil, fmt.Errorf("failed to validate path %q", path)
		}
		l.ManifestDir = p
		l.IPCount = int(req.GetMetallb().IpCount)
		l.Version = req.GetMetallb().Version
		d.Ingress = l
	default:
		return nil, fmt.Errorf("ingress spec not supported: %T", metallb)
	}
	switch meshnet := req.CniSpec.(type) {
	case *cpb.CreateClusterRequest_Meshnet:
		m := &deploy.MeshnetSpec{}
		var path string
		path = defaultMeshnetManifestDir
		if req.GetMeshnet().ManifestDir != "" {
			path = req.GetMeshnet().ManifestDir
		}
		p, err := validatePath(path)
		if err != nil {
			return nil, fmt.Errorf("failed to validate path %q", path)
		}
		m.Image = req.GetMeshnet().Image
		m.ManifestDir = p
		d.CNI = m
	default:
		return nil, fmt.Errorf("cni type not supported: %T", meshnet)
	}
	return d, nil
}

func (s *server) CreateCluster(ctx context.Context, req *cpb.CreateClusterRequest) (*cpb.CreateClusterResponse, error) {
	log.Infof("Received CreateCluster request: %+v", req)
	d, err := newDeployment(req)
	if err != nil {
		return nil, status.Errorf(codes.InvalidArgument, "unable to parse request: %v", err)
	}
	log.Infof("Parsed request into deployment: %v", d)
	if err := d.Deploy(ctx, defaultKubeCfg); err != nil {
		resp := &cpb.CreateClusterResponse{
			Name:  req.GetKind().Name,
			State: cpb.ClusterState_CLUSTER_STATE_ERROR,
		}
		return resp, status.Errorf(codes.Internal, "failed to deploy cluster: %v", err)
	}
	log.Infof("Cluster %q deployed and ready for topology", req.GetKind().Name)
	resp := &cpb.CreateClusterResponse{
		Name:  req.GetKind().Name,
		State: cpb.ClusterState_CLUSTER_STATE_RUNNING,
	}
	return resp, nil
}

func (s *server) DeleteCluster(ctx context.Context, req *cpb.DeleteClusterRequest) (*cpb.DeleteClusterResponse, error) {
	log.Infof("Received DeleteCluster request: %+v", req)
	if _, err := exec.LookPath("kind"); err != nil {
		return nil, status.Errorf(codes.Internal, "kind cli not installed on host")
	}
	var b bytes.Buffer
	execer.SetStdout(&b)
	if err := execer.Exec("kind", "get", "clusters"); err != nil {
		return nil, status.Errorf(codes.NotFound, "cannot check for existence of kind cluster")
	}
	execer.SetStdout(logOut)
	clusters := strings.Split(b.String(), "\n")
	found := false
	for _, c := range clusters {
		if c == req.GetName() {
			found = true
			break
		}
	}
	if !found {
		log.Infof("Cluster %q does not exist, or is not a kind cluster (%v)", req.GetName(), clusters)
		return nil, status.Errorf(codes.NotFound, "cluster %q does not exist, or is not a kind cluster (%v)", req.GetName(), clusters)
	}
	args := []string{"delete", "cluster"}
	if req.GetName() != "" {
		args = append(args, "--name", req.GetName())
	}
	if err := execer.Exec("kind", args...); err != nil {
		return nil, status.Errorf(codes.Internal, "failed to delete cluster using cli")
	}
	log.Infof("Deleted kind cluster %q", req.GetName())
	return &cpb.DeleteClusterResponse{}, nil
}

func validatePath(path string) (string, error) {
	path, err := filepath.Abs(path)
	if err != nil {
		return "", fmt.Errorf("failed to evaluate absolute path: %q", path)
	}
	if _, err := os.Stat(path); err != nil {
		return "", fmt.Errorf("path %q does not exist", path)
	}
	return path, nil
}

func main() {
	flag.Parse()
	addr := fmt.Sprintf(":%d", *port)
	lis, err := net.Listen("tcp6", addr)
	if err != nil {
		log.Fatalf("failed to listen: %v", err)
	}
	creds := alts.NewServerCreds(alts.DefaultServerOptions())
	s := grpc.NewServer(grpc.Creds(creds))
	cpb.RegisterTopologyManagerServer(s, &server{})
	log.Infof("Controller server listening at %v", lis.Addr())
	if err := s.Serve(lis); err != nil {
		log.Fatalf("failed to serve: %v", err)
	}
}
<|endoftext|>