<commit_before>package zkmeta

import (
	"strings"
	"sync"
	"time"

	"github.com/funkygao/gafka/cmd/kateway/meta"
	"github.com/funkygao/gafka/zk"
	log "github.com/funkygao/log4go"
	zklib "github.com/samuel/go-zookeeper/zk"
)

type zkMetaStore struct {
	cf *config
	mu sync.RWMutex

	shutdownCh chan struct{}
	refreshCh  chan struct{} // deadlock if no receiver

	zkzone *zk.ZkZone

	// cache
	brokerList map[string][]string      // key is cluster name
	clusters   map[string]*zk.ZkCluster // key is cluster name

	// cache
	partitionsMap map[string]map[string][]int32 // {cluster: {topic: partitions}}
	pmapLock      sync.RWMutex
}

func New(cf *config, zkzone *zk.ZkZone) meta.MetaStore {
	return &zkMetaStore{
		cf:         cf,
		zkzone:     zkzone,
		shutdownCh: make(chan struct{}),
		refreshCh:  make(chan struct{}, 5),

		brokerList:    make(map[string][]string),
		clusters:      make(map[string]*zk.ZkCluster),
		partitionsMap: make(map[string]map[string][]int32),
	}
}

func (this *zkMetaStore) Name() string {
	return "zk"
}

func (this *zkMetaStore) RefreshEvent() <-chan struct{} {
	return this.refreshCh
}

func (this *zkMetaStore) refreshTopologyCache() {
	// refresh live clusters from Zookeeper
	liveClusters := this.zkzone.Clusters()

	this.mu.Lock()

	// add new live clusters if not present in my cache
	for cluster, path := range liveClusters {
		if _, present := this.clusters[cluster]; !present {
			this.clusters[cluster] = this.zkzone.NewclusterWithPath(cluster, path)
		}

		this.brokerList[cluster] = this.clusters[cluster].BrokerList()
	}

	// remove dead clusters
	cachedClusters := this.clusters
	for cluster, _ := range cachedClusters {
		if _, present := liveClusters[cluster]; !present {
			delete(this.clusters, cluster)
			delete(this.brokerList, cluster)
		}
	}

	this.mu.Unlock()
}

func (this *zkMetaStore) Start() {
	// warm up
	this.refreshTopologyCache()

	go func() {
		ticker := time.NewTicker(this.cf.Refresh)
		defer ticker.Stop()

		booting := true
		for {
			select {
			case <-ticker.C:
				log.Debug("refreshing zk meta store")

				this.refreshTopologyCache()

				// clear the partition cache
				this.pmapLock.Lock()
				this.partitionsMap = make(map[string]map[string][]int32,
					len(this.partitionsMap))
				this.pmapLock.Unlock()

				// notify others that I have got the most recent data
				this.refreshCh <- struct{}{}

			case <-this.shutdownCh:
				return

			case evt := <-this.zkzone.SessionEvents():
				// after zk conn lost, zklib will automatically reconnect:
				// StateConnecting -> StateConnected -> StateHasSession
				if evt.State == zklib.StateHasSession {
					if booting {
						booting = false
					} else {
						log.Warn("zk reconnected after session lost")
					}
				}
			}
		}
	}()
}

func (this *zkMetaStore) Stop() {
	this.mu.Lock()
	defer this.mu.Unlock()

	for _, c := range this.clusters {
		c.Close()
	}

	close(this.shutdownCh)
}

func (this *zkMetaStore) OnlineConsumersCount(cluster, topic, group string) int {
	// without cache
	this.mu.Lock()
	c, present := this.clusters[cluster]
	this.mu.Unlock()

	if !present {
		log.Warn("invalid cluster: %s", cluster)
		return 0
	}

	// FIXME will always lookup zk
	return c.OnlineConsumersCount(topic, group)
}

func (this *zkMetaStore) KatewayDisqueAddrs() (map[string][]string, error) {
	return this.zkzone.KatewayDisqueAddrs()
}

func (this *zkMetaStore) TopicPartitions(cluster, topic string) []int32 {
	clusterNotPresent := true

	this.pmapLock.RLock()
	if c, present := this.partitionsMap[cluster]; present {
		clusterNotPresent = false
		if p, present := c[topic]; present {
			this.pmapLock.RUnlock()
			return p
		}
	}
	this.pmapLock.RUnlock()

	// cache miss
	this.mu.RLock()
	c, ok := this.clusters[cluster]
	this.mu.RUnlock()
	if !ok {
		log.Warn("invalid cluster: %s", cluster)
		return nil
	}

	p := c.Partitions(topic)

	this.pmapLock.Lock()
	if clusterNotPresent {
		this.partitionsMap[cluster] = make(map[string][]int32)
	}
	this.partitionsMap[cluster][topic] = p
	this.pmapLock.Unlock()

	return p
}

func (this *zkMetaStore) BrokerList(cluster string) []string {
	this.mu.RLock()
	r := this.brokerList[cluster]
	this.mu.RUnlock()
	return r
}

func (this *zkMetaStore) ZkAddrs() []string {
	return strings.Split(this.zkzone.ZkAddrs(), ",")
}

func (this *zkMetaStore) ZkChroot(cluster string) string {
	this.mu.RLock()
	c, ok := this.clusters[cluster]
	this.mu.RUnlock()

	if ok {
		return c.Chroot()
	} else {
		return ""
	}
}

func (this *zkMetaStore) Clusters() []map[string]string {
	r := make([]map[string]string, 0)

	this.mu.RLock()
	defer this.mu.RUnlock()

	this.zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		info := zkcluster.RegisteredInfo()
		if !info.Public || info.Nickname == "" {
			// ignored for kateway manager
			return
		}

		c := make(map[string]string)
		c["name"] = info.Name()
		c["nickname"] = info.Nickname
		r = append(r, c)
	})
	return r
}

func (this *zkMetaStore) ClusterNames() []string {
	this.mu.RLock()
	r := make([]string, 0, len(this.clusters))
	for name, _ := range this.clusters {
		r = append(r, name)
	}
	this.mu.RUnlock()
	return r
}

func (this *zkMetaStore) ZkCluster(cluster string) *zk.ZkCluster {
	this.mu.RLock()
	r, ok := this.clusters[cluster]
	this.mu.RUnlock()

	if !ok {
		log.Warn("invalid cluster: %s", cluster)
		return nil
	}

	return r
}
<commit_msg>meta's zkzone is passed in, shouldn't close inside meta pkg<commit_after>package zkmeta

import (
	"strings"
	"sync"
	"time"

	"github.com/funkygao/gafka/cmd/kateway/meta"
	"github.com/funkygao/gafka/zk"
	log "github.com/funkygao/log4go"
	zklib "github.com/samuel/go-zookeeper/zk"
)

type zkMetaStore struct {
	cf *config
	mu sync.RWMutex

	shutdownCh chan struct{}
	refreshCh  chan struct{} // deadlock if no receiver

	zkzone *zk.ZkZone

	// cache
	brokerList map[string][]string      // key is cluster name
	clusters   map[string]*zk.ZkCluster // key is cluster name

	// cache
	partitionsMap map[string]map[string][]int32 // {cluster: {topic: partitions}}
	pmapLock      sync.RWMutex
}

func New(cf *config, zkzone *zk.ZkZone) meta.MetaStore {
	return &zkMetaStore{
		cf:         cf,
		zkzone:     zkzone,
		shutdownCh: make(chan struct{}),
		refreshCh:  make(chan struct{}, 5),

		brokerList:    make(map[string][]string),
		clusters:      make(map[string]*zk.ZkCluster),
		partitionsMap: make(map[string]map[string][]int32),
	}
}

func (this *zkMetaStore) Name() string {
	return "zk"
}

func (this *zkMetaStore) RefreshEvent() <-chan struct{} {
	return this.refreshCh
}

func (this *zkMetaStore) refreshTopologyCache() {
	// refresh live clusters from Zookeeper
	liveClusters := this.zkzone.Clusters()

	this.mu.Lock()

	// add new live clusters if not present in my cache
	for cluster, path := range liveClusters {
		if _, present := this.clusters[cluster]; !present {
			this.clusters[cluster] = this.zkzone.NewclusterWithPath(cluster, path)
		}

		this.brokerList[cluster] = this.clusters[cluster].BrokerList()
	}

	// remove dead clusters
	cachedClusters := this.clusters
	for cluster, _ := range cachedClusters {
		if _, present := liveClusters[cluster]; !present {
			delete(this.clusters, cluster)
			delete(this.brokerList, cluster)
		}
	}

	this.mu.Unlock()
}

func (this *zkMetaStore) Start() {
	// warm up
	this.refreshTopologyCache()

	go func() {
		ticker := time.NewTicker(this.cf.Refresh)
		defer ticker.Stop()

		booting := true
		for {
			select {
			case <-ticker.C:
				log.Debug("refreshing zk meta store")

				this.refreshTopologyCache()

				// clear the partition cache
				this.pmapLock.Lock()
				this.partitionsMap = make(map[string]map[string][]int32,
					len(this.partitionsMap))
				this.pmapLock.Unlock()

				// notify others that I have got the most recent data
				this.refreshCh <- struct{}{}

			case <-this.shutdownCh:
				return

			case evt := <-this.zkzone.SessionEvents():
				// after zk conn lost, zklib will automatically reconnect:
				// StateConnecting -> StateConnected -> StateHasSession
				if evt.State == zklib.StateHasSession {
					if booting {
						booting = false
					} else {
						log.Warn("zk reconnected after session lost")
					}
				}
			}
		}
	}()
}

func (this *zkMetaStore) Stop() {
	this.mu.Lock()
	defer this.mu.Unlock()

	close(this.shutdownCh)
}

func (this *zkMetaStore) OnlineConsumersCount(cluster, topic, group string) int {
	// without cache
	this.mu.Lock()
	c, present := this.clusters[cluster]
	this.mu.Unlock()

	if !present {
		log.Warn("invalid cluster: %s", cluster)
		return 0
	}

	// FIXME will always lookup zk
	return c.OnlineConsumersCount(topic, group)
}

func (this *zkMetaStore) KatewayDisqueAddrs() (map[string][]string, error) {
	return this.zkzone.KatewayDisqueAddrs()
}

func (this *zkMetaStore) TopicPartitions(cluster, topic string) []int32 {
	clusterNotPresent := true

	this.pmapLock.RLock()
	if c, present := this.partitionsMap[cluster]; present {
		clusterNotPresent = false
		if p, present := c[topic]; present {
			this.pmapLock.RUnlock()
			return p
		}
	}
	this.pmapLock.RUnlock()

	// cache miss
	this.mu.RLock()
	c, ok := this.clusters[cluster]
	this.mu.RUnlock()
	if !ok {
		log.Warn("invalid cluster: %s", cluster)
		return nil
	}

	p := c.Partitions(topic)

	this.pmapLock.Lock()
	if clusterNotPresent {
		this.partitionsMap[cluster] = make(map[string][]int32)
	}
	this.partitionsMap[cluster][topic] = p
	this.pmapLock.Unlock()

	return p
}

func (this *zkMetaStore) BrokerList(cluster string) []string {
	this.mu.RLock()
	r := this.brokerList[cluster]
	this.mu.RUnlock()
	return r
}

func (this *zkMetaStore) ZkAddrs() []string {
	return strings.Split(this.zkzone.ZkAddrs(), ",")
}

func (this *zkMetaStore) ZkChroot(cluster string) string {
	this.mu.RLock()
	c, ok := this.clusters[cluster]
	this.mu.RUnlock()

	if ok {
		return c.Chroot()
	} else {
		return ""
	}
}

func (this *zkMetaStore) Clusters() []map[string]string {
	r := make([]map[string]string, 0)

	this.mu.RLock()
	defer this.mu.RUnlock()

	this.zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		info := zkcluster.RegisteredInfo()
		if !info.Public || info.Nickname == "" {
			// ignored for kateway manager
			return
		}

		c := make(map[string]string)
		c["name"] = info.Name()
		c["nickname"] = info.Nickname
		r = append(r, c)
	})
	return r
}

func (this *zkMetaStore) ClusterNames() []string {
	this.mu.RLock()
	r := make([]string, 0, len(this.clusters))
	for name, _ := range this.clusters {
		r = append(r, name)
	}
	this.mu.RUnlock()
	return r
}

func (this *zkMetaStore) ZkCluster(cluster string) *zk.ZkCluster {
	this.mu.RLock()
	r, ok := this.clusters[cluster]
	this.mu.RUnlock()

	if !ok {
		log.Warn("invalid cluster: %s", cluster)
		return nil
	}

	return r
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"github.com/uoregon-libraries/rais-image-server/iiif"
	"github.com/uoregon-libraries/rais-image-server/openjpeg"
	"github.com/uoregon-libraries/rais-image-server/transform"
	"image"
	"log"
	"os"
	"path"
)

var (
	ErrImageDoesNotExist = errors.New("Image file does not exist")
	ErrInvalidFiletype   = errors.New("Invalid or unknown file type")
	ErrDecodeImage       = errors.New("Unable to decode image")
)

// IIIFImage defines an interface for reading images in a generic way.  It's
// heavily biased toward the way we've had to do our JP2 images since they're
// the more unusual use-case.
type IIIFImage interface {
	CleanupResources()
	DecodeImage()            (image.Image, error)
	GetDimensions()          (image.Rectangle, error)
	SetCrop(image.Rectangle)
	SetResizeWH(int, int)
	SetScale(float64)
}

type ImageResource struct {
	Image    IIIFImage
	ID       iiif.ID
	FilePath string
}

// Initializes and returns an IIIFImage for the given id and path.  If the path
// doesn't resolve to a valid file, or resolves to a file type that isn't
// supported, an error is returned.  File type is determined by extension, so
// images will need standard extensions in order to work.
func NewImageResource(id iiif.ID, filepath string) (*ImageResource, error) {
	var err error

	// First, does the file exist?
	if _, err = os.Stat(filepath); err != nil {
		log.Printf("Image does not exist: %#v", filepath)
		return nil, ErrImageDoesNotExist
	}

	// File exists - is it a valid filetype?
	var i IIIFImage
	fileExt := path.Ext(filepath)
	switch fileExt {
	case ".jp2":
		i, err = openjpeg.NewJP2Image(filepath)
	default:
		log.Printf("Image type unknown / invalid: %#v", filepath)
		return nil, ErrInvalidFiletype
	}

	if err != nil {
		log.Printf("Unable to read image %#v: %s", filepath)
		return nil, errors.New(fmt.Sprintf("Unable to read image %#v: %s", id, err))
	}

	img := &ImageResource{ID: id, Image: i, FilePath: filepath}
	return img, nil
}

// Apply runs all image manipulation operations described by the IIIF URL, and
// returns an image.Image ready for encoding to the client
func (res *ImageResource) Apply(u *iiif.URL) (image.Image, error) {
	// Crop and resize have to be prepared before we can decode
	res.prepCrop(u.Region)
	res.prepResize(u.Size)

	img, err := res.Image.DecodeImage()
	if err != nil {
		log.Println("Unable to decode image: ", err)
		return nil, ErrDecodeImage
	}

	if u.Rotation.Degrees != 0 {
		img = rotate(img, u.Rotation)
	}

	return img, nil
}

func (res *ImageResource) prepCrop(r iiif.Region) {
	if r.Type == iiif.RTPixel {
		rect := image.Rect(int(r.X), int(r.Y), int(r.X+r.W), int(r.Y+r.H))
		res.Image.SetCrop(rect)
	}
}

func (res *ImageResource) prepResize(s iiif.Size) {
	switch s.Type {
	case iiif.STScaleToWidth:
		res.Image.SetResizeWH(s.W, 0)
	case iiif.STScaleToHeight:
		res.Image.SetResizeWH(0, s.H)
	case iiif.STExact:
		res.Image.SetResizeWH(s.W, s.H)
	case iiif.STScalePercent:
		res.Image.SetScale(s.Percent / 100.0)
	}
}

func rotate(img image.Image, rot iiif.Rotation) image.Image {
	var r transform.Rotator
	switch img0 := img.(type) {
	case *image.Gray:
		r = transform.GrayRotator{img0}
	case *image.RGBA:
		r = transform.RGBARotator{img0}
	}

	switch rot.Degrees {
	case 90:
		img = r.Rotate90()
	case 180:
		img = r.Rotate180()
	case 270:
		img = r.Rotate270()
	}

	return img
}
<commit_msg>Make file extension case-insensitive<commit_after>package main

import (
	"errors"
	"fmt"
	"github.com/uoregon-libraries/rais-image-server/iiif"
	"github.com/uoregon-libraries/rais-image-server/openjpeg"
	"github.com/uoregon-libraries/rais-image-server/transform"
	"image"
	"log"
	"os"
	"path"
	"strings"
)

var (
	ErrImageDoesNotExist = errors.New("Image file does not exist")
	ErrInvalidFiletype   = errors.New("Invalid or unknown file type")
	ErrDecodeImage       = errors.New("Unable to decode image")
)

// IIIFImage defines an interface for reading images in a generic way.  It's
// heavily biased toward the way we've had to do our JP2 images since they're
// the more unusual use-case.
type IIIFImage interface {
	CleanupResources()
	DecodeImage()            (image.Image, error)
	GetDimensions()          (image.Rectangle, error)
	SetCrop(image.Rectangle)
	SetResizeWH(int, int)
	SetScale(float64)
}

type ImageResource struct {
	Image    IIIFImage
	ID       iiif.ID
	FilePath string
}

// Initializes and returns an IIIFImage for the given id and path.  If the path
// doesn't resolve to a valid file, or resolves to a file type that isn't
// supported, an error is returned.  File type is determined by extension, so
// images will need standard extensions in order to work.
func NewImageResource(id iiif.ID, filepath string) (*ImageResource, error) {
	var err error

	// First, does the file exist?
	if _, err = os.Stat(filepath); err != nil {
		log.Printf("Image does not exist: %#v", filepath)
		return nil, ErrImageDoesNotExist
	}

	// File exists - is it a valid filetype?
	var i IIIFImage
	fileExt := strings.ToLower(path.Ext(filepath))
	switch fileExt {
	case ".jp2":
		i, err = openjpeg.NewJP2Image(filepath)
	default:
		log.Printf("Image type unknown / invalid: %#v", filepath)
		return nil, ErrInvalidFiletype
	}

	if err != nil {
		log.Printf("Unable to read image %#v: %s", filepath)
		return nil, errors.New(fmt.Sprintf("Unable to read image %#v: %s", id, err))
	}

	img := &ImageResource{ID: id, Image: i, FilePath: filepath}
	return img, nil
}

// Apply runs all image manipulation operations described by the IIIF URL, and
// returns an image.Image ready for encoding to the client
func (res *ImageResource) Apply(u *iiif.URL) (image.Image, error) {
	// Crop and resize have to be prepared before we can decode
	res.prepCrop(u.Region)
	res.prepResize(u.Size)

	img, err := res.Image.DecodeImage()
	if err != nil {
		log.Println("Unable to decode image: ", err)
		return nil, ErrDecodeImage
	}

	if u.Rotation.Degrees != 0 {
		img = rotate(img, u.Rotation)
	}

	return img, nil
}

func (res *ImageResource) prepCrop(r iiif.Region) {
	if r.Type == iiif.RTPixel {
		rect := image.Rect(int(r.X), int(r.Y), int(r.X+r.W), int(r.Y+r.H))
		res.Image.SetCrop(rect)
	}
}

func (res *ImageResource) prepResize(s iiif.Size) {
	switch s.Type {
	case iiif.STScaleToWidth:
		res.Image.SetResizeWH(s.W, 0)
	case iiif.STScaleToHeight:
		res.Image.SetResizeWH(0, s.H)
	case iiif.STExact:
		res.Image.SetResizeWH(s.W, s.H)
	case iiif.STScalePercent:
		res.Image.SetScale(s.Percent / 100.0)
	}
}

func rotate(img image.Image, rot iiif.Rotation) image.Image {
	var r transform.Rotator
	switch img0 := img.(type) {
	case *image.Gray:
		r = transform.GrayRotator{img0}
	case *image.RGBA:
		r = transform.RGBARotator{img0}
	}

	switch rot.Degrees {
	case 90:
		img = r.Rotate90()
	case 180:
		img = r.Rotate180()
	case 270:
		img = r.Rotate270()
	}

	return img
}
<|endoftext|>


<commit_before>package commands

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net"
	"net/http"
	"net/url"
	"path"
	"strconv"
	"time"

	"github.com/go-openapi/loads"
	"github.com/go-openapi/runtime/middleware"
	"github.com/go-openapi/swag"
	"github.com/gorilla/handlers"
	"github.com/toqueteos/webbrowser"
	"github.com/tylerb/graceful"
)

// ServeCmd to serve a swagger spec with docs ui
type ServeCmd struct {
	BasePath string `long:"base-path" description:"the base path to serve the spec and UI at"`
	Flavor   string `short:"F" long:"flavor" description:"the flavor of docs, can be swagger or redoc" default:"redoc" choice:"redoc" choice:"swagger"`
	DocURL   string `long:"doc-url" description:"override the url which takes a url query param to render the doc ui"`
	NoOpen   bool   `long:"no-open" description:"when present won't open the the browser to show the url"`
	NoUI     bool   `long:"no-ui" description:"when present, only the swagger spec will be served"`
	Port     int    `long:"port" short:"p" description:"the port to serve this site" env:"PORT"`
	Host     string `long:"host" description:"the interface to serve this site, defaults to 0.0.0.0" env:"HOST"`
}

// Execute the serve command
func (s *ServeCmd) Execute(args []string) error {
	if len(args) == 0 {
		return errors.New("specify the spec to serve as argument to the serve command")
	}

	specDoc, err := loads.Spec(args[0])
	if err != nil {
		return err
	}
	b, err := json.MarshalIndent(specDoc.Spec(), "", "  ")
	if err != nil {
		return err
	}

	basePath := s.BasePath
	if basePath == "" {
		basePath = "/"
	}

	listener, err := net.Listen("tcp", net.JoinHostPort(s.Host, strconv.Itoa(s.Port)))
	if err != nil {
		return err
	}
	sh, sp, err := swag.SplitHostPort(listener.Addr().String())
	if err != nil {
		return err
	}
	if sh == "0.0.0.0" {
		sh = "localhost"
	}

	visit := s.DocURL
	handler := http.NotFoundHandler()
	if !s.NoUI {
		if s.Flavor == "redoc" {
			handler = middleware.Redoc(middleware.RedocOpts{
				BasePath: basePath,
				SpecURL:  path.Join(basePath, "swagger.json"),
				Path:     "docs",
			}, handler)
			visit = fmt.Sprintf("http://%s:%d%s", sh, sp, path.Join(basePath, "docs"))
		} else if visit != "" || s.Flavor == "swagger" {
			if visit == "" {
				visit = "http://petstore.swagger.io/"
			}
			u, err := url.Parse(visit)
			if err != nil {
				return err
			}
			q := u.Query()
			q.Add("url", fmt.Sprintf("http://%s:%d%s", sh, sp, path.Join(basePath, "swagger.json")))
			u.RawQuery = q.Encode()
			visit = u.String()
		}
	}

	handler = handlers.CORS()(middleware.Spec(basePath, b, handler))
	errFuture := make(chan error)
	go func() {
		docServer := &graceful.Server{Server: new(http.Server)}
		docServer.SetKeepAlivesEnabled(true)
		docServer.TCPKeepAlive = 3 * time.Minute
		docServer.Handler = handler

		errFuture <- docServer.Serve(listener)
	}()

	if !s.NoOpen && !s.NoUI {
		_ = webbrowser.Open(visit)
	}
	log.Println("serving docs at", visit)
	return <-errFuture
}
<commit_msg>use tcp4 to serve spec<commit_after>package commands

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net"
	"net/http"
	"net/url"
	"path"
	"strconv"
	"time"

	"github.com/go-openapi/loads"
	"github.com/go-openapi/runtime/middleware"
	"github.com/go-openapi/swag"
	"github.com/gorilla/handlers"
	"github.com/toqueteos/webbrowser"
	"github.com/tylerb/graceful"
)

// ServeCmd to serve a swagger spec with docs ui
type ServeCmd struct {
	BasePath string `long:"base-path" description:"the base path to serve the spec and UI at"`
	Flavor   string `short:"F" long:"flavor" description:"the flavor of docs, can be swagger or redoc" default:"redoc" choice:"redoc" choice:"swagger"`
	DocURL   string `long:"doc-url" description:"override the url which takes a url query param to render the doc ui"`
	NoOpen   bool   `long:"no-open" description:"when present won't open the the browser to show the url"`
	NoUI     bool   `long:"no-ui" description:"when present, only the swagger spec will be served"`
	Port     int    `long:"port" short:"p" description:"the port to serve this site" env:"PORT"`
	Host     string `long:"host" description:"the interface to serve this site, defaults to 0.0.0.0" env:"HOST"`
}

// Execute the serve command
func (s *ServeCmd) Execute(args []string) error {
	if len(args) == 0 {
		return errors.New("specify the spec to serve as argument to the serve command")
	}

	specDoc, err := loads.Spec(args[0])
	if err != nil {
		return err
	}
	b, err := json.MarshalIndent(specDoc.Spec(), "", "  ")
	if err != nil {
		return err
	}

	basePath := s.BasePath
	if basePath == "" {
		basePath = "/"
	}

	listener, err := net.Listen("tcp4", net.JoinHostPort(s.Host, strconv.Itoa(s.Port)))
	if err != nil {
		return err
	}
	sh, sp, err := swag.SplitHostPort(listener.Addr().String())
	if err != nil {
		return err
	}
	if sh == "0.0.0.0" {
		sh = "localhost"
	}

	visit := s.DocURL
	handler := http.NotFoundHandler()
	if !s.NoUI {
		if s.Flavor == "redoc" {
			handler = middleware.Redoc(middleware.RedocOpts{
				BasePath: basePath,
				SpecURL:  path.Join(basePath, "swagger.json"),
				Path:     "docs",
			}, handler)
			visit = fmt.Sprintf("http://%s:%d%s", sh, sp, path.Join(basePath, "docs"))
		} else if visit != "" || s.Flavor == "swagger" {
			if visit == "" {
				visit = "http://petstore.swagger.io/"
			}
			u, err := url.Parse(visit)
			if err != nil {
				return err
			}
			q := u.Query()
			q.Add("url", fmt.Sprintf("http://%s:%d%s", sh, sp, path.Join(basePath, "swagger.json")))
			u.RawQuery = q.Encode()
			visit = u.String()
		}
	}

	handler = handlers.CORS()(middleware.Spec(basePath, b, handler))
	errFuture := make(chan error)
	go func() {
		docServer := &graceful.Server{Server: new(http.Server)}
		docServer.SetKeepAlivesEnabled(true)
		docServer.TCPKeepAlive = 3 * time.Minute
		docServer.Handler = handler

		errFuture <- docServer.Serve(listener)
	}()

	if !s.NoOpen && !s.NoUI {
		_ = webbrowser.Open(visit)
	}
	log.Println("serving docs at", visit)
	return <-errFuture
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This program manages a zoekt indexing deployment:
// * recycling logs
// * periodically fetching new data.
// * periodically reindexing all git repos.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"math"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/google/zoekt"
	"github.com/google/zoekt/gitindex"
)

const day = time.Hour * 24

func loggedRun(cmd *exec.Cmd) {
	out := &bytes.Buffer{}
	errOut := &bytes.Buffer{}
	cmd.Stdout = out
	cmd.Stderr = errOut

	if err := cmd.Run(); err != nil {
		log.Printf("command %s failed: %v\nOUT: %s\nERR: %s",
			cmd.Args, err, out.String(), errOut.String())
	} else {
		log.Printf("ran successfully %s", cmd.Args)
	}
}

func refresh(repoDir, indexDir, indexConfigFile string, indexFlags []string, fetchInterval time.Duration, cpuFraction float64) {
	// Start with indexing something, so we can start the webserver.
	runIndexCommand(indexDir, repoDir, indexConfigFile, indexFlags, cpuFraction)

	t := time.NewTicker(fetchInterval)
	for {
		repos, err := gitindex.FindGitRepos(repoDir)
		if err != nil {
			log.Println(err)
			continue
		}
		if len(repos) == 0 {
			log.Printf("no repos found under %s", repoDir)
		}
		for _, dir := range repos {
			cmd := exec.Command("git", "--git-dir", dir, "fetch", "origin")
			// Prevent prompting
			cmd.Stdin = &bytes.Buffer{}
			loggedRun(cmd)
		}

		runIndexCommand(indexDir, repoDir, indexConfigFile, indexFlags, cpuFraction)
		<-t.C
	}
}

func repoIndexCommand(indexDir, repoDir string, configs []RepoHostConfig) {
	for _, cfg := range configs {
		cmd := exec.Command("zoekt-repo-index",
			"-parallelism=1",
			"-repo_cache", repoDir,
			"-index", indexDir,
			"-base_url", cfg.BaseURL,
			"-rev_prefix", cfg.RevPrefix,
			"-max_sub_projects=5",
			"-manifest_repo_url", cfg.ManifestRepoURL,
			"-manifest_rev_prefix", cfg.ManifestRevPrefix)

		cmd.Args = append(cmd.Args, cfg.BranchXMLs...)
		loggedRun(cmd)
	}
}

func repositoryOnRepoHost(repoBaseDir, dir string, repoHosts []RepoHostConfig) bool {
	for _, rh := range repoHosts {
		u, _ := url.Parse(rh.BaseURL)

		if strings.HasPrefix(dir, filepath.Join(repoBaseDir, u.Host)) {
			return true
		}
	}
	return false
}

func runIndexCommand(indexDir, repoDir, indexConfigFile string, indexFlags []string, cpuFraction float64) {
	var indexConfig *IndexConfig
	if indexConfigFile != "" {
		var err error
		indexConfig, err = readIndexConfig(indexConfigFile)
		if err != nil {
			log.Printf("index config: %v", err)
		}

		repoIndexCommand(indexDir, repoDir, indexConfig.RepoHosts)
	}

	repos, err := gitindex.FindGitRepos(repoDir)
	if err != nil {
		log.Println("FindGitRepos", err)
		return
	}

	cpuCount := int(math.Trunc(float64(runtime.NumCPU()) * cpuFraction))
	if cpuCount < 1 {
		cpuCount = 1
	}
	for _, dir := range repos {
		if indexConfig != nil {
			// Don't want to index the subrepos of a repo
			// host separately.
			if repositoryOnRepoHost(repoDir, dir, indexConfig.RepoHosts) {
				continue
			}

			// TODO(hanwen): we should have similar
			// functionality for avoiding to index a
			// submodule separately too.
		}

		args := []string{
			"-require_ctags",
			fmt.Sprintf("-parallelism=%d", cpuCount),
			"-repo_cache", repoDir,
			"-index", indexDir,
			"-incremental",
		}
		args = append(args, indexFlags...)
		args = append(args, dir)
		cmd := exec.Command("zoekt-git-index", args...)
		loggedRun(cmd)
	}
}

// deleteLogs deletes old logs.
func deleteLogs(logDir string, maxAge time.Duration) {
	tick := time.NewTicker(maxAge / 100)
	for {
		fs, err := filepath.Glob(filepath.Join(logDir, "*"))
		if err != nil {
			log.Fatalf("filepath.Glob(%s): %v", logDir, err)
		}

		threshold := time.Now().Add(-maxAge)
		for _, fn := range fs {
			if fi, err := os.Lstat(fn); err == nil && fi.ModTime().Before(threshold) {
				os.Remove(fn)
			}
		}
		<-tick.C
	}
}

// Delete the shard if its corresponding git repo can't be found.
func deleteIfStale(repoDir string, fn string) error {
	f, err := os.Open(fn)
	if err != nil {
		return nil
	}
	defer f.Close()

	ifile, err := zoekt.NewIndexFile(f)
	if err != nil {
		return nil
	}
	defer ifile.Close()

	repo, _, err := zoekt.ReadMetadata(ifile)
	if err != nil {
		return nil
	}

	repoDir = gitindex.Path(repoDir, repo.Name)
	_, err = os.Stat(repoDir)
	if os.IsNotExist(err) {
		log.Printf("deleting stale shard %s; dir %q not found", fn, repoDir)
		return os.Remove(fn)
	}

	return err
}

func deleteStaleIndexes(indexDir, repoDir string, watchInterval time.Duration) {
	t := time.NewTicker(watchInterval)

	expr := indexDir + "/*"
	for {
		fs, err := filepath.Glob(expr)
		if err != nil {
			log.Printf("Glob(%q): %v", expr, err)
		}

		for _, f := range fs {
			if err := deleteIfStale(repoDir, f); err != nil {
				log.Printf("deleteIfStale(%q): %v", f, err)
			}
		}
		<-t.C
	}
}

func main() {
	maxLogAge := flag.Duration("max_log_age", 3*day, "recycle index logs after this much time")
	fetchInterval := flag.Duration("fetch_interval", time.Hour, "run fetches this often")
	dataDir := flag.String("data_dir",
		filepath.Join(os.Getenv("HOME"), "zoekt-serving"), "directory holding all data.")
	indexDir := flag.String("index_dir", "", "directory holding index shards. Defaults to $data_dir/index/")
	mirrorConfig := flag.String("mirror_config",
		"", "JSON file holding mirror configuration.")
	indexConfig := flag.String("index_config",
		"", "JSON file holding index configuration.")
	mirrorInterval := flag.Duration("mirror_duration", 24*time.Hour, "find and clone new repos at this frequency.")
	cpuFraction := flag.Float64("cpu_fraction", 0.25,
		"use this fraction of the cores for indexing.")
	indexFlagsStr := flag.String("git_index_flags", "", "space separated list of flags passed through to zoekt-git-index (e.g. -git_index_flags='-symbols=false -submodules=false'")
	flag.Parse()

	if *cpuFraction <= 0.0 || *cpuFraction > 1.0 {
		log.Fatal("cpu_fraction must be between 0.0 and 1.0")
	}
	if *dataDir == "" {
		log.Fatal("must set --data_dir")
	}

	var indexFlags []string
	if *indexFlagsStr != "" {
		indexFlags = strings.Split(*indexFlagsStr, " ")
	}

	// Automatically prepend our own path at the front, to minimize
	// required configuration.
	if l, err := os.Readlink("/proc/self/exe"); err == nil {
		os.Setenv("PATH", filepath.Dir(l)+":"+os.Getenv("PATH"))
	}

	logDir := filepath.Join(*dataDir, "logs")
	if *indexDir == "" {
		*indexDir = filepath.Join(*dataDir, "index")
	}
	repoDir := filepath.Join(*dataDir, "repos")
	for _, s := range []string{logDir, *indexDir, repoDir} {
		if _, err := os.Stat(s); err == nil {
			continue
		}

		if err := os.MkdirAll(s, 0755); err != nil {
			log.Fatalf("MkdirAll %s: %v", s, err)
		}
	}

	if strings.HasPrefix(*mirrorConfig, "https://") || strings.HasPrefix(*mirrorConfig, "http://") {
		_, err := readConfigURL(*mirrorConfig)
		if err != nil {
			log.Fatalf("readConfigURL(%s): %v", *mirrorConfig, err)
		} else {
			go periodicMirrorURL(repoDir, *mirrorConfig, *mirrorInterval)
		}
	} else {
		_, err := readConfigFile(*mirrorConfig)
		if err != nil {
			log.Fatalf("readConfig(%s): %v", *mirrorConfig, err)
		} else {
			go periodicMirrorFile(repoDir, *mirrorConfig, *mirrorInterval)
		}
	}

	go deleteLogs(logDir, *maxLogAge)
	go deleteStaleIndexes(*indexDir, repoDir, *fetchInterval)

	refresh(repoDir, *indexDir, *indexConfig, indexFlags, *fetchInterval, *cpuFraction)
}
<commit_msg>cmd/zoekt-indexserver: detect orphaned shards using Source data<commit_after>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This program manages a zoekt indexing deployment:
// * recycling logs
// * periodically fetching new data.
// * periodically reindexing all git repos.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"math"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/google/zoekt"
	"github.com/google/zoekt/gitindex"
)

const day = time.Hour * 24

func loggedRun(cmd *exec.Cmd) {
	out := &bytes.Buffer{}
	errOut := &bytes.Buffer{}
	cmd.Stdout = out
	cmd.Stderr = errOut

	if err := cmd.Run(); err != nil {
		log.Printf("command %s failed: %v\nOUT: %s\nERR: %s",
			cmd.Args, err, out.String(), errOut.String())
	} else {
		log.Printf("ran successfully %s", cmd.Args)
	}
}

func refresh(repoDir, indexDir, indexConfigFile string, indexFlags []string, fetchInterval time.Duration, cpuFraction float64) {
	// Start with indexing something, so we can start the webserver.
	runIndexCommand(indexDir, repoDir, indexConfigFile, indexFlags, cpuFraction)

	t := time.NewTicker(fetchInterval)
	for {
		repos, err := gitindex.FindGitRepos(repoDir)
		if err != nil {
			log.Println(err)
			continue
		}
		if len(repos) == 0 {
			log.Printf("no repos found under %s", repoDir)
		}
		for _, dir := range repos {
			cmd := exec.Command("git", "--git-dir", dir, "fetch", "origin")
			// Prevent prompting
			cmd.Stdin = &bytes.Buffer{}
			loggedRun(cmd)
		}

		runIndexCommand(indexDir, repoDir, indexConfigFile, indexFlags, cpuFraction)
		<-t.C
	}
}

func repoIndexCommand(indexDir, repoDir string, configs []RepoHostConfig) {
	for _, cfg := range configs {
		cmd := exec.Command("zoekt-repo-index",
			"-parallelism=1",
			"-repo_cache", repoDir,
			"-index", indexDir,
			"-base_url", cfg.BaseURL,
			"-rev_prefix", cfg.RevPrefix,
			"-max_sub_projects=5",
			"-manifest_repo_url", cfg.ManifestRepoURL,
			"-manifest_rev_prefix", cfg.ManifestRevPrefix)

		cmd.Args = append(cmd.Args, cfg.BranchXMLs...)
		loggedRun(cmd)
	}
}

func repositoryOnRepoHost(repoBaseDir, dir string, repoHosts []RepoHostConfig) bool {
	for _, rh := range repoHosts {
		u, _ := url.Parse(rh.BaseURL)

		if strings.HasPrefix(dir, filepath.Join(repoBaseDir, u.Host)) {
			return true
		}
	}
	return false
}

func runIndexCommand(indexDir, repoDir, indexConfigFile string, indexFlags []string, cpuFraction float64) {
	var indexConfig *IndexConfig
	if indexConfigFile != "" {
		var err error
		indexConfig, err = readIndexConfig(indexConfigFile)
		if err != nil {
			log.Printf("index config: %v", err)
		}

		repoIndexCommand(indexDir, repoDir, indexConfig.RepoHosts)
	}

	repos, err := gitindex.FindGitRepos(repoDir)
	if err != nil {
		log.Println("FindGitRepos", err)
		return
	}

	cpuCount := int(math.Trunc(float64(runtime.NumCPU()) * cpuFraction))
	if cpuCount < 1 {
		cpuCount = 1
	}
	for _, dir := range repos {
		if indexConfig != nil {
			// Don't want to index the subrepos of a repo
			// host separately.
			if repositoryOnRepoHost(repoDir, dir, indexConfig.RepoHosts) {
				continue
			}

			// TODO(hanwen): we should have similar
			// functionality for avoiding to index a
			// submodule separately too.
		}

		args := []string{
			"-require_ctags",
			fmt.Sprintf("-parallelism=%d", cpuCount),
			"-repo_cache", repoDir,
			"-index", indexDir,
			"-incremental",
		}
		args = append(args, indexFlags...)
		args = append(args, dir)
		cmd := exec.Command("zoekt-git-index", args...)
		loggedRun(cmd)
	}
}

// deleteLogs deletes old logs.
func deleteLogs(logDir string, maxAge time.Duration) {
	tick := time.NewTicker(maxAge / 100)
	for {
		fs, err := filepath.Glob(filepath.Join(logDir, "*"))
		if err != nil {
			log.Fatalf("filepath.Glob(%s): %v", logDir, err)
		}

		threshold := time.Now().Add(-maxAge)
		for _, fn := range fs {
			if fi, err := os.Lstat(fn); err == nil && fi.ModTime().Before(threshold) {
				os.Remove(fn)
			}
		}
		<-tick.C
	}
}

// Delete the shard if its corresponding git repo can't be found.
func deleteIfStale(repoDir string, fn string) error {
	f, err := os.Open(fn)
	if err != nil {
		return nil
	}
	defer f.Close()

	ifile, err := zoekt.NewIndexFile(f)
	if err != nil {
		return nil
	}
	defer ifile.Close()

	repo, _, err := zoekt.ReadMetadata(ifile)
	if err != nil {
		return nil
	}

	_, err = os.Stat(repo.Source)
	if os.IsNotExist(err) {
		log.Printf("deleting stale shard %s; source %q not found", fn, repo.Source)
		return os.Remove(fn)
	}

	return err
}

func deleteStaleIndexes(indexDir, repoDir string, watchInterval time.Duration) {
	t := time.NewTicker(watchInterval)

	expr := indexDir + "/*"
	for {
		fs, err := filepath.Glob(expr)
		if err != nil {
			log.Printf("Glob(%q): %v", expr, err)
		}

		for _, f := range fs {
			if err := deleteIfStale(repoDir, f); err != nil {
				log.Printf("deleteIfStale(%q): %v", f, err)
			}
		}
		<-t.C
	}
}

func main() {
	maxLogAge := flag.Duration("max_log_age", 3*day, "recycle index logs after this much time")
	fetchInterval := flag.Duration("fetch_interval", time.Hour, "run fetches this often")
	dataDir := flag.String("data_dir",
		filepath.Join(os.Getenv("HOME"), "zoekt-serving"), "directory holding all data.")
	indexDir := flag.String("index_dir", "", "directory holding index shards. Defaults to $data_dir/index/")
	mirrorConfig := flag.String("mirror_config",
		"", "JSON file holding mirror configuration.")
	indexConfig := flag.String("index_config",
		"", "JSON file holding index configuration.")
	mirrorInterval := flag.Duration("mirror_duration", 24*time.Hour, "find and clone new repos at this frequency.")
	cpuFraction := flag.Float64("cpu_fraction", 0.25,
		"use this fraction of the cores for indexing.")
	indexFlagsStr := flag.String("git_index_flags", "", "space separated list of flags passed through to zoekt-git-index (e.g. -git_index_flags='-symbols=false -submodules=false'")
	flag.Parse()

	if *cpuFraction <= 0.0 || *cpuFraction > 1.0 {
		log.Fatal("cpu_fraction must be between 0.0 and 1.0")
	}
	if *dataDir == "" {
		log.Fatal("must set --data_dir")
	}

	var indexFlags []string
	if *indexFlagsStr != "" {
		indexFlags = strings.Split(*indexFlagsStr, " ")
	}

	// Automatically prepend our own path at the front, to minimize
	// required configuration.
	if l, err := os.Readlink("/proc/self/exe"); err == nil {
		os.Setenv("PATH", filepath.Dir(l)+":"+os.Getenv("PATH"))
	}

	logDir := filepath.Join(*dataDir, "logs")
	if *indexDir == "" {
		*indexDir = filepath.Join(*dataDir, "index")
	}
	repoDir := filepath.Join(*dataDir, "repos")
	for _, s := range []string{logDir, *indexDir, repoDir} {
		if _, err := os.Stat(s); err == nil {
			continue
		}

		if err := os.MkdirAll(s, 0755); err != nil {
			log.Fatalf("MkdirAll %s: %v", s, err)
		}
	}

	if strings.HasPrefix(*mirrorConfig, "https://") || strings.HasPrefix(*mirrorConfig, "http://") {
		_, err := readConfigURL(*mirrorConfig)
		if err != nil {
			log.Fatalf("readConfigURL(%s): %v", *mirrorConfig, err)
		} else {
			go periodicMirrorURL(repoDir, *mirrorConfig, *mirrorInterval)
		}
	} else {
		_, err := readConfigFile(*mirrorConfig)
		if err != nil {
			log.Fatalf("readConfig(%s): %v", *mirrorConfig, err)
		} else {
			go periodicMirrorFile(repoDir, *mirrorConfig, *mirrorInterval)
		}
	}

	go deleteLogs(logDir, *maxLogAge)
	go deleteStaleIndexes(*indexDir, repoDir, *fetchInterval)

	refresh(repoDir, *indexDir, *indexConfig, indexFlags, *fetchInterval, *cpuFraction)
}
<|endoftext|>


<commit_before>// Copyright 2019 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// esxiboot executes ESXi kernel over the running kernel.
//
// Synopsis:
//     esxiboot [-d --device] [-c --config] [-r --cdrom]
//
// Description:
//     Loads and executes ESXi kernel.
//
// Options:
//     --config=FILE or -c=FILE: set the ESXi config
//     --device=FILE or -d=FILE: set an ESXi disk to boot from
//     --cdrom=FILE or -r=FILE: set an ESXI CDROM to boot from
//
// --device is required to kexec installed ESXi instance.
// You don't need it if you kexec ESXi installer.
//
// The config file has the following syntax:
//
// kernel=PATH
// kernelopt=OPTS
// modules=MOD1 [ARGS] --- MOD2 [ARGS] --- ...
//
// Lines starting with '#' are ignored.

package main

import (
	"log"
	"os"

	flag "github.com/spf13/pflag"

	"github.com/u-root/u-root/pkg/boot"
	"github.com/u-root/u-root/pkg/boot/esxi"
)

var (
	cfg     = flag.StringP("config", "c", "", "ESXi config file")
	cdrom   = flag.StringP("cdrom", "r", "", "ESXi CDROM boot device")
	diskDev = flag.StringP("device", "d", "", "ESXi disk boot device")
	dryRun  = flag.Bool("dry-run", false, "dry run (just mount + load the kernel, don't kexec)")
)

func main() {
	flag.Parse()
	if *diskDev == "" && *cfg == "" && *cdrom == "" {
		log.Printf("Either --config, --device, or --cdrom must be specified")
		flag.PrintDefaults()
		os.Exit(1)
	}

	if len(*diskDev) > 0 {
		imgs, err := esxi.LoadDisk(*diskDev)
		if err != nil {
			log.Fatalf("Failed to load ESXi configuration: %v", err)
		}

		loaded := false
		for _, img := range imgs {
			if err := img.Load(false); err != nil {
				log.Printf("Failed to load ESXi image (%v) into memory: %v", img, err)
			} else {
				log.Printf("Loaded image: %v", img)
				// We loaded one, that's it.
				loaded = true
				break
			}
		}
		if !loaded {
			log.Fatalf("Failed to load all ESXi images found.")
		}
	} else {
		var err error
		var img *boot.MultibootImage
		if len(*cfg) > 0 {
			img, err = esxi.LoadConfig(*cfg)
		} else if len(*cdrom) > 0 {
			img, err = esxi.LoadCDROM(*cdrom)
		}
		if err != nil {
			log.Fatalf("Failed to load ESXi configuration: %v", err)
		}
		if err := img.Load(false); err != nil {
			log.Fatalf("Failed to load ESXi image (%v) into memory: %v", img, err)
		}
		log.Printf("Loaded image: %v", img)
	}

	if *dryRun {
		log.Printf("Dry run: not booting kernel.")
		os.Exit(0)
	}
	if err := boot.Execute(); err != nil {
		log.Fatalf("Failed to boot image: %v", err)
	}
}
<commit_msg>esxiboot: add --append cmdline option<commit_after>// Copyright 2019 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// esxiboot executes ESXi kernel over the running kernel.
//
// Synopsis:
//     esxiboot [-d --device] [-c --config] [-r --cdrom]
//
// Description:
//     Loads and executes ESXi kernel.
//
// Options:
//     --config=FILE or -c=FILE: set the ESXi config
//     --device=FILE or -d=FILE: set an ESXi disk to boot from
//     --cdrom=FILE or -r=FILE: set an ESXI CDROM to boot from
//     --append: append kernel cmdline arguments
//
// --device is required to kexec installed ESXi instance.
// You don't need it if you kexec ESXi installer.
//
// The config file has the following syntax:
//
// kernel=PATH
// kernelopt=OPTS
// modules=MOD1 [ARGS] --- MOD2 [ARGS] --- ...
//
// Lines starting with '#' are ignored.

package main

import (
	"log"
	"os"
	"strings"

	flag "github.com/spf13/pflag"
	"github.com/u-root/u-root/pkg/boot"
	"github.com/u-root/u-root/pkg/boot/esxi"
)

var (
	cfg           = flag.StringP("config", "c", "", "ESXi config file")
	cdrom         = flag.StringP("cdrom", "r", "", "ESXi CDROM boot device")
	diskDev       = flag.StringP("device", "d", "", "ESXi disk boot device")
	appendCmdline = flag.StringArray("append", nil, "Arguments to append to kernel cmdline")
	dryRun        = flag.Bool("dry-run", false, "dry run (just mount + load the kernel, don't kexec)")
)

func main() {
	flag.Parse()
	if *diskDev == "" && *cfg == "" && *cdrom == "" {
		log.Printf("Either --config, --device, or --cdrom must be specified")
		flag.PrintDefaults()
		os.Exit(1)
	}

	if len(*diskDev) > 0 {
		imgs, err := esxi.LoadDisk(*diskDev)
		if err != nil {
			log.Fatalf("Failed to load ESXi configuration: %v", err)
		}

		loaded := false
		for _, img := range imgs {
			if len(*appendCmdline) > 0 {
				img.Cmdline = img.Cmdline + " " + strings.Join(*appendCmdline, " ")
			}
			if err := img.Load(false); err != nil {
				log.Printf("Failed to load ESXi image (%v) into memory: %v", img, err)
			} else {
				log.Printf("Loaded image: %v", img)
				// We loaded one, that's it.
				loaded = true
				break
			}
		}
		if !loaded {
			log.Fatalf("Failed to load all ESXi images found.")
		}
	} else {
		var err error
		var img *boot.MultibootImage
		if len(*cfg) > 0 {
			img, err = esxi.LoadConfig(*cfg)
		} else if len(*cdrom) > 0 {
			img, err = esxi.LoadCDROM(*cdrom)
		}
		if err != nil {
			log.Fatalf("Failed to load ESXi configuration: %v", err)
		}
		if len(*appendCmdline) > 0 {
			img.Cmdline = img.Cmdline + " " + strings.Join(*appendCmdline, " ")
		}
		if err := img.Load(false); err != nil {
			log.Fatalf("Failed to load ESXi image (%v) into memory: %v", img, err)
		}
		log.Printf("Loaded image: %v", img)
	}

	if *dryRun {
		log.Printf("Dry run: not booting kernel.")
		os.Exit(0)
	}
	if err := boot.Execute(); err != nil {
		log.Fatalf("Failed to boot image: %v", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"os"
	"strings"

	"github.com/135yshr/scratchgo"
)

func main() {
	conn, err := scratchgo.NewDefaultConnect()
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	defer conn.Close()

	go func() {
		for {
			msg, err := conn.Recv()
			if err != nil {
				fmt.Println(err)
			}
			fmt.Println(*msg)
		}
	}()

	for {
		fmt.Printf("> ")
		reader := bufio.NewReader(os.Stdin)
		line, err := reader.ReadString('\n')
		exitOnErr("read buffer", err)

		line = strings.Replace(line, "\n", "", -1)
		buff := strings.Split(line, " ")

		switch strings.ToLower(buff[0]) {
		case "send":
			err = conn.SensorUpdate(buff[1], buff[2:])
			exitOnErr("update sensor", err)
		case "broadcast":
			err = conn.BroadCast(buff[1:])
			exitOnErr("broadcast message", err)
		case "exit", "quit":
			os.Exit(0)
		default:
			fmt.Println("unknown command.\n  send or broadcast")
		}
	}
	os.Exit(0)
}

func exitOnErr(msg string, err error) {
	if err != nil {
		fmt.Println(msg, err)
		os.Exit(1)
	}
}
<commit_msg>mod SensorUpdate parameter & BroadCast parameter<commit_after>package main

import (
	"bufio"
	"fmt"
	"os"
	"strings"

	"github.com/135yshr/scratchgo"
)

func main() {
	conn, err := scratchgo.NewDefaultConnect()
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	defer conn.Close()

	go func() {
		for {
			msg, err := conn.Recv()
			if err != nil {
				fmt.Println(err)
			}
			fmt.Println(*msg)
		}
	}()

	for {
		fmt.Printf("> ")
		reader := bufio.NewReader(os.Stdin)
		line, err := reader.ReadString('\n')
		exitOnErr("read buffer", err)

		line = strings.Replace(line, "\n", "", -1)
		buff := strings.Split(line, " ")

		switch strings.ToLower(buff[0]) {
		case "send":
			err = conn.SensorUpdate(map[string]interface{}{buff[1]: buff[2]})
			exitOnErr("update sensor", err)
		case "broadcast":
			err = conn.BroadCast(buff[1])
			exitOnErr("broadcast message", err)
		case "exit", "quit":
			os.Exit(0)
		default:
			fmt.Println("unknown command.\n  send or broadcast")
		}
	}
	os.Exit(0)
}

func exitOnErr(msg string, err error) {
	if err != nil {
		fmt.Println(msg, err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 The Serulian Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package builder

import (
	"io/ioutil"
	"path"
	"strings"
	"testing"

	"github.com/serulian/compiler/bundle"
	"github.com/serulian/compiler/compilercommon"
	"github.com/serulian/compiler/graphs/scopegraph"
	"github.com/serulian/compiler/graphs/typegraph"
	"github.com/serulian/compiler/integration"
	"github.com/serulian/compiler/packageloader"

	"github.com/stretchr/testify/assert"
)

const TESTLIB_PATH = "../testlib"

type testLangIntegration struct {
	sourceHandler testSourceHandler
}

func (t testLangIntegration) SourceHandler() packageloader.SourceHandler {
	return t.sourceHandler
}

func (t testLangIntegration) TypeConstructor() typegraph.TypeGraphConstructor {
	return nil
}

func (t testLangIntegration) PathHandler() integration.PathHandler {
	return nil
}

func (t testLangIntegration) PopulateFilesToBundle(bundler bundle.Bundler) {
	for filename, contents := range t.sourceHandler.parser.files {
		bundler.AddFile(bundle.FileFromString(path.Base(filename), bundle.Resource, contents))
	}
}

type testSourceHandler struct {
	parser testParser
}

func (t testSourceHandler) Kind() string {
	return "test"
}

func (t testSourceHandler) PackageFileExtension() string {
	return ".test"
}

func (t testSourceHandler) NewParser() packageloader.SourceHandlerParser {
	return t.parser
}

type testParser struct {
	files map[string]string
}

func (t testParser) Parse(source compilercommon.InputSource, input string, importHandler packageloader.ImportHandler) {
	t.files[string(source)] = input
}

func (t testParser) Apply(packageMap packageloader.LoadedPackageMap, sourceTracker packageloader.SourceTracker) {
}

func (t testParser) Verify(errorReporter packageloader.ErrorReporter, warningReporter packageloader.WarningReporter) {
}

func TestBundling(t *testing.T) {
	entrypointFile := "tests/simple.seru"
	tli := testLangIntegration{
		sourceHandler: testSourceHandler{
			parser: testParser{
				files: map[string]string{},
			},
		},
	}

	result, _ := scopegraph.ParseAndBuildScopeGraphWithConfig(scopegraph.Config{
		Entrypoint:                packageloader.Entrypoint(entrypointFile),
		VCSDevelopmentDirectories: []string{},
		Libraries:                 []packageloader.Library{packageloader.Library{TESTLIB_PATH, false, "", "testcore"}},
		Target:                    scopegraph.Compilation,
		PathLoader:                packageloader.LocalFilePathLoader{},
		LanguageIntegrations:      []integration.LanguageIntegration{tli},
	})

	if !assert.True(t, result.Status, "Expected no failure") {
		return
	}

	sourceAndBundle := GenerateSourceAndBundle(result)
	assert.True(t, len(sourceAndBundle.Source()) > 0)
	assert.NotNil(t, sourceAndBundle.SourceMap())

	bundledFiles := sourceAndBundle.BundledFiles()
	bundledWithSource := sourceAndBundle.BundleWithSource("simple.js", "")

	if !assert.Equal(t, len(bundledWithSource.Files()), len(bundledFiles.Files())+2) {
		return
	}

	// Ensure that the test file was added.
	assert.Equal(t, 1, len(sourceAndBundle.BundledFiles().Files()))

	// Make sure the source file and source map are present, properly named, and that the source is properly annotated.
	_, sourcemapExists := bundledWithSource.LookupFile("simple.js.map")
	assert.True(t, sourcemapExists)

	sourceFile, sourceExists := bundledWithSource.LookupFile("simple.js")
	if !assert.True(t, sourceExists) {
		return
	}

	source, err := ioutil.ReadAll(sourceFile.Reader())
	if !assert.Nil(t, err) {
		return
	}

	assert.True(t, strings.HasSuffix(string(source), "\n//# sourceMappingURL=simple.js.map"))

	_, someFileExists := bundledWithSource.LookupFile("somefile.test")
	assert.True(t, someFileExists)
}
<commit_msg>Output errors if builder bundle test fails<commit_after>// Copyright 2018 The Serulian Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package builder

import (
	"io/ioutil"
	"path"
	"strings"
	"testing"

	"github.com/serulian/compiler/bundle"
	"github.com/serulian/compiler/compilercommon"
	"github.com/serulian/compiler/graphs/scopegraph"
	"github.com/serulian/compiler/graphs/typegraph"
	"github.com/serulian/compiler/integration"
	"github.com/serulian/compiler/packageloader"

	"github.com/stretchr/testify/assert"
)

const TESTLIB_PATH = "../testlib"

type testLangIntegration struct {
	sourceHandler testSourceHandler
}

func (t testLangIntegration) SourceHandler() packageloader.SourceHandler {
	return t.sourceHandler
}

func (t testLangIntegration) TypeConstructor() typegraph.TypeGraphConstructor {
	return nil
}

func (t testLangIntegration) PathHandler() integration.PathHandler {
	return nil
}

func (t testLangIntegration) PopulateFilesToBundle(bundler bundle.Bundler) {
	for filename, contents := range t.sourceHandler.parser.files {
		bundler.AddFile(bundle.FileFromString(path.Base(filename), bundle.Resource, contents))
	}
}

type testSourceHandler struct {
	parser testParser
}

func (t testSourceHandler) Kind() string {
	return "test"
}

func (t testSourceHandler) PackageFileExtension() string {
	return ".test"
}

func (t testSourceHandler) NewParser() packageloader.SourceHandlerParser {
	return t.parser
}

type testParser struct {
	files map[string]string
}

func (t testParser) Parse(source compilercommon.InputSource, input string, importHandler packageloader.ImportHandler) {
	t.files[string(source)] = input
}

func (t testParser) Apply(packageMap packageloader.LoadedPackageMap, sourceTracker packageloader.SourceTracker) {
}

func (t testParser) Verify(errorReporter packageloader.ErrorReporter, warningReporter packageloader.WarningReporter) {
}

func TestBundling(t *testing.T) {
	entrypointFile := "tests/simple.seru"
	tli := testLangIntegration{
		sourceHandler: testSourceHandler{
			parser: testParser{
				files: map[string]string{},
			},
		},
	}

	result, _ := scopegraph.ParseAndBuildScopeGraphWithConfig(scopegraph.Config{
		Entrypoint:                packageloader.Entrypoint(entrypointFile),
		VCSDevelopmentDirectories: []string{},
		Libraries:                 []packageloader.Library{packageloader.Library{TESTLIB_PATH, false, "", "testcore"}},
		Target:                    scopegraph.Compilation,
		PathLoader:                packageloader.LocalFilePathLoader{},
		LanguageIntegrations:      []integration.LanguageIntegration{tli},
	})

	if !assert.True(t, result.Status, "Expected no failure. Got: %v", result.Errors) {
		return
	}

	sourceAndBundle := GenerateSourceAndBundle(result)
	assert.True(t, len(sourceAndBundle.Source()) > 0)
	assert.NotNil(t, sourceAndBundle.SourceMap())

	bundledFiles := sourceAndBundle.BundledFiles()
	bundledWithSource := sourceAndBundle.BundleWithSource("simple.js", "")

	if !assert.Equal(t, len(bundledWithSource.Files()), len(bundledFiles.Files())+2) {
		return
	}

	// Ensure that the test file was added.
	assert.Equal(t, 1, len(sourceAndBundle.BundledFiles().Files()))

	// Make sure the source file and source map are present, properly named, and that the source is properly annotated.
	_, sourcemapExists := bundledWithSource.LookupFile("simple.js.map")
	assert.True(t, sourcemapExists)

	sourceFile, sourceExists := bundledWithSource.LookupFile("simple.js")
	if !assert.True(t, sourceExists) {
		return
	}

	source, err := ioutil.ReadAll(sourceFile.Reader())
	if !assert.Nil(t, err) {
		return
	}

	assert.True(t, strings.HasSuffix(string(source), "\n//# sourceMappingURL=simple.js.map"))

	_, someFileExists := bundledWithSource.LookupFile("somefile.test")
	assert.True(t, someFileExists)
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pod

import (
	"encoding/json"
	"log"

	"github.com/kubernetes/dashboard/src/app/backend/client"
	"github.com/kubernetes/dashboard/src/app/backend/resource/common"
	daemonsetlist "github.com/kubernetes/dashboard/src/app/backend/resource/daemonset/list"
	"github.com/kubernetes/dashboard/src/app/backend/resource/dataselect"
	joblist "github.com/kubernetes/dashboard/src/app/backend/resource/job/list"
	"github.com/kubernetes/dashboard/src/app/backend/resource/metric"
	replicasetlist "github.com/kubernetes/dashboard/src/app/backend/resource/replicaset/list"
	replicationcontrollerlist "github.com/kubernetes/dashboard/src/app/backend/resource/replicationcontroller/list"
	statefulsetlist "github.com/kubernetes/dashboard/src/app/backend/resource/statefulset/list"
	metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	k8sClient "k8s.io/client-go/kubernetes"
	api "k8s.io/client-go/pkg/api/v1"
	apps "k8s.io/client-go/pkg/apis/apps/v1beta1"
	batch "k8s.io/client-go/pkg/apis/batch/v1"
	extensions "k8s.io/client-go/pkg/apis/extensions/v1beta1"
)

// PodDetail is a presentation layer view of Kubernetes PodDetail resource.
// This means it is PodDetail plus additional augmented data we can get
// from other sources (like services that target it).
type PodDetail struct {
	ObjectMeta common.ObjectMeta `json:"objectMeta"`
	TypeMeta   common.TypeMeta   `json:"typeMeta"`

	// Status of the Pod. See Kubernetes API for reference.
	PodPhase api.PodPhase `json:"podPhase"`

	// IP address of the Pod.
	PodIP string `json:"podIP"`

	// Name of the Node this Pod runs on.
	NodeName string `json:"nodeName"`

	// Count of containers restarts.
	RestartCount int32 `json:"restartCount"`

	// Reference to the Controller
	Controller Controller `json:"controller"`

	// List of container of this pod.
	Containers []Container `json:"containers"`

	// Metrics collected for this resource
	Metrics []metric.Metric `json:"metrics"`

	// Conditions of this pod.
	Conditions []common.Condition `json:"conditions"`

	// Events is list of events associated with a pod.
	EventList common.EventList `json:"eventList"`
}

// Creator is a view of the creator of a given pod, in List for for ease of use
// in the frontend logic.
//
// Has 'oneof' semantics on the non-Kind fields decided by which Kind is there
// TODO(maciaszczykm): Refactor.
type Controller struct {
	// Kind of the Controller, will also define wich of the other members will be non nil
	Kind string `json:"kind"`

	JobList                   *joblist.JobList                                     `json:"joblist,omitempty"`
	ReplicaSetList            *replicasetlist.ReplicaSetList                       `json:"replicasetlist,omitempty"`
	ReplicationControllerList *replicationcontrollerlist.ReplicationControllerList `json:"replicationcontrollerlist,omitempty"`
	DaemonSetList             *daemonsetlist.DaemonSetList                         `json:"daemonsetlist,omitempty"`
	StatefulSetList           *statefulsetlist.StatefulSetList                     `json:"statefulsetlist,omitempty"`
}

// Container represents a docker/rkt/etc. container that lives in a pod.
type Container struct {
	// Name of the container.
	Name string `json:"name"`

	// Image URI of the container.
	Image string `json:"image"`

	// List of environment variables.
	Env []EnvVar `json:"env"`

	// Commands of the container
	Commands []string `json:"commands"`

	// Command arguments
	Args []string `json:"args"`
}

// EnvVar represents an environment variable of a container.
type EnvVar struct {
	// Name of the variable.
	Name string `json:"name"`

	// Value of the variable. May be empty if value from is defined.
	Value string `json:"value"`

	// Defined for derived variables. If non-null, the value is get from the reference.
	// Note that this is an API struct. This is intentional, as EnvVarSources are plain struct
	// references.
	ValueFrom *api.EnvVarSource `json:"valueFrom"`
}

// GetPodDetail returns the details (PodDetail) of a named Pod from a particular
// namespace.
func GetPodDetail(client k8sClient.Interface, heapsterClient client.HeapsterClient, namespace, name string) (*PodDetail, error) {

	log.Printf("Getting details of %s pod in %s namespace", name, namespace)

	channels := &common.ResourceChannels{
		ConfigMapList: common.GetConfigMapListChannel(client, common.NewSameNamespaceQuery(namespace), 1),
		PodMetrics:    common.GetPodMetricsChannel(heapsterClient, name, namespace),
	}

	pod, err := client.Core().Pods(namespace).Get(name, metaV1.GetOptions{})

	if err != nil {
		return nil, err
	}

	controller := Controller{
		Kind: "unknown",
	}
	creatorAnnotation, found := pod.ObjectMeta.Annotations[api.CreatedByAnnotation]
	if found {
		creatorRef, err := getPodCreator(client, creatorAnnotation, common.NewSameNamespaceQuery(namespace), heapsterClient)
		if err != nil {
			return nil, err
		}
		controller = *creatorRef
	}

	// Download metrics
	_, metricPromises := dataselect.GenericDataSelectWithMetrics(toCells([]api.Pod{*pod}),
		dataselect.StdMetricsDataSelect, dataselect.NoResourceCache, &heapsterClient)
	metrics, _ := metricPromises.GetMetrics()

	if err = <-channels.ConfigMapList.Error; err != nil {
		return nil, err
	}
	configMapList := <-channels.ConfigMapList.List

	eventList, err := GetEventsForPod(client, dataselect.DefaultDataSelect, pod.Namespace, pod.Name)
	if err != nil {
		return nil, err
	}

	podDetail := toPodDetail(pod, metrics, configMapList, controller, eventList)
	return &podDetail, nil
}

func getPodCreator(client k8sClient.Interface, creatorAnnotation string, nsQuery *common.NamespaceQuery, heapsterClient client.HeapsterClient) (*Controller, error) {
	var serializedReference api.SerializedReference
	err := json.Unmarshal([]byte(creatorAnnotation), &serializedReference)
	if err != nil {
		return nil, err
	}

	channels := &common.ResourceChannels{
		PodList:   common.GetPodListChannel(client, nsQuery, 1),
		EventList: common.GetEventListChannel(client, nsQuery, 1),
	}
	pods := <-channels.PodList.List
	if err := <-channels.PodList.Error; err != nil {
		return nil, err
	}

	events := <-channels.EventList.List
	if err := <-channels.EventList.Error; err != nil {
		return nil, err
	}
	reference := serializedReference.Reference
	return toPodController(client, reference, pods.Items, events.Items, heapsterClient)
}

func toPodController(client k8sClient.Interface, reference api.ObjectReference, pods []api.Pod, events []api.Event, heapsterClient client.HeapsterClient) (*Controller, error) {
	kind := reference.Kind
	switch kind {
	case "Job":
		return toJobPodController(client, reference, pods, events, heapsterClient)
	case "ReplicaSet":
		return toReplicaSetPodController(client, reference, pods, events, heapsterClient)
	case "ReplicationController":
		return toReplicationControllerPodController(client, reference, pods, events, heapsterClient)
	case "DaemonSet":
		return toDaemonSetPodController(client, reference, pods, events, heapsterClient)
	case "StatefulSet":
		return toStatefulSetPodController(client, reference, pods, events, heapsterClient)
	default:
	}
	// Will be moved into the default case once all cases are implemented
	return &Controller{
		Kind: kind,
	}, nil
}

func toJobPodController(client k8sClient.Interface, reference api.ObjectReference, pods []api.Pod, events []api.Event, heapsterClient client.HeapsterClient) (*Controller, error) {
	job, err := client.Batch().Jobs(reference.Namespace).Get(reference.Name, metaV1.GetOptions{})
	if err != nil {
		return nil, err
	}
	jobs := []batch.Job{*job}
	jobList := joblist.CreateJobList(jobs, pods, events, dataselect.StdMetricsDataSelect, &heapsterClient)
	return &Controller{
		Kind:    "Job",
		JobList: jobList,
	}, nil
}

func toReplicaSetPodController(client k8sClient.Interface, reference api.ObjectReference, pods []api.Pod, events []api.Event, heapsterClient client.HeapsterClient) (*Controller, error) {
	rs, err := client.Extensions().ReplicaSets(reference.Namespace).Get(reference.Name, metaV1.GetOptions{})
	if err != nil {
		return nil, err
	}
	replicaSets := []extensions.ReplicaSet{*rs}
	replicaSetList := replicasetlist.CreateReplicaSetList(replicaSets, pods, events, dataselect.StdMetricsDataSelect, &heapsterClient)
	return &Controller{
		Kind:           "ReplicaSet",
		ReplicaSetList: replicaSetList,
	}, nil
}

func toReplicationControllerPodController(client k8sClient.Interface, reference api.ObjectReference, pods []api.Pod, events []api.Event, heapsterClient client.HeapsterClient) (*Controller, error) {
	rc, err := client.Core().ReplicationControllers(reference.Namespace).Get(reference.Name, metaV1.GetOptions{})
	if err != nil {
		return nil, err
	}
	rcs := []api.ReplicationController{*rc}
	replicationControllerList := replicationcontrollerlist.CreateReplicationControllerList(rcs, dataselect.StdMetricsDataSelect, pods, events, &heapsterClient)
	return &Controller{
		Kind: "ReplicationController",
		ReplicationControllerList: replicationControllerList,
	}, nil
}

func toDaemonSetPodController(client k8sClient.Interface, reference api.ObjectReference, pods []api.Pod, events []api.Event, heapsterClient client.HeapsterClient) (*Controller, error) {
	daemonset, err := client.Extensions().DaemonSets(reference.Namespace).Get(reference.Name, metaV1.GetOptions{})
	if err != nil {
		return nil, err
	}
	daemonsets := []extensions.DaemonSet{*daemonset}

	daemonSetList := daemonsetlist.CreateDaemonSetList(daemonsets, pods, events, dataselect.StdMetricsDataSelect, &heapsterClient)
	return &Controller{
		Kind:          "DaemonSet",
		DaemonSetList: daemonSetList,
	}, nil
}

func toStatefulSetPodController(client k8sClient.Interface, reference api.ObjectReference, pods []api.Pod, events []api.Event, heapsterClient client.HeapsterClient) (*Controller, error) {
	statefulset, err := client.Apps().StatefulSets(reference.Namespace).Get(reference.Name, metaV1.GetOptions{})
	if err != nil {
		return nil, err
	}
	statefulsets := []apps.StatefulSet{*statefulset}

	statefulSetList := statefulsetlist.CreateStatefulSetList(statefulsets, pods, events, dataselect.StdMetricsDataSelect, &heapsterClient)
	return &Controller{
		Kind:            "StatefulSet",
		StatefulSetList: statefulSetList,
	}, nil
}

func toPodDetail(pod *api.Pod, metrics []metric.Metric, configMaps *api.ConfigMapList, controller Controller, eventList *common.EventList) PodDetail {

	containers := make([]Container, 0)
	for _, container := range pod.Spec.Containers {
		vars := make([]EnvVar, 0)
		for _, envVar := range container.Env {
			variable := EnvVar{
				Name:      envVar.Name,
				Value:     envVar.Value,
				ValueFrom: envVar.ValueFrom,
			}
			if variable.ValueFrom != nil {
				variable.Value = evalValueFrom(variable.ValueFrom, configMaps)
			}
			vars = append(vars, variable)
		}
		containers = append(containers, Container{
			Name:     container.Name,
			Image:    container.Image,
			Env:      vars,
			Commands: container.Command,
			Args:     container.Args,
		})
	}
	podDetail := PodDetail{
		ObjectMeta:   common.NewObjectMeta(pod.ObjectMeta),
		TypeMeta:     common.NewTypeMeta(common.ResourceKindPod),
		PodPhase:     pod.Status.Phase,
		PodIP:        pod.Status.PodIP,
		RestartCount: getRestartCount(*pod),
		NodeName:     pod.Spec.NodeName,
		Controller:   controller,
		Containers:   containers,
		Metrics:      metrics,
		Conditions:   getPodConditions(*pod),
		EventList:    *eventList,
	}

	return podDetail
}

func evalValueFrom(src *api.EnvVarSource, configMaps *api.ConfigMapList) string {
	if src.ConfigMapKeyRef != nil {
		name := src.ConfigMapKeyRef.LocalObjectReference.Name

		for _, configMap := range configMaps.Items {
			if configMap.ObjectMeta.Name == name {
				return configMap.Data[src.ConfigMapKeyRef.Key]
			}
		}
	}
	return ""
}
<commit_msg>Fix orphaned pods view bug (#1700)<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pod

import (
	"encoding/json"
	"log"

	"github.com/kubernetes/dashboard/src/app/backend/client"
	"github.com/kubernetes/dashboard/src/app/backend/resource/common"
	daemonsetlist "github.com/kubernetes/dashboard/src/app/backend/resource/daemonset/list"
	"github.com/kubernetes/dashboard/src/app/backend/resource/dataselect"
	joblist "github.com/kubernetes/dashboard/src/app/backend/resource/job/list"
	"github.com/kubernetes/dashboard/src/app/backend/resource/metric"
	replicasetlist "github.com/kubernetes/dashboard/src/app/backend/resource/replicaset/list"
	replicationcontrollerlist "github.com/kubernetes/dashboard/src/app/backend/resource/replicationcontroller/list"
	statefulsetlist "github.com/kubernetes/dashboard/src/app/backend/resource/statefulset/list"
	"k8s.io/apimachinery/pkg/api/errors"
	metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	k8sClient "k8s.io/client-go/kubernetes"
	api "k8s.io/client-go/pkg/api/v1"
	apps "k8s.io/client-go/pkg/apis/apps/v1beta1"
	batch "k8s.io/client-go/pkg/apis/batch/v1"
	extensions "k8s.io/client-go/pkg/apis/extensions/v1beta1"
)

// PodDetail is a presentation layer view of Kubernetes PodDetail resource.
// This means it is PodDetail plus additional augmented data we can get
// from other sources (like services that target it).
type PodDetail struct {
	ObjectMeta common.ObjectMeta `json:"objectMeta"`
	TypeMeta   common.TypeMeta   `json:"typeMeta"`

	// Status of the Pod. See Kubernetes API for reference.
	PodPhase api.PodPhase `json:"podPhase"`

	// IP address of the Pod.
	PodIP string `json:"podIP"`

	// Name of the Node this Pod runs on.
	NodeName string `json:"nodeName"`

	// Count of containers restarts.
	RestartCount int32 `json:"restartCount"`

	// Reference to the Controller
	Controller Controller `json:"controller"`

	// List of container of this pod.
	Containers []Container `json:"containers"`

	// Metrics collected for this resource
	Metrics []metric.Metric `json:"metrics"`

	// Conditions of this pod.
	Conditions []common.Condition `json:"conditions"`

	// Events is list of events associated with a pod.
	EventList common.EventList `json:"eventList"`
}

// Creator is a view of the creator of a given pod, in List for for ease of use
// in the frontend logic.
//
// Has 'oneof' semantics on the non-Kind fields decided by which Kind is there
// TODO(maciaszczykm): Refactor.
type Controller struct {
	// Kind of the Controller, will also define wich of the other members will be non nil
	Kind string `json:"kind"`

	JobList                   *joblist.JobList                                     `json:"joblist,omitempty"`
	ReplicaSetList            *replicasetlist.ReplicaSetList                       `json:"replicasetlist,omitempty"`
	ReplicationControllerList *replicationcontrollerlist.ReplicationControllerList `json:"replicationcontrollerlist,omitempty"`
	DaemonSetList             *daemonsetlist.DaemonSetList                         `json:"daemonsetlist,omitempty"`
	StatefulSetList           *statefulsetlist.StatefulSetList                     `json:"statefulsetlist,omitempty"`
}

// Container represents a docker/rkt/etc. container that lives in a pod.
type Container struct {
	// Name of the container.
	Name string `json:"name"`

	// Image URI of the container.
	Image string `json:"image"`

	// List of environment variables.
	Env []EnvVar `json:"env"`

	// Commands of the container
	Commands []string `json:"commands"`

	// Command arguments
	Args []string `json:"args"`
}

// EnvVar represents an environment variable of a container.
type EnvVar struct {
	// Name of the variable.
	Name string `json:"name"`

	// Value of the variable. May be empty if value from is defined.
	Value string `json:"value"`

	// Defined for derived variables. If non-null, the value is get from the reference.
	// Note that this is an API struct. This is intentional, as EnvVarSources are plain struct
	// references.
	ValueFrom *api.EnvVarSource `json:"valueFrom"`
}

// GetPodDetail returns the details (PodDetail) of a named Pod from a particular
// namespace.
func GetPodDetail(client k8sClient.Interface, heapsterClient client.HeapsterClient, namespace, name string) (*PodDetail, error) {

	log.Printf("Getting details of %s pod in %s namespace", name, namespace)

	channels := &common.ResourceChannels{
		ConfigMapList: common.GetConfigMapListChannel(client, common.NewSameNamespaceQuery(namespace), 1),
		PodMetrics:    common.GetPodMetricsChannel(heapsterClient, name, namespace),
	}

	pod, err := client.Core().Pods(namespace).Get(name, metaV1.GetOptions{})

	if err != nil {
		return nil, err
	}

	controller := Controller{
		Kind: "unknown",
	}
	creatorAnnotation, found := pod.ObjectMeta.Annotations[api.CreatedByAnnotation]
	if found {
		creatorRef, err := getPodCreator(client, creatorAnnotation, common.NewSameNamespaceQuery(namespace), heapsterClient)
		if err != nil {
			return nil, err
		}
		controller = *creatorRef
	}

	// Download metrics
	_, metricPromises := dataselect.GenericDataSelectWithMetrics(toCells([]api.Pod{*pod}),
		dataselect.StdMetricsDataSelect, dataselect.NoResourceCache, &heapsterClient)
	metrics, _ := metricPromises.GetMetrics()

	if err = <-channels.ConfigMapList.Error; err != nil {
		return nil, err
	}
	configMapList := <-channels.ConfigMapList.List

	eventList, err := GetEventsForPod(client, dataselect.DefaultDataSelect, pod.Namespace, pod.Name)
	if err != nil {
		return nil, err
	}

	podDetail := toPodDetail(pod, metrics, configMapList, controller, eventList)
	return &podDetail, nil
}

func getPodCreator(client k8sClient.Interface, creatorAnnotation string, nsQuery *common.NamespaceQuery, heapsterClient client.HeapsterClient) (*Controller, error) {
	var serializedReference api.SerializedReference
	err := json.Unmarshal([]byte(creatorAnnotation), &serializedReference)
	if err != nil {
		return nil, err
	}

	channels := &common.ResourceChannels{
		PodList:   common.GetPodListChannel(client, nsQuery, 1),
		EventList: common.GetEventListChannel(client, nsQuery, 1),
	}
	pods := <-channels.PodList.List
	if err := <-channels.PodList.Error; err != nil {
		return nil, err
	}

	events := <-channels.EventList.List
	if err := <-channels.EventList.Error; err != nil {
		return nil, err
	}
	reference := serializedReference.Reference

	controller, err := toPodController(client, reference, pods.Items, events.Items, heapsterClient)
	if err != nil && isNotFoundError(err) {
		return &Controller{}, nil
	}

	return controller, err
}

// isNotFoundError returns true when the given error is 404-NotFound error.
func isNotFoundError(err error) bool {
	statusErr, ok := err.(*errors.StatusError)
	if !ok {
		return false
	}
	return statusErr.ErrStatus.Code == 404
}

func toPodController(client k8sClient.Interface, reference api.ObjectReference, pods []api.Pod, events []api.Event, heapsterClient client.HeapsterClient) (*Controller, error) {
	kind := reference.Kind
	switch kind {
	case "Job":
		return toJobPodController(client, reference, pods, events, heapsterClient)
	case "ReplicaSet":
		return toReplicaSetPodController(client, reference, pods, events, heapsterClient)
	case "ReplicationController":
		return toReplicationControllerPodController(client, reference, pods, events, heapsterClient)
	case "DaemonSet":
		return toDaemonSetPodController(client, reference, pods, events, heapsterClient)
	case "StatefulSet":
		return toStatefulSetPodController(client, reference, pods, events, heapsterClient)
	default:
	}
	// Will be moved into the default case once all cases are implemented
	return &Controller{
		Kind: kind,
	}, nil
}

func toJobPodController(client k8sClient.Interface, reference api.ObjectReference, pods []api.Pod, events []api.Event, heapsterClient client.HeapsterClient) (*Controller, error) {
	job, err := client.Batch().Jobs(reference.Namespace).Get(reference.Name, metaV1.GetOptions{})
	if err != nil {
		return nil, err
	}
	jobs := []batch.Job{*job}
	jobList := joblist.CreateJobList(jobs, pods, events, dataselect.StdMetricsDataSelect, &heapsterClient)
	return &Controller{
		Kind:    "Job",
		JobList: jobList,
	}, nil
}

func toReplicaSetPodController(client k8sClient.Interface, reference api.ObjectReference, pods []api.Pod, events []api.Event, heapsterClient client.HeapsterClient) (*Controller, error) {
	rs, err := client.Extensions().ReplicaSets(reference.Namespace).Get(reference.Name, metaV1.GetOptions{})
	if err != nil {
		return nil, err
	}
	replicaSets := []extensions.ReplicaSet{*rs}
	replicaSetList := replicasetlist.CreateReplicaSetList(replicaSets, pods, events, dataselect.StdMetricsDataSelect, &heapsterClient)
	return &Controller{
		Kind:           "ReplicaSet",
		ReplicaSetList: replicaSetList,
	}, nil
}

func toReplicationControllerPodController(client k8sClient.Interface, reference api.ObjectReference, pods []api.Pod, events []api.Event, heapsterClient client.HeapsterClient) (*Controller, error) {
	rc, err := client.Core().ReplicationControllers(reference.Namespace).Get(reference.Name, metaV1.GetOptions{})
	if err != nil {
		return nil, err
	}
	rcs := []api.ReplicationController{*rc}
	replicationControllerList := replicationcontrollerlist.CreateReplicationControllerList(rcs, dataselect.StdMetricsDataSelect, pods, events, &heapsterClient)
	return &Controller{
		Kind: "ReplicationController",
		ReplicationControllerList: replicationControllerList,
	}, nil
}

func toDaemonSetPodController(client k8sClient.Interface, reference api.ObjectReference, pods []api.Pod, events []api.Event, heapsterClient client.HeapsterClient) (*Controller, error) {
	daemonset, err := client.Extensions().DaemonSets(reference.Namespace).Get(reference.Name, metaV1.GetOptions{})
	if err != nil {
		return nil, err
	}
	daemonsets := []extensions.DaemonSet{*daemonset}

	daemonSetList := daemonsetlist.CreateDaemonSetList(daemonsets, pods, events, dataselect.StdMetricsDataSelect, &heapsterClient)
	return &Controller{
		Kind:          "DaemonSet",
		DaemonSetList: daemonSetList,
	}, nil
}

func toStatefulSetPodController(client k8sClient.Interface, reference api.ObjectReference, pods []api.Pod, events []api.Event, heapsterClient client.HeapsterClient) (*Controller, error) {
	statefulset, err := client.Apps().StatefulSets(reference.Namespace).Get(reference.Name, metaV1.GetOptions{})
	if err != nil {
		return nil, err
	}
	statefulsets := []apps.StatefulSet{*statefulset}

	statefulSetList := statefulsetlist.CreateStatefulSetList(statefulsets, pods, events, dataselect.StdMetricsDataSelect, &heapsterClient)
	return &Controller{
		Kind:            "StatefulSet",
		StatefulSetList: statefulSetList,
	}, nil
}

func toPodDetail(pod *api.Pod, metrics []metric.Metric, configMaps *api.ConfigMapList, controller Controller, eventList *common.EventList) PodDetail {

	containers := make([]Container, 0)
	for _, container := range pod.Spec.Containers {
		vars := make([]EnvVar, 0)
		for _, envVar := range container.Env {
			variable := EnvVar{
				Name:      envVar.Name,
				Value:     envVar.Value,
				ValueFrom: envVar.ValueFrom,
			}
			if variable.ValueFrom != nil {
				variable.Value = evalValueFrom(variable.ValueFrom, configMaps)
			}
			vars = append(vars, variable)
		}
		containers = append(containers, Container{
			Name:     container.Name,
			Image:    container.Image,
			Env:      vars,
			Commands: container.Command,
			Args:     container.Args,
		})
	}
	podDetail := PodDetail{
		ObjectMeta:   common.NewObjectMeta(pod.ObjectMeta),
		TypeMeta:     common.NewTypeMeta(common.ResourceKindPod),
		PodPhase:     pod.Status.Phase,
		PodIP:        pod.Status.PodIP,
		RestartCount: getRestartCount(*pod),
		NodeName:     pod.Spec.NodeName,
		Controller:   controller,
		Containers:   containers,
		Metrics:      metrics,
		Conditions:   getPodConditions(*pod),
		EventList:    *eventList,
	}

	return podDetail
}

func evalValueFrom(src *api.EnvVarSource, configMaps *api.ConfigMapList) string {
	if src.ConfigMapKeyRef != nil {
		name := src.ConfigMapKeyRef.LocalObjectReference.Name

		for _, configMap := range configMaps.Items {
			if configMap.ObjectMeta.Name == name {
				return configMap.Data[src.ConfigMapKeyRef.Key]
			}
		}
	}
	return ""
}
<|endoftext|>


<commit_before>package sched

import (
	"bytes"
	"fmt"
	"log"
	"math"
	"time"

	"github.com/StackExchange/bosun/_third_party/github.com/StackExchange/scollector/collect"
	"github.com/StackExchange/bosun/_third_party/github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/bosun/conf"
	"github.com/StackExchange/bosun/expr"
)

func (s *Schedule) Status(ak expr.AlertKey) *State {
	s.Lock()
	state := s.status[ak]
	if state == nil {
		g := ak.Group()
		state = &State{
			Alert: ak.Name(),
			Tags:  g.Tags(),
			Group: g,
		}
		s.status[ak] = state
	}
	state.Touch()
	s.Unlock()
	return state
}

type RunHistory map[expr.AlertKey]*Event

// Check evaluates all critical and warning alert rules.
func (s *Schedule) Check() {
	r := make(RunHistory)
	s.CheckStart = time.Now().UTC()
	s.cache = opentsdb.NewCache(s.Conf.TsdbHost, s.Conf.ResponseLimit)
	for _, a := range s.Conf.Alerts {
		s.CheckAlert(r, a)
	}
	s.RunHistory(r)
}

// RunHistory processes an event history and triggers notifications if needed.
func (s *Schedule) RunHistory(r RunHistory) {
	checkNotify := false
	silenced := s.Silenced()
	s.Lock()
	defer s.Unlock()
	for ak, event := range r {
		state := s.status[ak]
		last := state.Append(event)
		a := s.Conf.Alerts[ak.Name()]
		if event.Status > StNormal {
			var subject = new(bytes.Buffer)
			if event.Status != StUnknown {
				if err := s.ExecuteSubject(subject, a, state); err != nil {
					log.Println(err)
				}
			}
			state.Subject = subject.String()
			state.Open = true
		}
		// On state increase, clear old notifications and notify current.
		// On state decrease, and if the old alert was already acknowledged, notify current.
		// If the old alert was not acknowledged, do nothing.
		// Do nothing if state did not change.
		notify := func(notifications map[string]*conf.Notification) {
			for _, n := range notifications {
				s.Notify(state, n)
				checkNotify = true
			}
		}
		notifyCurrent := func() {
			state.NeedAck = true
			if _, present := silenced[ak]; present {
				return
			}
			switch event.Status {
			case StCritical, StUnknown:
				notify(a.CritNotification)
			case StWarning:
				notify(a.WarnNotification)
			}
		}
		clearOld := func() {
			state.NeedAck = false
			delete(s.Notifications, ak)
		}
		if event.Status > last {
			clearOld()
			notifyCurrent()
		} else if event.Status < last {
			if _, hasOld := s.Notifications[ak]; hasOld {
				notifyCurrent()
			}
		}
	}
	if checkNotify {
		s.nc <- true
	}
	s.Save()
}

// CheckUnknown checks for unknown alerts.
func (s *Schedule) CheckUnknown() {
	for _ = range time.Tick(s.Conf.CheckFrequency / 4) {
		log.Println("checkUnknown")
		r := make(RunHistory)
		s.Lock()
		for ak, st := range s.status {
			if st.Forgotten {
				continue
			}
			t := s.Conf.Alerts[ak.Name()].Unknown
			if t == 0 {
				t = s.Conf.CheckFrequency * 2
			}
			if t == 0 {
				continue
			}
			if time.Since(st.Touched) < t {
				continue
			}
			r[ak] = &Event{Status: StUnknown}
		}
		s.Unlock()
		s.RunHistory(r)
	}
}

func (s *Schedule) CheckAlert(r RunHistory, a *conf.Alert) {
	log.Printf("checking alert %v", a.Name)
	start := time.Now()
	var warns expr.AlertKeys
	crits, err := s.CheckExpr(r, a, a.Crit, StCritical, nil)
	if err == nil {
		warns, _ = s.CheckExpr(r, a, a.Warn, StWarning, crits)
	}
	collect.Put("check.duration", opentsdb.TagSet{"name": a.Name}, time.Since(start).Seconds())
	log.Printf("done checking alert %v (%s): %v crits, %v warns", a.Name, time.Since(start), len(crits), len(warns))
}

func (s *Schedule) CheckExpr(rh RunHistory, a *conf.Alert, e *expr.Expr, checkStatus Status, ignore expr.AlertKeys) (alerts expr.AlertKeys, err error) {
	if e == nil {
		return
	}
	defer func() {
		if err == nil {
			return
		}
		collect.Add("check.errs", opentsdb.TagSet{"metric": a.Name}, 1)
		log.Println(err)
	}()
	results, _, err := e.Execute(s.cache, nil, s.CheckStart, 0, a.UnjoinedOK, s.Search, s.Conf.GetLookups())
	if err != nil {
		ak := expr.NewAlertKey(a.Name, nil)
		state := s.Status(ak)
		state.Result = &Result{
			Result: &expr.Result{
				Computations: []expr.Computation{
					{
						Text:  e.String(),
						Value: err.Error(),
					},
				},
			},
		}
		rh[ak] = &Event{
			Status: StError,
		}
		return
	}
Loop:
	for _, r := range results.Results {
		if s.Conf.Squelched(a, r.Group) {
			continue
		}
		ak := expr.NewAlertKey(a.Name, r.Group)
		for _, v := range ignore {
			if ak == v {
				continue Loop
			}
		}
		state := s.Status(ak)
		status := checkStatus
		var n float64
		switch v := r.Value.(type) {
		case expr.Number:
			n = float64(v)
		case expr.Scalar:
			n = float64(v)
		default:
			err = fmt.Errorf("expected number or scalar")
			return
		}
		event := rh[ak]
		if event == nil {
			event = new(Event)
			rh[ak] = event
		}
		result := Result{
			Result: r,
			Expr:   e.String(),
		}
		switch checkStatus {
		case StWarning:
			event.Warn = &result
		case StCritical:
			event.Crit = &result
		}
		if math.IsNaN(n) {
			status = StError
		} else if n == 0 {
			status = StNormal
		}
		if status != StNormal {
			alerts = append(alerts, ak)
		}
		if status > rh[ak].Status {
			event.Status = status
			state.Result = &result
		}
	}
	return
}
<commit_msg>Don't attempt to notify if the chan hasn't been initialized<commit_after>package sched

import (
	"bytes"
	"fmt"
	"log"
	"math"
	"time"

	"github.com/StackExchange/bosun/_third_party/github.com/StackExchange/scollector/collect"
	"github.com/StackExchange/bosun/_third_party/github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/bosun/conf"
	"github.com/StackExchange/bosun/expr"
)

func (s *Schedule) Status(ak expr.AlertKey) *State {
	s.Lock()
	state := s.status[ak]
	if state == nil {
		g := ak.Group()
		state = &State{
			Alert: ak.Name(),
			Tags:  g.Tags(),
			Group: g,
		}
		s.status[ak] = state
	}
	state.Touch()
	s.Unlock()
	return state
}

type RunHistory map[expr.AlertKey]*Event

// Check evaluates all critical and warning alert rules.
func (s *Schedule) Check() {
	r := make(RunHistory)
	s.CheckStart = time.Now().UTC()
	s.cache = opentsdb.NewCache(s.Conf.TsdbHost, s.Conf.ResponseLimit)
	for _, a := range s.Conf.Alerts {
		s.CheckAlert(r, a)
	}
	s.RunHistory(r)
}

// RunHistory processes an event history and triggers notifications if needed.
func (s *Schedule) RunHistory(r RunHistory) {
	checkNotify := false
	silenced := s.Silenced()
	s.Lock()
	defer s.Unlock()
	for ak, event := range r {
		state := s.status[ak]
		last := state.Append(event)
		a := s.Conf.Alerts[ak.Name()]
		if event.Status > StNormal {
			var subject = new(bytes.Buffer)
			if event.Status != StUnknown {
				if err := s.ExecuteSubject(subject, a, state); err != nil {
					log.Println(err)
				}
			}
			state.Subject = subject.String()
			state.Open = true
		}
		// On state increase, clear old notifications and notify current.
		// On state decrease, and if the old alert was already acknowledged, notify current.
		// If the old alert was not acknowledged, do nothing.
		// Do nothing if state did not change.
		notify := func(notifications map[string]*conf.Notification) {
			for _, n := range notifications {
				s.Notify(state, n)
				checkNotify = true
			}
		}
		notifyCurrent := func() {
			state.NeedAck = true
			if _, present := silenced[ak]; present {
				return
			}
			switch event.Status {
			case StCritical, StUnknown:
				notify(a.CritNotification)
			case StWarning:
				notify(a.WarnNotification)
			}
		}
		clearOld := func() {
			state.NeedAck = false
			delete(s.Notifications, ak)
		}
		if event.Status > last {
			clearOld()
			notifyCurrent()
		} else if event.Status < last {
			if _, hasOld := s.Notifications[ak]; hasOld {
				notifyCurrent()
			}
		}
	}
	if checkNotify && s.nc != nil {
		s.nc <- true
	}
	s.Save()
}

// CheckUnknown checks for unknown alerts.
func (s *Schedule) CheckUnknown() {
	for _ = range time.Tick(s.Conf.CheckFrequency / 4) {
		log.Println("checkUnknown")
		r := make(RunHistory)
		s.Lock()
		for ak, st := range s.status {
			if st.Forgotten {
				continue
			}
			t := s.Conf.Alerts[ak.Name()].Unknown
			if t == 0 {
				t = s.Conf.CheckFrequency * 2
			}
			if t == 0 {
				continue
			}
			if time.Since(st.Touched) < t {
				continue
			}
			r[ak] = &Event{Status: StUnknown}
		}
		s.Unlock()
		s.RunHistory(r)
	}
}

func (s *Schedule) CheckAlert(r RunHistory, a *conf.Alert) {
	log.Printf("checking alert %v", a.Name)
	start := time.Now()
	var warns expr.AlertKeys
	crits, err := s.CheckExpr(r, a, a.Crit, StCritical, nil)
	if err == nil {
		warns, _ = s.CheckExpr(r, a, a.Warn, StWarning, crits)
	}
	collect.Put("check.duration", opentsdb.TagSet{"name": a.Name}, time.Since(start).Seconds())
	log.Printf("done checking alert %v (%s): %v crits, %v warns", a.Name, time.Since(start), len(crits), len(warns))
}

func (s *Schedule) CheckExpr(rh RunHistory, a *conf.Alert, e *expr.Expr, checkStatus Status, ignore expr.AlertKeys) (alerts expr.AlertKeys, err error) {
	if e == nil {
		return
	}
	defer func() {
		if err == nil {
			return
		}
		collect.Add("check.errs", opentsdb.TagSet{"metric": a.Name}, 1)
		log.Println(err)
	}()
	results, _, err := e.Execute(s.cache, nil, s.CheckStart, 0, a.UnjoinedOK, s.Search, s.Conf.GetLookups())
	if err != nil {
		ak := expr.NewAlertKey(a.Name, nil)
		state := s.Status(ak)
		state.Result = &Result{
			Result: &expr.Result{
				Computations: []expr.Computation{
					{
						Text:  e.String(),
						Value: err.Error(),
					},
				},
			},
		}
		rh[ak] = &Event{
			Status: StError,
		}
		return
	}
Loop:
	for _, r := range results.Results {
		if s.Conf.Squelched(a, r.Group) {
			continue
		}
		ak := expr.NewAlertKey(a.Name, r.Group)
		for _, v := range ignore {
			if ak == v {
				continue Loop
			}
		}
		state := s.Status(ak)
		status := checkStatus
		var n float64
		switch v := r.Value.(type) {
		case expr.Number:
			n = float64(v)
		case expr.Scalar:
			n = float64(v)
		default:
			err = fmt.Errorf("expected number or scalar")
			return
		}
		event := rh[ak]
		if event == nil {
			event = new(Event)
			rh[ak] = event
		}
		result := Result{
			Result: r,
			Expr:   e.String(),
		}
		switch checkStatus {
		case StWarning:
			event.Warn = &result
		case StCritical:
			event.Crit = &result
		}
		if math.IsNaN(n) {
			status = StError
		} else if n == 0 {
			status = StNormal
		}
		if status != StNormal {
			alerts = append(alerts, ak)
		}
		if status > rh[ak].Status {
			event.Status = status
			state.Result = &result
		}
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"autoscale"
	"autoscale/api"
	"autoscale/watcher"
	"net/http"

	log "github.com/Sirupsen/logrus"
	"github.com/kelseyhightower/envconfig"
)

// Specification describes our expected environment.
type Specification struct {
	DBUser      string `envconfig:"db_user"`
	DBPassword  string `envconfig:"db_password"`
	DBAddr      string `envconfig:"db_addr"`
	DBName      string `envconfig:"db_name"`
	HTTPAddr    string `envconfig:"http_addr" default:"localhost:8888"`
	AccessToken string `envconfig:"access_token"`
}

func main() {
	var s Specification
	err := envconfig.Process("autoscale", &s)
	if err != nil {
		log.WithError(err).Fatal("unable to read environment")
	}

	db, err := autoscale.NewDB(s.DBUser, s.DBPassword, s.DBAddr, s.DBName)
	if err != nil {
		log.WithError(err).Fatal("unable to create database connection")
	}

	repo, err := autoscale.NewRepository(db)
	if err != nil {
		log.WithError(err).Fatal("unable to setup data repository")
	}

	watcher := watcher.New(s.AccessToken, repo)
	go func() {
		watcher.Watch()
	}()

	a := api.New(repo)
	http.Handle("/", a.Mux)

	log.WithFields(log.Fields{
		"http-addr": s.HTTPAddr,
	}).Info("created http server")
	log.Fatal(http.ListenAndServe(s.HTTPAddr, nil))
}
<commit_msg>make required arguments required<commit_after>package main

import (
	"autoscale"
	"autoscale/api"
	"autoscale/watcher"
	"net/http"

	log "github.com/Sirupsen/logrus"
	"github.com/kelseyhightower/envconfig"
)

// Specification describes our expected environment.
type Specification struct {
	DBUser      string `envconfig:"db_user" required:"true"`
	DBPassword  string `envconfig:"db_password" required:"true"`
	DBAddr      string `envconfig:"db_addr" required:"true"`
	DBName      string `envconfig:"db_name" required:"true"`
	HTTPAddr    string `envconfig:"http_addr" default:"localhost:8888"`
	AccessToken string `envconfig:"access_token" required:"true"`
}

func main() {
	var s Specification
	err := envconfig.Process("autoscale", &s)
	if err != nil {
		log.WithError(err).Fatal("unable to read environment")
	}

	db, err := autoscale.NewDB(s.DBUser, s.DBPassword, s.DBAddr, s.DBName)
	if err != nil {
		log.WithError(err).Fatal("unable to create database connection")
	}

	repo, err := autoscale.NewRepository(db)
	if err != nil {
		log.WithError(err).Fatal("unable to setup data repository")
	}

	watcher := watcher.New(s.AccessToken, repo)
	go func() {
		watcher.Watch()
	}()

	a := api.New(repo)
	http.Handle("/", a.Mux)

	log.WithFields(log.Fields{
		"http-addr": s.HTTPAddr,
	}).Info("created http server")
	log.Fatal(http.ListenAndServe(s.HTTPAddr, nil))
}
<|endoftext|>