<commit_before>package sched

import (
	"bytes"
	"compress/gzip"
	"crypto/md5"
	"encoding/base64"
	"encoding/gob"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"os"
	"time"

	"bosun.org/_third_party/github.com/boltdb/bolt"
	"bosun.org/cmd/bosun/conf"
	"bosun.org/cmd/bosun/expr"
	"bosun.org/collect"
	"bosun.org/opentsdb"
)

var savePending bool

func (s *Schedule) Save() {
	go func() {
		s.Lock()
		defer s.Unlock()
		if savePending {
			return
		}
		savePending = true
		time.AfterFunc(time.Second*5, s.save)
	}()
}

type counterWriter struct {
	written int
	w       io.Writer
}

func (c *counterWriter) Write(p []byte) (n int, err error) {
	n, err = c.w.Write(p)
	c.written += n
	return n, err
}

const (
	dbBucket           = "bindata"
	dbConfigTextBucket = "configText"
	dbMetric           = "metric"
	dbTagk             = "tagk"
	dbTagv             = "tagv"
	dbMetricTags       = "metrictags"
	dbNotifications    = "notifications"
	dbSilence          = "silence"
	dbStatus           = "status"
	dbMetadata         = "metadata"
	dbIncidents        = "incidents"
)

func (s *Schedule) save() {
	defer func() {
		savePending = false
	}()
	if s.db == nil {
		return
	}
	store := map[string]interface{}{
		dbMetric:        s.Search.Read.Metric,
		dbTagk:          s.Search.Read.Tagk,
		dbTagv:          s.Search.Read.Tagv,
		dbMetricTags:    s.Search.Read.MetricTags,
		dbNotifications: s.Notifications,
		dbSilence:       s.Silence,
		dbStatus:        s.status,
		dbMetadata:      s.Metadata,
		dbIncidents:     s.Incidents,
	}
	tostore := make(map[string][]byte)
	for name, data := range store {
		f := new(bytes.Buffer)
		gz := gzip.NewWriter(f)
		cw := &counterWriter{w: gz}
		enc := gob.NewEncoder(cw)
		if err := enc.Encode(data); err != nil {
			log.Printf("error saving %s: %v", name, err)
			return
		}
		if err := gz.Flush(); err != nil {
			log.Printf("gzip flush error saving %s: %v", name, err)
		}
		if err := gz.Close(); err != nil {
			log.Printf("gzip close error saving %s: %v", name, err)
		}
		tostore[name] = f.Bytes()
		log.Printf("wrote %s: %v", name, conf.ByteSize(cw.written))
		collect.Put("statefile.size", opentsdb.TagSet{"object": name}, cw.written)
	}
	err := s.db.Update(func(tx *bolt.Tx) error {
		b, err := tx.CreateBucketIfNotExists([]byte(dbBucket))
		if err != nil {
			return err
		}
		for name, data := range tostore {
			if err := b.Put([]byte(name), data); err != nil {
				return err
			}
		}
		return nil
	})
	if err != nil {
		log.Printf("save db update error: %v", err)
		return
	}
	fi, err := os.Stat(s.Conf.StateFile)
	if err == nil {
		collect.Put("statefile.size", opentsdb.TagSet{"object": "total"}, fi.Size())
	}
	log.Println("save to db complete")
}

// RestoreState restores notification and alert state from the file on disk.
func (s *Schedule) RestoreState() error {
	log.Println("RestoreState")
	start := time.Now()
	s.Lock()
	defer s.Unlock()
	s.Search.Lock()
	defer s.Search.Unlock()
	s.Notifications = nil
	decode := func(name string, dst interface{}) error {
		var data []byte
		err := s.db.View(func(tx *bolt.Tx) error {
			b := tx.Bucket([]byte(dbBucket))
			if b == nil {
				return fmt.Errorf("unknown bucket: %v", dbBucket)
			}
			data = b.Get([]byte(name))
			return nil
		})
		if err != nil {
			return err
		}
		gr, err := gzip.NewReader(bytes.NewReader(data))
		if err != nil {
			return err
		}
		defer gr.Close()
		return gob.NewDecoder(gr).Decode(dst)
	}
	if err := decode(dbMetric, &s.Search.Metric); err != nil {
		log.Println(dbMetric, err)
	}
	if err := decode(dbTagk, &s.Search.Tagk); err != nil {
		log.Println(dbTagk, err)
	}
	if err := decode(dbTagv, &s.Search.Tagv); err != nil {
		log.Println(dbTagv, err)
	}
	if err := decode(dbMetricTags, &s.Search.MetricTags); err != nil {
		log.Println(dbMetricTags, err)
	}

	notifications := make(map[expr.AlertKey]map[string]time.Time)
	if err := decode(dbNotifications, &notifications); err != nil {
		log.Println(dbNotifications, err)
	}
	if err := decode(dbSilence, &s.Silence); err != nil {
		log.Println(dbSilence, err)
	}
	if err := decode(dbIncidents, &s.Incidents); err != nil {
		log.Println(dbIncidents, err)
	}
	// Calculate next incident id.
	for _, i := range s.Incidents {
		if i.Id > s.maxIncidentId {
			s.maxIncidentId = i.Id
		}
	}
	status := make(States)
	if err := decode(dbStatus, &status); err != nil {
		log.Println(dbStatus, err)
	}
	clear := func(r *Result) {
		if r == nil {
			return
		}
		r.Computations = nil
	}
	for ak, st := range status {
		a, present := s.Conf.Alerts[ak.Name()]
		if !present {
			log.Println("sched: alert no longer present, ignoring:", ak)
			continue
		} else if s.Conf.Squelched(a, st.Group) {
			log.Println("sched: alert now squelched:", ak)
			continue
		} else {
			t := a.Unknown
			if t == 0 {
				t = s.Conf.CheckFrequency
			}
			if t == 0 && st.Last().Status == StUnknown {
				st.Append(&Event{Status: StNormal, IncidentId: st.Last().IncidentId})
			}
		}
		clear(st.Result)
		for _, e := range st.History {
			clear(e.Warn)
			clear(e.Crit)
			clear(e.Error)
		}
		s.status[ak] = st
		if a.Log && st.Open {
			st.Open = false
			log.Printf("sched: alert %s is now log, closing, was %s", ak, st.Status())
		}
		for name, t := range notifications[ak] {
			n, present := s.Conf.Notifications[name]
			if !present {
				log.Println("sched: notification not present during restore:", name)
				continue
			}
			if a.Log {
				log.Println("sched: alert is now log, removing notification:", ak)
				continue
			}
			s.AddNotification(ak, n, t)
		}
	}
	if err := decode(dbMetadata, &s.Metadata); err != nil {
		log.Println(dbMetadata, err)
	}
	//if s.maxIncidentId == 0 {
	s.maxIncidentId = 0 //temporarily start over every time we start up.
	s.Incidents = make(map[uint64]*Incident)
	s.createHistoricIncidents()
	//}
	s.Search.Copy()
	log.Println("RestoreState done in", time.Since(start))
	return nil
}

type storedConfig struct {
	Text     string
	LastUsed time.Time
}

// Saves the provided config text in state file for later access.
// Returns a hash of the file to be used as a retreival key.
func (s *Schedule) SaveTempConfig(text string) (hash string, err error) {
	sig := md5.Sum([]byte(text))
	b64 := base64.StdEncoding.EncodeToString(sig[0:5])
	data := storedConfig{Text: text, LastUsed: time.Now()}
	bindata, err := json.Marshal(data)
	if err != nil {
		return "", err
	}
	err = s.db.Update(func(tx *bolt.Tx) error {
		b, err := tx.CreateBucketIfNotExists([]byte(dbConfigTextBucket))
		if err != nil {
			return err
		}
		return b.Put([]byte(b64), bindata)
	})
	if err != nil {
		return "", err
	}
	return b64, nil
}

// Retreive the specified config text from state file.
func (s *Schedule) LoadTempConfig(hash string) (text string, err error) {
	config := storedConfig{}
	err = s.db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(dbConfigTextBucket))
		data := b.Get([]byte(hash))
		if data == nil || len(data) == 0 {
			return fmt.Errorf("Config text '%s' not found", hash)
		}
		return json.Unmarshal(data, &config)
	})
	if err != nil {
		return "", err
	}
	go s.SaveTempConfig(config.Text) //refresh timestamp.
	return config.Text, nil
}
<commit_msg>Only run historic incidents one time.<commit_after>package sched

import (
	"bytes"
	"compress/gzip"
	"crypto/md5"
	"encoding/base64"
	"encoding/gob"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"os"
	"time"

	"bosun.org/_third_party/github.com/boltdb/bolt"
	"bosun.org/cmd/bosun/conf"
	"bosun.org/cmd/bosun/expr"
	"bosun.org/collect"
	"bosun.org/opentsdb"
)

var savePending bool

func (s *Schedule) Save() {
	go func() {
		s.Lock()
		defer s.Unlock()
		if savePending {
			return
		}
		savePending = true
		time.AfterFunc(time.Second*5, s.save)
	}()
}

type counterWriter struct {
	written int
	w       io.Writer
}

func (c *counterWriter) Write(p []byte) (n int, err error) {
	n, err = c.w.Write(p)
	c.written += n
	return n, err
}

const (
	dbBucket           = "bindata"
	dbConfigTextBucket = "configText"
	dbMetric           = "metric"
	dbTagk             = "tagk"
	dbTagv             = "tagv"
	dbMetricTags       = "metrictags"
	dbNotifications    = "notifications"
	dbSilence          = "silence"
	dbStatus           = "status"
	dbMetadata         = "metadata"
	dbIncidents        = "incidents"
)

func (s *Schedule) save() {
	defer func() {
		savePending = false
	}()
	if s.db == nil {
		return
	}
	store := map[string]interface{}{
		dbMetric:        s.Search.Read.Metric,
		dbTagk:          s.Search.Read.Tagk,
		dbTagv:          s.Search.Read.Tagv,
		dbMetricTags:    s.Search.Read.MetricTags,
		dbNotifications: s.Notifications,
		dbSilence:       s.Silence,
		dbStatus:        s.status,
		dbMetadata:      s.Metadata,
		dbIncidents:     s.Incidents,
	}
	tostore := make(map[string][]byte)
	for name, data := range store {
		f := new(bytes.Buffer)
		gz := gzip.NewWriter(f)
		cw := &counterWriter{w: gz}
		enc := gob.NewEncoder(cw)
		if err := enc.Encode(data); err != nil {
			log.Printf("error saving %s: %v", name, err)
			return
		}
		if err := gz.Flush(); err != nil {
			log.Printf("gzip flush error saving %s: %v", name, err)
		}
		if err := gz.Close(); err != nil {
			log.Printf("gzip close error saving %s: %v", name, err)
		}
		tostore[name] = f.Bytes()
		log.Printf("wrote %s: %v", name, conf.ByteSize(cw.written))
		collect.Put("statefile.size", opentsdb.TagSet{"object": name}, cw.written)
	}
	err := s.db.Update(func(tx *bolt.Tx) error {
		b, err := tx.CreateBucketIfNotExists([]byte(dbBucket))
		if err != nil {
			return err
		}
		for name, data := range tostore {
			if err := b.Put([]byte(name), data); err != nil {
				return err
			}
		}
		return nil
	})
	if err != nil {
		log.Printf("save db update error: %v", err)
		return
	}
	fi, err := os.Stat(s.Conf.StateFile)
	if err == nil {
		collect.Put("statefile.size", opentsdb.TagSet{"object": "total"}, fi.Size())
	}
	log.Println("save to db complete")
}

// RestoreState restores notification and alert state from the file on disk.
func (s *Schedule) RestoreState() error {
	log.Println("RestoreState")
	start := time.Now()
	s.Lock()
	defer s.Unlock()
	s.Search.Lock()
	defer s.Search.Unlock()
	s.Notifications = nil
	decode := func(name string, dst interface{}) error {
		var data []byte
		err := s.db.View(func(tx *bolt.Tx) error {
			b := tx.Bucket([]byte(dbBucket))
			if b == nil {
				return fmt.Errorf("unknown bucket: %v", dbBucket)
			}
			data = b.Get([]byte(name))
			return nil
		})
		if err != nil {
			return err
		}
		gr, err := gzip.NewReader(bytes.NewReader(data))
		if err != nil {
			return err
		}
		defer gr.Close()
		return gob.NewDecoder(gr).Decode(dst)
	}
	if err := decode(dbMetric, &s.Search.Metric); err != nil {
		log.Println(dbMetric, err)
	}
	if err := decode(dbTagk, &s.Search.Tagk); err != nil {
		log.Println(dbTagk, err)
	}
	if err := decode(dbTagv, &s.Search.Tagv); err != nil {
		log.Println(dbTagv, err)
	}
	if err := decode(dbMetricTags, &s.Search.MetricTags); err != nil {
		log.Println(dbMetricTags, err)
	}

	notifications := make(map[expr.AlertKey]map[string]time.Time)
	if err := decode(dbNotifications, &notifications); err != nil {
		log.Println(dbNotifications, err)
	}
	if err := decode(dbSilence, &s.Silence); err != nil {
		log.Println(dbSilence, err)
	}
	if err := decode(dbIncidents, &s.Incidents); err != nil {
		log.Println(dbIncidents, err)
	}
	// Calculate next incident id.
	for _, i := range s.Incidents {
		if i.Id > s.maxIncidentId {
			s.maxIncidentId = i.Id
		}
	}
	status := make(States)
	if err := decode(dbStatus, &status); err != nil {
		log.Println(dbStatus, err)
	}
	clear := func(r *Result) {
		if r == nil {
			return
		}
		r.Computations = nil
	}
	for ak, st := range status {
		a, present := s.Conf.Alerts[ak.Name()]
		if !present {
			log.Println("sched: alert no longer present, ignoring:", ak)
			continue
		} else if s.Conf.Squelched(a, st.Group) {
			log.Println("sched: alert now squelched:", ak)
			continue
		} else {
			t := a.Unknown
			if t == 0 {
				t = s.Conf.CheckFrequency
			}
			if t == 0 && st.Last().Status == StUnknown {
				st.Append(&Event{Status: StNormal, IncidentId: st.Last().IncidentId})
			}
		}
		clear(st.Result)
		for _, e := range st.History {
			clear(e.Warn)
			clear(e.Crit)
			clear(e.Error)
		}
		s.status[ak] = st
		if a.Log && st.Open {
			st.Open = false
			log.Printf("sched: alert %s is now log, closing, was %s", ak, st.Status())
		}
		for name, t := range notifications[ak] {
			n, present := s.Conf.Notifications[name]
			if !present {
				log.Println("sched: notification not present during restore:", name)
				continue
			}
			if a.Log {
				log.Println("sched: alert is now log, removing notification:", ak)
				continue
			}
			s.AddNotification(ak, n, t)
		}
	}
	if err := decode(dbMetadata, &s.Metadata); err != nil {
		log.Println(dbMetadata, err)
	}
	if s.maxIncidentId == 0 {
		s.createHistoricIncidents()
	}
	s.Search.Copy()
	log.Println("RestoreState done in", time.Since(start))
	return nil
}

type storedConfig struct {
	Text     string
	LastUsed time.Time
}

// Saves the provided config text in state file for later access.
// Returns a hash of the file to be used as a retreival key.
func (s *Schedule) SaveTempConfig(text string) (hash string, err error) {
	sig := md5.Sum([]byte(text))
	b64 := base64.StdEncoding.EncodeToString(sig[0:5])
	data := storedConfig{Text: text, LastUsed: time.Now()}
	bindata, err := json.Marshal(data)
	if err != nil {
		return "", err
	}
	err = s.db.Update(func(tx *bolt.Tx) error {
		b, err := tx.CreateBucketIfNotExists([]byte(dbConfigTextBucket))
		if err != nil {
			return err
		}
		return b.Put([]byte(b64), bindata)
	})
	if err != nil {
		return "", err
	}
	return b64, nil
}

// Retreive the specified config text from state file.
func (s *Schedule) LoadTempConfig(hash string) (text string, err error) {
	config := storedConfig{}
	err = s.db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(dbConfigTextBucket))
		data := b.Get([]byte(hash))
		if data == nil || len(data) == 0 {
			return fmt.Errorf("Config text '%s' not found", hash)
		}
		return json.Unmarshal(data, &config)
	})
	if err != nil {
		return "", err
	}
	go s.SaveTempConfig(config.Text) //refresh timestamp.
	return config.Text, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"

	"github.com/yml/nsqthumbnailer"
)

var (
	supportedExt = []string{".jpeg", ".jpg", ".gif"}
	srcDir       string
	dstDir       string
	thumbOpts    string
	postURL      string
	tOpts        []nsqthumbnailer.ThumbnailOpt
)

func init() {
	flag.StringVar(&srcDir, "src-directory", "", "Directory containing the src images.")
	flag.StringVar(&dstDir, "dst-directory", "", "Destination Directory.")
	flag.StringVar(&thumbOpts, "thumbnail-options", "", "Thumbnail options")
	flag.StringVar(&postURL, "post-url", "", "Url to post the thumbnail generation request")
}

func thumbnailFileRequest(file string) error {
	tmJson, err := json.Marshal(nsqthumbnailer.ThumbnailerMessage{
		SrcImage:  fmt.Sprintf("file://%s", file),
		DstFolder: dstDir,
		Opts:      tOpts,
	})

	resp, err := http.Post(postURL, "application/json", bytes.NewReader(tmJson))
	if err != nil {
		return fmt.Errorf("[ERROR] An error occured while POSTing the thumbnail generation request, %s, ", err)
	}
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("[ERROR] Status code for the thumbnail request %d", resp.StatusCode)
	}
	return nil
}

func fileWalkFn(file string, info os.FileInfo, err error) error {
	ext := path.Ext(file)
	if !info.IsDir() {
		for _, e := range supportedExt {
			if e == ext {
				err := thumbnailFileRequest(file)
				if err != nil {
					// Print the error and carry on
					fmt.Println(err)
				}
				return nil
			}
		}
		fmt.Println("[ERROR] This extension is not supported:", file)
		return filepath.SkipDir
	}
	return nil
}

func fileWalk(p string) {
	filepath.Walk(p, fileWalkFn)
}

func main() {
	var srcURL *url.URL
	flag.Parse()
	if srcDir == "" {
		fmt.Println("\nbulk-loader requires a `src-directory`\n")
		flag.Usage()
		return
	}
	srcURL, err := url.Parse(srcDir)
	if err != nil {
		fmt.Println("\nfailed to parse srcDir into an URL, %s \n\n")
		flag.Usage()
		return
	}

	if dstDir == "" {
		fmt.Println("\nbulk-loader requires a `dst-directory`\n")
		flag.Usage()
		return
	}
	if postURL == "" {
		fmt.Println("\nbulk-loader requires a `post-url`\n")
		flag.Usage()
		return
	}
	if thumbOpts == "" {
		fmt.Println("\nbulk-loader requires a `thumbnail-options`\n")
		flag.Usage()
		return
	}
	err = json.Unmarshal([]byte(thumbOpts), &tOpts)
	if err != nil {
		fmt.Printf("\nFailed to parse the thumbnail-options, %s \n\n", err)
		flag.Usage()
		return
	}

	if srcURL.Scheme == "file" {
		fileWalk(srcURL.Path)
		return
	} else {
		fmt.Printf("\nsrc-directory scheme (%s) is not supported\n\n", srcURL.Scheme)
		flag.Usage()
		return
	}
}
<commit_msg>Adding a cmd flag to preserve the dir structure<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"

	"github.com/yml/nsqthumbnailer"
)

var (
	supportedExt      = []string{".jpeg", ".jpg", ".gif"}
	srcDir            string
	srcPath           string
	dstDir            string
	thumbOpts         string
	postURL           string
	preserveStructure bool
	tOpts             []nsqthumbnailer.ThumbnailOpt
)

func init() {
	flag.StringVar(&srcDir, "src-directory", "", "Directory containing the src images.")
	flag.StringVar(&dstDir, "dst-directory", "", "Destination Directory.")
	flag.StringVar(&thumbOpts, "thumbnail-options", "", "Thumbnail options")
	flag.StringVar(&postURL, "post-url", "", "Url to post the thumbnail generation request")
	flag.BoolVar(&preserveStructure, "preserve-structure", false, "Preseve the folder structure from `src-directory` to `dst-directory`")
}

func thumbnailFileRequest(file string) error {
	var dstPath string
	if preserveStructure {
		rel, err := filepath.Rel(srcPath, file)
		if err != nil {
			return fmt.Errorf("[ERROR] failed to retrieve the relative path", err)
		}
		dstPath = fmt.Sprintf("%s%s", dstDir, filepath.Dir(rel))
	} else {
		dstPath = dstDir
	}
	tmJson, err := json.Marshal(nsqthumbnailer.ThumbnailerMessage{
		SrcImage:  fmt.Sprintf("file://%s", file),
		DstFolder: dstPath,
		Opts:      tOpts,
	})

	resp, err := http.Post(postURL, "application/json", bytes.NewReader(tmJson))
	if err != nil {
		return fmt.Errorf("[ERROR] An error occured while POSTing the thumbnail generation request, %s, ", err)
	}
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("[ERROR] Status code for the thumbnail request %d", resp.StatusCode)
	}
	return nil
}

func fileWalkFn(file string, info os.FileInfo, err error) error {
	ext := path.Ext(file)
	if !info.IsDir() {
		for _, e := range supportedExt {
			if e == ext {
				err := thumbnailFileRequest(file)
				if err != nil {
					// Print the error and carry on
					fmt.Println(err)
				}
				return nil
			}
		}
		fmt.Println("[ERROR] This extension is not supported:", file)
		return filepath.SkipDir
	}
	return nil
}

func fileWalk(p string) {
	filepath.Walk(p, fileWalkFn)
}

func main() {
	var srcURL *url.URL
	flag.Parse()
	if srcDir == "" {
		fmt.Println("\nbulk-loader requires a `src-directory`\n")
		flag.Usage()
		return
	}
	srcURL, err := url.Parse(srcDir)
	if err != nil {
		fmt.Println("\nfailed to parse srcDir into an URL, %s \n\n")
		flag.Usage()
		return
	}

	if dstDir == "" {
		fmt.Println("\nbulk-loader requires a `dst-directory`\n")
		flag.Usage()
		return
	}
	if postURL == "" {
		fmt.Println("\nbulk-loader requires a `post-url`\n")
		flag.Usage()
		return
	}
	if thumbOpts == "" {
		fmt.Println("\nbulk-loader requires a `thumbnail-options`\n")
		flag.Usage()
		return
	}
	err = json.Unmarshal([]byte(thumbOpts), &tOpts)
	if err != nil {
		fmt.Printf("\nFailed to parse the thumbnail-options, %s \n\n", err)
		flag.Usage()
		return
	}

	if srcURL.Scheme == "file" {
		srcPath = srcURL.Path
		fileWalk(srcPath)
		return
	} else {
		fmt.Printf("\nsrc-directory scheme (%s) is not supported\n\n", srcURL.Scheme)
		flag.Usage()
		return
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"encoding/base64"
	"encoding/binary"
	"encoding/gob"
	"image"
	_ "image/jpeg"
	_ "image/png"
	"io"
)

type ImageSource interface {
	Read() (image.Image, error)
}

type GobImageSource struct {
	dec *gob.Decoder
}

func NewGobImageSource(r io.Reader) *GobImageSource {
	gob.Register(&image.RGBA{})

	return &GobImageSource{
		dec: gob.NewDecoder(r),
	}
}

func (gis *GobImageSource) Read() (image.Image, error) {
	var img image.Image
	if err := gis.dec.Decode(&img); err != nil {
		return nil, err
	}
	return img, nil
}

type Base64ImageSource struct {
	dec *bufio.Reader
}

func NewBase64ImageSource(r io.Reader) *Base64ImageSource {
	return &Base64ImageSource{
		dec: bufio.NewReader(base64.NewDecoder(base64.StdEncoding, r)),
	}
}

func (bis *Base64ImageSource) Read() (image.Image, error) {
	m, _, err := image.Decode(bis.dec)
	if err != nil {
		return nil, err
	}
	return m, nil
}

type RawImageSource struct {
	r io.Reader
}

func NewRawImageSource(r io.Reader) *RawImageSource {
	return &RawImageSource{
		r: r,
	}
}

func (ris *RawImageSource) Read() (image.Image, error) {
	var size uint64
	if err := binary.Read(ris.r, binary.LittleEndian, &size); err != nil {
		return nil, err
	}

	data := make([]byte, size)
	if _, err := io.ReadFull(ris.r, data); err != nil {
		return nil, err
	}

	m, _, err := image.Decode(bytes.NewReader(data))
	if err != nil {
		return nil, err
	}
	return m, nil
}
<commit_msg>dspd: preallocate image data buffer when possible<commit_after>package main

import (
	"bufio"
	"bytes"
	"encoding/base64"
	"encoding/binary"
	"encoding/gob"
	"image"
	_ "image/jpeg"
	_ "image/png"
	"io"
)

const (
	cPreallocSize = 256 * 16 * 24 // preallocate enough space for a 256x16 image
)

type ImageSource interface {
	Read() (image.Image, error)
}

type GobImageSource struct {
	dec *gob.Decoder
}

func NewGobImageSource(r io.Reader) *GobImageSource {
	gob.Register(&image.RGBA{})

	return &GobImageSource{
		dec: gob.NewDecoder(r),
	}
}

func (gis *GobImageSource) Read() (image.Image, error) {
	var img image.Image
	if err := gis.dec.Decode(&img); err != nil {
		return nil, err
	}
	return img, nil
}

type Base64ImageSource struct {
	dec *bufio.Reader
}

func NewBase64ImageSource(r io.Reader) *Base64ImageSource {
	return &Base64ImageSource{
		dec: bufio.NewReader(base64.NewDecoder(base64.StdEncoding, r)),
	}
}

func (bis *Base64ImageSource) Read() (image.Image, error) {
	m, _, err := image.Decode(bis.dec)
	if err != nil {
		return nil, err
	}
	return m, nil
}

type RawImageSource struct {
	r   io.Reader
	buf []byte
}

func NewRawImageSource(r io.Reader) *RawImageSource {
	return &RawImageSource{
		r:   r,
		buf: make([]byte, cPreallocSize),
	}
}

func (ris *RawImageSource) Read() (image.Image, error) {
	var size uint64
	if err := binary.Read(ris.r, binary.LittleEndian, &size); err != nil {
		return nil, err
	}

	// permanently increase buffer size if a bigger image is received
	if size > uint64(len(ris.buf)) {
		ris.buf = make([]byte, size)
	}

	if _, err := io.ReadFull(ris.r, ris.buf[:size]); err != nil {
		return nil, err
	}

	m, _, err := image.Decode(bytes.NewReader(ris.buf))
	if err != nil {
		return nil, err
	}
	return m, nil
}
<|endoftext|>


<commit_before>package entrypoint

import (
	"context"
	"fmt"
	"os"
	"time"

	"github.com/datawire/dlib/dcontext"
	"github.com/datawire/dlib/dgroup"
)

func runEnvoy(ctx context.Context, envoyHUP chan os.Signal) error {
	// Wait until we get a SIGHUP to start envoy.
	//var bootstrap string
	select {
	case <-envoyHUP:
		/*bytes, err := ioutil.ReadFile(GetEnvoyBootstrapFile())
		if err != nil {
			panic(err)
		}
		bootstrap = string(bytes)*/
	case <-ctx.Done():
		return nil
	}

	// Try to run envoy directly, but fallback to running it inside docker if there is
	// no envoy executable available.
	if IsEnvoyAvailable() {
		cmd := subcommand(ctx, "envoy", GetEnvoyFlags()...)
		if envbool("DEV_SHUTUP_ENVOY") {
			cmd.Stdout = nil
			cmd.Stderr = nil
		}
		return cmd.Run()
	} else {
		// For some reason docker only sometimes passes the signal onto the process inside
		// the container, so we setup this cleanup function so that in the docker case we
		// can do a docker kill, just to be sure it is really dead and we don't leave an
		// envoy lying around.

		// Create a label unique to this invocation so we can use it to do a docker
		// kill for cleanup.
		label := fmt.Sprintf("amb-envoy-label-%d", os.Getpid())

		snapdir := GetSnapshotDir()

		group := dgroup.NewGroup(ctx, dgroup.GroupConfig{
			ShutdownOnNonError: true,
		})
		group.Go("envoy", func(ctx context.Context) error {
			// XXX: will host networking work on a mac? (probably not)
			dockerArgs := []string{
				"run", "-l", label, "--rm", "--network", "host",
				"-v", fmt.Sprintf("%s:%s", snapdir, snapdir),
				"-v", fmt.Sprintf("%s:%s", GetEnvoyBootstrapFile(), GetEnvoyBootstrapFile()),
			}

			if envbool("DEV_ENVOY_DOCKER_PRIVILEGED") {
				dockerArgs = append(dockerArgs, "--privileged")
			}

			dockerArgs = append(dockerArgs, "--entrypoint", "envoy", "docker.io/datawire/aes:1.6.2")
			cmd := subcommand(ctx, "docker", append(dockerArgs, GetEnvoyFlags()...)...)
			if envbool("DEV_SHUTUP_ENVOY") {
				cmd.Stdout = nil
				cmd.Stderr = nil
			}
			return cmd.Run()
		})
		group.Go("cleanup", func(ctx context.Context) error {
			<-ctx.Done()
			ctx = dcontext.HardContext(ctx)

			for {
				cids := cidsForLabel(ctx, label)
				if len(cids) == 0 {
					return nil
				}

				// Give the container one second to exit
				tctx, _ := context.WithTimeout(ctx, 1*time.Second)
				wait := subcommand(tctx, "docker", append([]string{"wait"}, cids...)...)
				wait.Stdout = nil
				logExecError("docker wait", wait.Run())

				cids = cidsForLabel(ctx, label)
				if len(cids) == 0 {
					return nil
				}

				kill := subcommand(ctx, "docker", append([]string{"kill"}, cids...)...)
				kill.Stdout = nil
				logExecError("docker kill", kill.Run())
			}
		})
		return group.Wait()
	}
}
<commit_msg>(from AES) update image version<commit_after>package entrypoint

import (
	"context"
	"fmt"
	"os"
	"time"

	"github.com/datawire/dlib/dcontext"
	"github.com/datawire/dlib/dgroup"
)

func runEnvoy(ctx context.Context, envoyHUP chan os.Signal) error {
	// Wait until we get a SIGHUP to start envoy.
	//var bootstrap string
	select {
	case <-envoyHUP:
		/*bytes, err := ioutil.ReadFile(GetEnvoyBootstrapFile())
		if err != nil {
			panic(err)
		}
		bootstrap = string(bytes)*/
	case <-ctx.Done():
		return nil
	}

	// Try to run envoy directly, but fallback to running it inside docker if there is
	// no envoy executable available.
	if IsEnvoyAvailable() {
		cmd := subcommand(ctx, "envoy", GetEnvoyFlags()...)
		if envbool("DEV_SHUTUP_ENVOY") {
			cmd.Stdout = nil
			cmd.Stderr = nil
		}
		return cmd.Run()
	} else {
		// For some reason docker only sometimes passes the signal onto the process inside
		// the container, so we setup this cleanup function so that in the docker case we
		// can do a docker kill, just to be sure it is really dead and we don't leave an
		// envoy lying around.

		// Create a label unique to this invocation so we can use it to do a docker
		// kill for cleanup.
		label := fmt.Sprintf("amb-envoy-label-%d", os.Getpid())

		snapdir := GetSnapshotDir()

		group := dgroup.NewGroup(ctx, dgroup.GroupConfig{
			ShutdownOnNonError: true,
		})
		group.Go("envoy", func(ctx context.Context) error {
			// XXX: will host networking work on a mac? (probably not)
			dockerArgs := []string{
				"run", "-l", label, "--rm", "--network", "host",
				"-v", fmt.Sprintf("%s:%s", snapdir, snapdir),
				"-v", fmt.Sprintf("%s:%s", GetEnvoyBootstrapFile(), GetEnvoyBootstrapFile()),
			}

			if envbool("DEV_ENVOY_DOCKER_PRIVILEGED") {
				dockerArgs = append(dockerArgs, "--privileged")
			}

			dockerArgs = append(dockerArgs, "--entrypoint", "envoy", "docker.io/datawire/aes:1.12.2")
			cmd := subcommand(ctx, "docker", append(dockerArgs, GetEnvoyFlags()...)...)
			if envbool("DEV_SHUTUP_ENVOY") {
				cmd.Stdout = nil
				cmd.Stderr = nil
			}
			return cmd.Run()
		})
		group.Go("cleanup", func(ctx context.Context) error {
			<-ctx.Done()
			ctx = dcontext.HardContext(ctx)

			for {
				cids := cidsForLabel(ctx, label)
				if len(cids) == 0 {
					return nil
				}

				// Give the container one second to exit
				tctx, _ := context.WithTimeout(ctx, 1*time.Second)
				wait := subcommand(tctx, "docker", append([]string{"wait"}, cids...)...)
				wait.Stdout = nil
				logExecError("docker wait", wait.Run())

				cids = cidsForLabel(ctx, label)
				if len(cids) == 0 {
					return nil
				}

				kill := subcommand(ctx, "docker", append([]string{"kill"}, cids...)...)
				kill.Stdout = nil
				logExecError("docker kill", kill.Run())
			}
		})
		return group.Wait()
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 Marc-Antoine Ruel. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

package main

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"
	"unicode/utf8"

	"github.com/pbnjay/memory"
	"periph.io/x/gohci"
)

// gohciBranch is a git branch name that doesn't have an high likelihood of
// conflicting.
const gohciBranch = "_gohci"

var muCmd sync.Mutex

// normalizeUTF8 returns valid UTF8 from potentially incorrectly encoded data
// from an untrusted process.
func normalizeUTF8(b []byte) []byte {
	if utf8.Valid(b) {
		return b
	}
	var out []byte
	for len(b) != 0 {
		r, size := utf8.DecodeRune(b)
		if r != utf8.RuneError {
			out = append(out, b[:size]...)
		}
		b = b[size:]
	}
	return out
}

// roundDuration returns rounded time with approximatively 4~5 digits.
func roundDuration(t time.Duration) time.Duration {
	// Cheezy but good enough for now.
	for r := time.Second; r > time.Microsecond; r /= 10 {
		if t >= r {
			d := r / 1000
			return (t + d/2) / d * d
		}
	}
	return t
}

// roundSize rounds a size to something reasonable.
func roundSize(t uint64) string {
	orders := []string{"bytes", "Kib", "Mib", "Gib", "Tib", "Pib", "Eib"}
	i := 0
	for ; i < len(orders)-1; i++ {
		if t/1024*1024 != t || t == 0 {
			break
		}
		t /= 1024
	}
	if t > 1024 {
		return fmt.Sprintf("%.1f%s", float64(t)/1024., orders[i+1])
	}
	return fmt.Sprintf("%d%s", t, orders[i])
}

// Wrap the exec.Command() call with PATH value override.
//
// exec.Command() calls exec.Lookup() right away, and there is no way to
// override the PATH variable used by exec.Lookup(), so the process' value
// must be temporarily changed.
func getCmd(path string, cmd []string) *exec.Cmd {
	muCmd.Lock()
	defer muCmd.Unlock()
	if path != "" {
		oldpath := os.Getenv("PATH")
		_ = os.Setenv("PATH", path)
		// Restore PATH.
		defer func() {
			_ = os.Setenv("PATH", oldpath)
		}()
	}
	return exec.Command(cmd[0], cmd[1:]...)
}

// gistFile is an item in the gist.
//
// It represents either the stdout of a command or metadata. They are created
// by processing fileToPush.
type gistFile struct {
	name, content string
	success       bool
	d             time.Duration
}

//

// jobRequest is the details to run a verification job.
//
// It defines a github repository being tested in the worker gohci.yml
// configuration file, along the alternate path to use and the checks to run.
type jobRequest struct {
	org        string // Organisation name (e.g. a user)
	repo       string // Project name
	altPath    string // Alternative package path to use. Defaults to the github canonical path.
	commitHash string // commit hash, not a ref
	useSSH     bool   // useSSH tells to use ssh instead of https
	pullID     int    // pullID is the PR ID if relevant

	gopath string   // Cache of GOPATH
	path   string   // Cache of PATH
	env    []string // Precomputed environment variables
}

// newJobRequest creates a new test request for project 'org/repo' on commitHash
// and/or pullID.
func newJobRequest(org, repo, altPath, commitHash string, useSSH bool, pullID int, wd string) *jobRequest {
	// Organization names cannot contain an underscore so it 'should' be fine.
	gopath := filepath.Join(wd, org+"_"+repo)
	path := filepath.Join(gopath, "bin") + string(os.PathListSeparator) + os.Getenv("PATH")
	// Setup the environment variables.
	oldenv := os.Environ()
	env := make([]string, 0, len(oldenv))
	for _, v := range oldenv {
		if strings.HasPrefix(v, "GOPATH=") || strings.HasPrefix(v, "PATH=") {
			continue
		}
		env = append(env, v)
	}
	// GOPATH may not be set especially when running from systemd, so use the
	// local GOPATH. This is safer as this doesn't modify the host environment.
	env = append(env, "GOPATH="+gopath)
	env = append(env, "PATH="+path)
	if commitHash != "" {
		env = append(env, "GIT_SHA="+commitHash)
	}

	return &jobRequest{
		org:        org,
		repo:       repo,
		altPath:    altPath,
		commitHash: commitHash,
		useSSH:     useSSH,
		pullID:     pullID,
		gopath:     gopath,
		path:       path,
		env:        env,
	}
}

func (j *jobRequest) String() string {
	if j.pullID != 0 {
		return fmt.Sprintf("https://github.com/%s/pull/%d at https://github.com/%s/commit/%s", j.getID(), j.pullID, j.getID(), j.commitHash[:12])
	}
	return fmt.Sprintf("https://github.com/%s/commit/%s", j.getID(), j.commitHash[:12])
}

// getPath returns the path to checkout the repository into. It may be
// different than "github.com/<org>/<repo>".
func (j *jobRequest) getPath() string {
	if len(j.altPath) != 0 {
		return strings.Replace(j.altPath, "/", string(os.PathSeparator), -1)
	}
	return filepath.Join("github.com", j.org, j.repo)
}

func (j *jobRequest) cloneURL() string {
	if j.useSSH {
		return "git@github.com:" + j.getID()
	}
	return "https://github.com/" + j.getID()
}

// getID returns the "org/repo" identifier for a project.
func (j *jobRequest) getID() string {
	return j.org + "/" + j.repo
}

// findCommitHash tries to get the HEAD commit for the PR # or default branch.
func (j *jobRequest) findCommitHash() bool {
	if err := j.assertDir(); err != nil {
		return false
	}
	stdout, ok := j.run("", nil, []string{"git", "ls-remote", j.cloneURL()}, false)
	if !ok {
		log.Printf("  git ls-remote failed:\n%s", stdout)
		return false
	}
	p := "HEAD"
	if j.pullID != 0 {
		p = fmt.Sprintf("refs/pull/%d/head", j.pullID)
	}
	for _, l := range strings.Split(stdout, "\n") {
		if strings.HasSuffix(l, p) {
			j.commitHash = strings.SplitN(l, "\t", 2)[0]
			log.Printf("  Found %s for PR #%d", j.commitHash, j.pullID)
			return true
		}
	}
	log.Printf("  Didn't find remote")
	return false
}

// metadata generates the pseudo-file to present information about the worker.
func (j *jobRequest) metadata() string {
	out := fmt.Sprintf(
		"Commit:  %s\nCPUs:    %d\nRAM:     %s\nVersion: %s\nGOROOT:  %s\nGOPATH:  %s\nPATH:    %s\n",
		j.commitHash, runtime.NumCPU(), roundSize(memory.TotalMemory()), runtime.Version(), runtime.GOROOT(), j.gopath, j.path)
	if runtime.GOOS != "windows" {
		if s, err := exec.Command("uname", "-a").CombinedOutput(); err == nil {
			out += "uname:   " + strings.TrimSpace(string(s)) + "\n"
		}
	}
	return out
}

// run runs an executable and returns mangled merged stdout+stderr.
//
// Use pathOverride when running checks.
func (j *jobRequest) run(relwd string, env, cmd []string, pathOverride bool) (string, bool) {
	// Keep a copy of the one off environment variables, as we'll print them
	// later.
	dbg := strings.Join(env, " ")

	// Setup the environment variables.
	if len(env) != 0 {
		// TODO(maruel): Remove previous existing definition.
		env = append(append([]string(nil), j.env...), env...)
	} else {
		env = j.env
	}

	// Evaluate environment variables.
	cmd = append([]string(nil), cmd...)
	for i := range cmd {
		cmd[i] = os.Expand(cmd[i], func(key string) string {
			key += "="
			for _, e := range env {
				if strings.HasPrefix(e, key) {
					return e[len(key):]
				}
			}
			return ""
		})
	}
	// Log the final command.
	if len(dbg) != 0 {
		dbg += " "
	}
	dbg += strings.Join(cmd, " ")
	log.Printf("- relwd=%s : %s", relwd, dbg)

	var c *exec.Cmd
	if pathOverride {
		c = getCmd(j.path, cmd)
	} else {
		c = getCmd("", cmd)
	}
	c.Env = env
	c.Dir = filepath.Join(j.gopath, relwd)
	start := time.Now()
	out, err := c.CombinedOutput()
	duration := time.Since(start)
	exit := 0
	if err != nil {
		exit = -1
		if len(out) == 0 {
			out = []byte("<failure>\n" + err.Error() + "\n")
		}
		if exiterr, ok := err.(*exec.ExitError); ok {
			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
				exit = status.ExitStatus()
			}
		}
	}
	return fmt.Sprintf("%s $ %s  (exit:%d in %s)\n%s",
		filepath.Join("$GOPATH/src", relwd), dbg, exit, roundDuration(duration), normalizeUTF8(out)), err == nil
}

func (j *jobRequest) assertDir() error {
	repoPath := filepath.Join(j.gopath, "src", j.getPath())
	up := filepath.Dir(repoPath)
	err := os.MkdirAll(up, 0700)
	log.Printf("MkdirAll(%q) -> %v", up, err)
	if err != nil && !os.IsExist(err) {
		return err
	}
	return nil
}

// setupWorkResult is metadata to add to the 'setup' pseudo-steps.
//
// It is used to track the work as all repositories are pulled concurrently,
// then used.
type setupWorkResult struct {
	content string
	ok      bool
}

// checkout is the first part of a job.
//
// It checkouts out the primary repository at the right commit.
func (j *jobRequest) checkout() (string, bool) {
	sha := j.commitHash
	if j.pullID != 0 {
		sha = fmt.Sprintf("pull/%d/head", j.pullID)
	}
	p := filepath.Join("src", j.getPath())
	if err := os.MkdirAll(filepath.Join(j.gopath, p), 0o700); err != nil {
		return err.Error(), false
	}
	// There's a trick to checkout a single exact commit which works on older git
	// clients.
	setupCmds := [][]string{
		{"git", "init", "--quiet"},
		{"git", "remote", "add", "origin", j.cloneURL()},
		{"git", "fetch", "--quiet", "--depth", "1", "origin", sha},
		{"git", "checkout", "--quiet", "FETCH_HEAD"},
	}
	out := ""
	ok := true
	for _, c := range setupCmds {
		stdout, ok2 := j.run(p, nil, c, false)
		out += stdout
		if ok = ok && ok2; !ok {
			break
		}
	}
	return out, ok

}

// parseConfig is the third part of a job.
//
// It reads the ".gohci.yml" if there's one.
func (j *jobRequest) parseConfig(name string) ([]gohci.Check, string) {
	if p := loadProjectConfig(filepath.Join(j.gopath, "src", j.getPath(), ".gohci.yml")); p != nil {
		for _, w := range p.Workers {
			if w.Name == name {
				return w.Checks, "Using worker specific checks from the repo's .gohci.yml"
			}
		}
		for _, w := range p.Workers {
			if w.Name == "" {
				return w.Checks, "Using generic checks from the repo's .gohci.yml"
			}
		}
	}
	// Returns the default.
	return []gohci.Check{{Cmd: []string{"go", "test", "./..."}}}, "Using default check"
}

// runChecks is the fourth part of a job.
func (j *jobRequest) runChecks(checks []gohci.Check, results chan<- gistFile) bool {
	ok := true
	nb := len(strconv.Itoa(len(checks)))
	for i, c := range checks {
		start := time.Now()
		d := filepath.Join("src", j.getPath())
		if c.Dir != "" {
			// TODO(maruel): Make sure it's still within the workspace. Including
			// symlinks. That said we can't do miracles without a proper namespace.
			d = filepath.Join(d, c.Dir)
		}
		stdout, ok2 := j.run(d, c.Env, c.Cmd, true)
		results <- gistFile{fmt.Sprintf("cmd%0*d", nb, i+1), stdout, ok2, time.Since(start)}
		// Still run the other tests.
		ok = ok && ok2
	}
	return ok
}

// cleanup is both the first and the last part of a job.
func (j *jobRequest) cleanup(name string, results chan<- gistFile) bool {
	start := time.Now()
	out := ""
	ok := true
	for _, x := range []string{"bin", "src"} {
		p := filepath.Join(j.gopath, x)
		if _, err := os.Stat(p); os.IsNotExist(err) {
			// Nothing was checked out, skip silently.
			continue
		}
		if err := os.RemoveAll(p); err != nil {
			out += err.Error() + "\n"
			ok = false
		} else {
			out += "Removed " + x + "\n"
		}
	}
	results <- gistFile{name, out, ok, time.Since(start)}
	return ok
}
<commit_msg>Skip the clean up step if unnecessary<commit_after>// Copyright 2018 Marc-Antoine Ruel. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

package main

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"
	"unicode/utf8"

	"github.com/pbnjay/memory"
	"periph.io/x/gohci"
)

// gohciBranch is a git branch name that doesn't have an high likelihood of
// conflicting.
const gohciBranch = "_gohci"

var muCmd sync.Mutex

// normalizeUTF8 returns valid UTF8 from potentially incorrectly encoded data
// from an untrusted process.
func normalizeUTF8(b []byte) []byte {
	if utf8.Valid(b) {
		return b
	}
	var out []byte
	for len(b) != 0 {
		r, size := utf8.DecodeRune(b)
		if r != utf8.RuneError {
			out = append(out, b[:size]...)
		}
		b = b[size:]
	}
	return out
}

// roundDuration returns rounded time with approximatively 4~5 digits.
func roundDuration(t time.Duration) time.Duration {
	// Cheezy but good enough for now.
	for r := time.Second; r > time.Microsecond; r /= 10 {
		if t >= r {
			d := r / 1000
			return (t + d/2) / d * d
		}
	}
	return t
}

// roundSize rounds a size to something reasonable.
func roundSize(t uint64) string {
	orders := []string{"bytes", "Kib", "Mib", "Gib", "Tib", "Pib", "Eib"}
	i := 0
	for ; i < len(orders)-1; i++ {
		if t/1024*1024 != t || t == 0 {
			break
		}
		t /= 1024
	}
	if t > 1024 {
		return fmt.Sprintf("%.1f%s", float64(t)/1024., orders[i+1])
	}
	return fmt.Sprintf("%d%s", t, orders[i])
}

// Wrap the exec.Command() call with PATH value override.
//
// exec.Command() calls exec.Lookup() right away, and there is no way to
// override the PATH variable used by exec.Lookup(), so the process' value
// must be temporarily changed.
func getCmd(path string, cmd []string) *exec.Cmd {
	muCmd.Lock()
	defer muCmd.Unlock()
	if path != "" {
		oldpath := os.Getenv("PATH")
		_ = os.Setenv("PATH", path)
		// Restore PATH.
		defer func() {
			_ = os.Setenv("PATH", oldpath)
		}()
	}
	return exec.Command(cmd[0], cmd[1:]...)
}

// gistFile is an item in the gist.
//
// It represents either the stdout of a command or metadata. They are created
// by processing fileToPush.
type gistFile struct {
	name, content string
	success       bool
	d             time.Duration
}

//

// jobRequest is the details to run a verification job.
//
// It defines a github repository being tested in the worker gohci.yml
// configuration file, along the alternate path to use and the checks to run.
type jobRequest struct {
	org        string // Organisation name (e.g. a user)
	repo       string // Project name
	altPath    string // Alternative package path to use. Defaults to the github canonical path.
	commitHash string // commit hash, not a ref
	useSSH     bool   // useSSH tells to use ssh instead of https
	pullID     int    // pullID is the PR ID if relevant

	gopath string   // Cache of GOPATH
	path   string   // Cache of PATH
	env    []string // Precomputed environment variables
}

// newJobRequest creates a new test request for project 'org/repo' on commitHash
// and/or pullID.
func newJobRequest(org, repo, altPath, commitHash string, useSSH bool, pullID int, wd string) *jobRequest {
	// Organization names cannot contain an underscore so it 'should' be fine.
	gopath := filepath.Join(wd, org+"_"+repo)
	path := filepath.Join(gopath, "bin") + string(os.PathListSeparator) + os.Getenv("PATH")
	// Setup the environment variables.
	oldenv := os.Environ()
	env := make([]string, 0, len(oldenv))
	for _, v := range oldenv {
		if strings.HasPrefix(v, "GOPATH=") || strings.HasPrefix(v, "PATH=") {
			continue
		}
		env = append(env, v)
	}
	// GOPATH may not be set especially when running from systemd, so use the
	// local GOPATH. This is safer as this doesn't modify the host environment.
	env = append(env, "GOPATH="+gopath)
	env = append(env, "PATH="+path)
	if commitHash != "" {
		env = append(env, "GIT_SHA="+commitHash)
	}

	return &jobRequest{
		org:        org,
		repo:       repo,
		altPath:    altPath,
		commitHash: commitHash,
		useSSH:     useSSH,
		pullID:     pullID,
		gopath:     gopath,
		path:       path,
		env:        env,
	}
}

func (j *jobRequest) String() string {
	if j.pullID != 0 {
		return fmt.Sprintf("https://github.com/%s/pull/%d at https://github.com/%s/commit/%s", j.getID(), j.pullID, j.getID(), j.commitHash[:12])
	}
	return fmt.Sprintf("https://github.com/%s/commit/%s", j.getID(), j.commitHash[:12])
}

// getPath returns the path to checkout the repository into. It may be
// different than "github.com/<org>/<repo>".
func (j *jobRequest) getPath() string {
	if len(j.altPath) != 0 {
		return strings.Replace(j.altPath, "/", string(os.PathSeparator), -1)
	}
	return filepath.Join("github.com", j.org, j.repo)
}

func (j *jobRequest) cloneURL() string {
	if j.useSSH {
		return "git@github.com:" + j.getID()
	}
	return "https://github.com/" + j.getID()
}

// getID returns the "org/repo" identifier for a project.
func (j *jobRequest) getID() string {
	return j.org + "/" + j.repo
}

// findCommitHash tries to get the HEAD commit for the PR # or default branch.
func (j *jobRequest) findCommitHash() bool {
	if err := j.assertDir(); err != nil {
		return false
	}
	stdout, ok := j.run("", nil, []string{"git", "ls-remote", j.cloneURL()}, false)
	if !ok {
		log.Printf("  git ls-remote failed:\n%s", stdout)
		return false
	}
	p := "HEAD"
	if j.pullID != 0 {
		p = fmt.Sprintf("refs/pull/%d/head", j.pullID)
	}
	for _, l := range strings.Split(stdout, "\n") {
		if strings.HasSuffix(l, p) {
			j.commitHash = strings.SplitN(l, "\t", 2)[0]
			log.Printf("  Found %s for PR #%d", j.commitHash, j.pullID)
			return true
		}
	}
	log.Printf("  Didn't find remote")
	return false
}

// metadata generates the pseudo-file to present information about the worker.
func (j *jobRequest) metadata() string {
	out := fmt.Sprintf(
		"Commit:  %s\nCPUs:    %d\nRAM:     %s\nVersion: %s\nGOROOT:  %s\nGOPATH:  %s\nPATH:    %s\n",
		j.commitHash, runtime.NumCPU(), roundSize(memory.TotalMemory()), runtime.Version(), runtime.GOROOT(), j.gopath, j.path)
	if runtime.GOOS != "windows" {
		if s, err := exec.Command("uname", "-a").CombinedOutput(); err == nil {
			out += "uname:   " + strings.TrimSpace(string(s)) + "\n"
		}
	}
	return out
}

// run runs an executable and returns mangled merged stdout+stderr.
//
// Use pathOverride when running checks.
func (j *jobRequest) run(relwd string, env, cmd []string, pathOverride bool) (string, bool) {
	// Keep a copy of the one off environment variables, as we'll print them
	// later.
	dbg := strings.Join(env, " ")

	// Setup the environment variables.
	if len(env) != 0 {
		// TODO(maruel): Remove previous existing definition.
		env = append(append([]string(nil), j.env...), env...)
	} else {
		env = j.env
	}

	// Evaluate environment variables.
	cmd = append([]string(nil), cmd...)
	for i := range cmd {
		cmd[i] = os.Expand(cmd[i], func(key string) string {
			key += "="
			for _, e := range env {
				if strings.HasPrefix(e, key) {
					return e[len(key):]
				}
			}
			return ""
		})
	}
	// Log the final command.
	if len(dbg) != 0 {
		dbg += " "
	}
	dbg += strings.Join(cmd, " ")
	log.Printf("- relwd=%s : %s", relwd, dbg)

	var c *exec.Cmd
	if pathOverride {
		c = getCmd(j.path, cmd)
	} else {
		c = getCmd("", cmd)
	}
	c.Env = env
	c.Dir = filepath.Join(j.gopath, relwd)
	start := time.Now()
	out, err := c.CombinedOutput()
	duration := time.Since(start)
	exit := 0
	if err != nil {
		exit = -1
		if len(out) == 0 {
			out = []byte("<failure>\n" + err.Error() + "\n")
		}
		if exiterr, ok := err.(*exec.ExitError); ok {
			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
				exit = status.ExitStatus()
			}
		}
	}
	return fmt.Sprintf("%s $ %s  (exit:%d in %s)\n%s",
		filepath.Join("$GOPATH/src", relwd), dbg, exit, roundDuration(duration), normalizeUTF8(out)), err == nil
}

func (j *jobRequest) assertDir() error {
	repoPath := filepath.Join(j.gopath, "src", j.getPath())
	up := filepath.Dir(repoPath)
	err := os.MkdirAll(up, 0700)
	log.Printf("MkdirAll(%q) -> %v", up, err)
	if err != nil && !os.IsExist(err) {
		return err
	}
	return nil
}

// setupWorkResult is metadata to add to the 'setup' pseudo-steps.
//
// It is used to track the work as all repositories are pulled concurrently,
// then used.
type setupWorkResult struct {
	content string
	ok      bool
}

// checkout is the first part of a job.
//
// It checkouts out the primary repository at the right commit.
func (j *jobRequest) checkout() (string, bool) {
	sha := j.commitHash
	if j.pullID != 0 {
		sha = fmt.Sprintf("pull/%d/head", j.pullID)
	}
	p := filepath.Join("src", j.getPath())
	if err := os.MkdirAll(filepath.Join(j.gopath, p), 0o700); err != nil {
		return err.Error(), false
	}
	// There's a trick to checkout a single exact commit which works on older git
	// clients.
	setupCmds := [][]string{
		{"git", "init", "--quiet"},
		{"git", "remote", "add", "origin", j.cloneURL()},
		{"git", "fetch", "--quiet", "--depth", "1", "origin", sha},
		{"git", "checkout", "--quiet", "FETCH_HEAD"},
	}
	out := ""
	ok := true
	for _, c := range setupCmds {
		stdout, ok2 := j.run(p, nil, c, false)
		out += stdout
		if ok = ok && ok2; !ok {
			break
		}
	}
	return out, ok

}

// parseConfig is the third part of a job.
//
// It reads the ".gohci.yml" if there's one.
func (j *jobRequest) parseConfig(name string) ([]gohci.Check, string) {
	if p := loadProjectConfig(filepath.Join(j.gopath, "src", j.getPath(), ".gohci.yml")); p != nil {
		for _, w := range p.Workers {
			if w.Name == name {
				return w.Checks, "Using worker specific checks from the repo's .gohci.yml"
			}
		}
		for _, w := range p.Workers {
			if w.Name == "" {
				return w.Checks, "Using generic checks from the repo's .gohci.yml"
			}
		}
	}
	// Returns the default.
	return []gohci.Check{{Cmd: []string{"go", "test", "./..."}}}, "Using default check"
}

// runChecks is the fourth part of a job.
func (j *jobRequest) runChecks(checks []gohci.Check, results chan<- gistFile) bool {
	ok := true
	nb := len(strconv.Itoa(len(checks)))
	for i, c := range checks {
		start := time.Now()
		d := filepath.Join("src", j.getPath())
		if c.Dir != "" {
			// TODO(maruel): Make sure it's still within the workspace. Including
			// symlinks. That said we can't do miracles without a proper namespace.
			d = filepath.Join(d, c.Dir)
		}
		stdout, ok2 := j.run(d, c.Env, c.Cmd, true)
		results <- gistFile{fmt.Sprintf("cmd%0*d", nb, i+1), stdout, ok2, time.Since(start)}
		// Still run the other tests.
		ok = ok && ok2
	}
	return ok
}

// cleanup is both the first and the last part of a job.
func (j *jobRequest) cleanup(name string, results chan<- gistFile) bool {
	start := time.Now()
	out := ""
	ok := true
	for _, x := range []string{"bin", "src"} {
		p := filepath.Join(j.gopath, x)
		if _, err := os.Stat(p); os.IsNotExist(err) {
			// Nothing was checked out, skip silently.
			continue
		}
		if err := os.RemoveAll(p); err != nil {
			out += err.Error() + "\n"
			ok = false
		} else {
			out += "Removed " + x + "\n"
		}
	}
	if out != "" {
		results <- gistFile{name, out, ok, time.Since(start)}
	}
	return ok
}
<|endoftext|>


<commit_before>/* Copyright (c) 2014-2015, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/mvdan/jutgelint"
)

const (
	// Name of the HTTP form fields when uploading code
	fieldCode = "code"
	fieldLang = "lang"
	// Content-Type when serving text
	contentType = "text/plain; charset=utf-8"

	// HTTP response strings
	invalidID     = "invalid id"
	unknownAction = "unsupported action"

	maxSize = 1 * 1024 * 1024
	timeout = 10 * time.Second

	cmdName = "jutge-lintd"
	version = "0.0.1"
)

var (
	siteURL = flag.String("u", "http://localhost:8080", "URL of the site")
	listen  = flag.String("l", ":8080", "Host and port to listen to")
	workers = flag.Int("w", 4, "Number of POST workers")
)

func getCodeFromForm(r *http.Request) ([]byte, jutgelint.Lang, error) {
	var l jutgelint.Lang
	if code := r.FormValue(fieldCode); len(code) > 0 {
		lang, err := jutgelint.ParseLang(r.FormValue(fieldLang))
		if err != nil {
			return nil, l, err
		}
		return []byte(code), lang, nil
	}
	if f, h, err := r.FormFile(fieldCode); err == nil {
		defer f.Close()
		content, err := ioutil.ReadAll(f)
		if err == nil && len(content) > 0 {
			lang, err := jutgelint.ParseLangFilename(h.Filename)
			if err != nil {
				return nil, l, err
			}
			return content, lang, nil
		}
	}
	return nil, l, errors.New("no code provided")
}

func setHeaders(header http.Header, id ID, review Review) {
	modTime := review.ModTime()
	header.Set("Etag", fmt.Sprintf(`"%d-%s"`, modTime.Unix(), id))
	header.Set("Content-Type", contentType)
}

type httpHandler struct {
	store Store
	post  chan postRequest
}

type postRequest struct {
	code []byte
	lang jutgelint.Lang
	ret  chan postResult
}

type postResult struct {
	url string
	err error
}

func (h httpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		h.handleGet(w, r)
	case "POST":
		h.handlePost(w, r)
	default:
		http.Error(w, unknownAction, http.StatusBadRequest)
	}
}

func logPrintfReq(r *http.Request, format string, v ...interface{}) {
	format = "%s %s %s " + format
	rv := []interface{}{r.RemoteAddr, r.Proto, r.Method}
	v = append(rv, v...)
	log.Printf(format, v...)
}

func (h *httpHandler) handleGet(w http.ResponseWriter, r *http.Request) {
	if _, e := templates[r.URL.Path]; e {
		err := tmpl.ExecuteTemplate(w, r.URL.Path,
			struct {
				SiteURL   string
				FieldCode string
			}{
				SiteURL:   *siteURL,
				FieldCode: fieldCode,
			})
		if err != nil {
			logPrintfReq(r, "Error executing template for %s: %v", r.URL.Path, err)
		}
		return
	}
	id, err := IDFromString(r.URL.Path[1:])
	if err != nil {
		logPrintfReq(r, "Bad request: %v", err)
		http.Error(w, invalidID, http.StatusBadRequest)
		return
	}
	review, err := h.store.Get(id)
	if err != nil {
		status := http.StatusInternalServerError
		if err == ErrReviewNotFound {
			status = http.StatusNotFound
		}
		logPrintfReq(r, "Error: %v", err)
		http.Error(w, err.Error(), status)
		return
	}
	defer review.Close()
	setHeaders(w.Header(), id, review)
	logPrintfReq(r, "Served: %s/%s", *siteURL, id)
	http.ServeContent(w, r, "", review.ModTime(), review)
}

func commentCode(code []byte, lang jutgelint.Lang) ([]byte, error) {
	in := bytes.NewReader(code)
	var out bytes.Buffer
	if err := jutgelint.CheckAndCommentCode(lang, in, &out); err != nil {
		return nil, err
	}
	return out.Bytes(), nil
}

func (h *httpHandler) postWorker() {
	for {
		req := <-h.post
		comm, err := commentCode(req.code, req.lang)
		if err != nil {
			req.ret <- postResult{err: err}
			continue
		}
		id, err := h.store.Put(comm)
		if err != nil {
			req.ret <- postResult{err: err}
			continue
		}
		req.ret <- postResult{
			url: fmt.Sprintf("%s/%s", *siteURL, id),
		}
	}
}

func (h *httpHandler) handlePost(w http.ResponseWriter, r *http.Request) {
	r.Body = http.MaxBytesReader(w, r.Body, maxSize)
	code, lang, err := getCodeFromForm(r)
	if err != nil {
		logPrintfReq(r, "Bad request: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	ret := make(chan postResult)
	h.post <- postRequest{
		code: code,
		lang: lang,
		ret:  ret,
	}
	res := <-ret
	if res.err != nil {
		logPrintfReq(r, "Error: %v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	logPrintfReq(r, "Created: %s", res.url)
	fmt.Fprintf(w, "%s\n", res.url)
}

func main() {
	flag.Parse()
	loadTemplates()
	if *workers < 1 {
		log.Fatalf("Cannot have less than 1 workers!")
	}
	log.SetOutput(os.Stdout)
	log.SetPrefix(fmt.Sprintf("%s %s ", cmdName, version))
	log.Printf("siteURL = %s", *siteURL)
	log.Printf("listen  = %s", *listen)
	log.Printf("workers = %d", *workers)

	handler := httpHandler{
		post: make(chan postRequest),
	}
	var err error
	handler.store, err = NewFileStore("code")
	if err != nil {
		log.Fatalf("Could not setup file store: %v", err)
	}

	for i := 0; i < *workers; i++ {
		go handler.postWorker()
	}

	//http.Handle("/", http.TimeoutHandler(handler, timeout, ""))
	http.Handle("/", handler)
	log.Println("Up and running!")
	log.Fatal(http.ListenAndServe(*listen, nil))
}
<commit_msg>Small style fixes; use alpha1 as version<commit_after>/* Copyright (c) 2014-2015, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/mvdan/jutgelint"
)

const (
	// Name of the HTTP form fields when uploading code
	fieldCode = "code"
	fieldLang = "lang"

	// Content-Type when serving text
	contentType = "text/plain; charset=utf-8"

	// HTTP response strings
	invalidID     = "invalid id"
	unknownAction = "unsupported action"

	maxSize = 1 * 1024 * 1024
	timeout = 10 * time.Second

	cmdName = "jutge-lintd"
	version = "alpha1"
)

var (
	siteURL = flag.String("u", "http://localhost:8080", "URL of the site")
	listen  = flag.String("l", ":8080", "Host and port to listen to")
	workers = flag.Int("w", 4, "Number of POST workers")
)

func getCodeFromForm(r *http.Request) ([]byte, jutgelint.Lang, error) {
	var l jutgelint.Lang
	if code := r.FormValue(fieldCode); len(code) > 0 {
		lang, err := jutgelint.ParseLang(r.FormValue(fieldLang))
		if err != nil {
			return nil, l, err
		}
		return []byte(code), lang, nil
	}
	if f, h, err := r.FormFile(fieldCode); err == nil {
		defer f.Close()
		content, err := ioutil.ReadAll(f)
		if err == nil && len(content) > 0 {
			lang, err := jutgelint.ParseLangFilename(h.Filename)
			if err != nil {
				return nil, l, err
			}
			return content, lang, nil
		}
	}
	return nil, l, errors.New("no code provided")
}

func setHeaders(header http.Header, id ID, review Review) {
	modTime := review.ModTime()
	header.Set("Etag", fmt.Sprintf(`"%d-%s"`, modTime.Unix(), id))
	header.Set("Content-Type", contentType)
}

type httpHandler struct {
	store Store
	post  chan postRequest
}

type postRequest struct {
	code []byte
	lang jutgelint.Lang
	ret  chan postResult
}

type postResult struct {
	url string
	err error
}

func (h httpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		h.handleGet(w, r)
	case "POST":
		h.handlePost(w, r)
	default:
		http.Error(w, unknownAction, http.StatusBadRequest)
	}
}

func logPrintfReq(r *http.Request, format string, v ...interface{}) {
	format = "%s %s %s " + format
	rv := []interface{}{r.RemoteAddr, r.Proto, r.Method}
	v = append(rv, v...)
	log.Printf(format, v...)
}

func (h *httpHandler) handleGet(w http.ResponseWriter, r *http.Request) {
	if _, e := templates[r.URL.Path]; e {
		err := tmpl.ExecuteTemplate(w, r.URL.Path,
			struct {
				SiteURL   string
				FieldCode string
			}{
				SiteURL:   *siteURL,
				FieldCode: fieldCode,
			})
		if err != nil {
			logPrintfReq(r, "Error executing template for %s: %v", r.URL.Path, err)
		}
		return
	}
	id, err := IDFromString(r.URL.Path[1:])
	if err != nil {
		logPrintfReq(r, "Bad request: %v", err)
		http.Error(w, invalidID, http.StatusBadRequest)
		return
	}
	review, err := h.store.Get(id)
	if err != nil {
		status := http.StatusInternalServerError
		if err == ErrReviewNotFound {
			status = http.StatusNotFound
		}
		logPrintfReq(r, "Error: %v", err)
		http.Error(w, err.Error(), status)
		return
	}
	defer review.Close()
	setHeaders(w.Header(), id, review)
	logPrintfReq(r, "Served: %s/%s", *siteURL, id)
	http.ServeContent(w, r, "", review.ModTime(), review)
}

func commentCode(code []byte, lang jutgelint.Lang) ([]byte, error) {
	in := bytes.NewReader(code)
	var out bytes.Buffer
	if err := jutgelint.CheckAndCommentCode(lang, in, &out); err != nil {
		return nil, err
	}
	return out.Bytes(), nil
}

func (h *httpHandler) postWorker() {
	for {
		req := <-h.post
		comm, err := commentCode(req.code, req.lang)
		if err != nil {
			req.ret <- postResult{err: err}
			continue
		}
		id, err := h.store.Put(comm)
		if err != nil {
			req.ret <- postResult{err: err}
			continue
		}
		req.ret <- postResult{
			url: fmt.Sprintf("%s/%s", *siteURL, id),
		}
	}
}

func (h *httpHandler) handlePost(w http.ResponseWriter, r *http.Request) {
	r.Body = http.MaxBytesReader(w, r.Body, maxSize)
	code, lang, err := getCodeFromForm(r)
	if err != nil {
		logPrintfReq(r, "Bad request: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	ret := make(chan postResult)
	h.post <- postRequest{
		code: code,
		lang: lang,
		ret:  ret,
	}
	res := <-ret
	if res.err != nil {
		logPrintfReq(r, "Error: %v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	logPrintfReq(r, "Created: %s", res.url)
	fmt.Fprintln(w, res.url)
}

func main() {
	flag.Parse()
	loadTemplates()
	if *workers < 1 {
		log.Fatalf("Cannot have less than 1 workers!")
	}
	log.SetOutput(os.Stdout)
	log.SetPrefix(fmt.Sprintf("%s %s ", cmdName, version))
	log.Printf("siteURL = %s", *siteURL)
	log.Printf("listen  = %s", *listen)
	log.Printf("workers = %d", *workers)

	handler := httpHandler{
		post: make(chan postRequest),
	}
	var err error
	handler.store, err = NewFileStore("code")
	if err != nil {
		log.Fatalf("Could not setup file store: %v", err)
	}

	for i := 0; i < *workers; i++ {
		go handler.postWorker()
	}
	//http.Handle("/", http.TimeoutHandler(handler, timeout, ""))
	http.Handle("/", handler)
	log.Println("Up and running!")
	log.Fatal(http.ListenAndServe(*listen, nil))
}
<|endoftext|>


<commit_before>package topology

import (
	"fmt"
	"github.com/codegangsta/cli"
	"os"
	"pfi/sensorbee/sensorbee/client"
)

var (
	testMode     bool
	testExitCode int
)

func panicHandler() {
	// Because all tests uses this feature, they cannot be run in parallel.
	// Commands use this dirty logic because there's no other ways to report
	// errors due to cli library's limitation.
	if e := recover(); e != nil {
		ec, ok := e.(int)
		if !ok {
			panic(e)
		}

		if testMode {
			testExitCode = ec
		} else {
			os.Exit(e.(int))
		}

	} else if testMode {
		testExitCode = 0
	}
}

// SetUp sets up a subcommand for topology manipulation.
func SetUp() cli.Command {
	cmd := cli.Command{
		Name:        "topology",
		Aliases:     []string{"t"},
		Usage:       "manipulate topologies on SensorBee",
		Description: "topology command allows users to create, delete, list topologies",
		Action:      cli.ShowSubcommandHelp,
		Subcommands: []cli.Command{
			setUpCreate(),
			setUpList(),
			setUpDelete(),
		},
	}
	return cmd
}

var (
	commonFlags = []cli.Flag{
		cli.StringFlag{ // TODO: share this flag with others
			Name:  "uri",
			Value: "http://localhost:8090/",
			Usage: "the address of the target SensorBee server",
		},
		cli.StringFlag{ // TODO: share this flag with others
			Name:  "api-version",
			Value: "v1",
			Usage: "target API version",
		},
	}
)

func validateFlags(c *cli.Context) {
	if err := client.ValidateURL(c.String("uri")); err != nil {
		fmt.Fprintln(os.Stderr, "--uri flag has an invalid value: %v", err)
		panic(1)
	}
	if err := client.ValidateAPIVersion(c.String("api-version")); err != nil {
		fmt.Fprintln(os.Stderr, err)
		panic(1)
	}
	// TODO: check other flags
}

func newRequester(c *cli.Context) *client.Requester {
	r, err := client.NewRequester(c.String("uri"), c.String("api-version"))
	if err != nil {
		fmt.Fprintf(os.Stderr, "Cannot create a API requester: %v\n", err)
		panic(1)
	}
	return r
}

func do(c *cli.Context, method client.Method, path string, body interface{}, baseErrMsg string) *client.Response {
	req := newRequester(c)
	res, err := req.Do(method, path, body)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v: %v\n", baseErrMsg, err)
		panic(1)
	}
	if res.IsError() {
		errRes, err := res.Error()
		if err != nil {
			fmt.Fprintf(os.Stderr, "%v and failed to parse error information: %v\n", baseErrMsg, err)
			panic(1)
		}
		fmt.Fprintf(os.Stderr, "%v: %v, %v: %v\n", baseErrMsg, errRes.Code, errRes.RequestID, errRes.Message)
		panic(1)
	}
	return res
}
<commit_msg>Add EnvVar to sensorbee topology --uri option<commit_after>package topology

import (
	"fmt"
	"github.com/codegangsta/cli"
	"os"
	"pfi/sensorbee/sensorbee/client"
)

var (
	testMode     bool
	testExitCode int
)

func panicHandler() {
	// Because all tests uses this feature, they cannot be run in parallel.
	// Commands use this dirty logic because there's no other ways to report
	// errors due to cli library's limitation.
	if e := recover(); e != nil {
		ec, ok := e.(int)
		if !ok {
			panic(e)
		}

		if testMode {
			testExitCode = ec
		} else {
			os.Exit(e.(int))
		}

	} else if testMode {
		testExitCode = 0
	}
}

// SetUp sets up a subcommand for topology manipulation.
func SetUp() cli.Command {
	cmd := cli.Command{
		Name:        "topology",
		Aliases:     []string{"t"},
		Usage:       "manipulate topologies on SensorBee",
		Description: "topology command allows users to create, delete, list topologies",
		Action:      cli.ShowSubcommandHelp,
		Subcommands: []cli.Command{
			setUpCreate(),
			setUpList(),
			setUpDelete(),
		},
	}
	return cmd
}

var (
	commonFlags = []cli.Flag{
		cli.StringFlag{ // TODO: share this flag with others
			Name:   "uri",
			Value:  "http://localhost:8090/",
			Usage:  "the address of the target SensorBee server",
			EnvVar: "SENSORBEE_URI",
		},
		cli.StringFlag{ // TODO: share this flag with others
			Name:  "api-version",
			Value: "v1",
			Usage: "target API version",
		},
	}
)

func validateFlags(c *cli.Context) {
	if err := client.ValidateURL(c.String("uri")); err != nil {
		fmt.Fprintln(os.Stderr, "--uri flag has an invalid value: %v", err)
		panic(1)
	}
	if err := client.ValidateAPIVersion(c.String("api-version")); err != nil {
		fmt.Fprintln(os.Stderr, err)
		panic(1)
	}
	// TODO: check other flags
}

func newRequester(c *cli.Context) *client.Requester {
	r, err := client.NewRequester(c.String("uri"), c.String("api-version"))
	if err != nil {
		fmt.Fprintf(os.Stderr, "Cannot create a API requester: %v\n", err)
		panic(1)
	}
	return r
}

func do(c *cli.Context, method client.Method, path string, body interface{}, baseErrMsg string) *client.Response {
	req := newRequester(c)
	res, err := req.Do(method, path, body)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v: %v\n", baseErrMsg, err)
		panic(1)
	}
	if res.IsError() {
		errRes, err := res.Error()
		if err != nil {
			fmt.Fprintf(os.Stderr, "%v and failed to parse error information: %v\n", baseErrMsg, err)
			panic(1)
		}
		fmt.Fprintf(os.Stderr, "%v: %v, %v: %v\n", baseErrMsg, errRes.Code, errRes.RequestID, errRes.Message)
		panic(1)
	}
	return res
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"os"
	"time"

	"github.com/bpicode/fritzctl/console"
	"github.com/bpicode/fritzctl/fritz"
	"github.com/bpicode/fritzctl/logger"
	"github.com/bpicode/fritzctl/stringutils"
	"github.com/spf13/cobra"
)

var listThermostatsCmd = &cobra.Command{
	Use:     "thermostats",
	Short:   "List the available smart home thermostats",
	Long:    "List the available smart home devices [thermostats] and associated data.",
	Example: "fritzctl list thermostats",
	RunE:    listThermostats,
}

func init() {
	listCmd.AddCommand(listThermostatsCmd)
}

func listThermostats(_ *cobra.Command, _ []string) error {
	c := homeAutoClient()
	devs, err := c.List()
	assertNoErr(err, "cannot obtain thermostats device data")
	logger.Success("Device data:")

	table := thermostatsTable()
	appendThermostats(devs, table)
	table.Print(os.Stdout)
	return nil
}
var errorCodesVsDescriptions = map[string]string{
	"":  "",
	"0": "",
	"1": " Thermostat adjustment not possible. Is the device mounted correctly?",
	"2": " Valve plunger cannot be driven far enough. Possible solutions: Open and close the plunger a couple of times by hand. Check if the battery is too weak.",
	"3": " Valve plunger cannot be moved. Is it blocked?",
	"4": " Preparing installation.",
	"5": " Device in mode 'INSTALLATION'. It can be mounted now.",
	"6": " Device is adjusting to the valve plunger.",
}

func thermostatsTable() *console.Table {
	return console.NewTable(console.Headers(
		"NAME",
		"PRODUCT",
		"PRESENT",
		"LOCK (BOX/DEV)",
		"MEASURED [°C]",
		"OFFSET [°C]",
		"WANT [°C]",
		"SAVING [°C]",
		"COMFORT [°C]",
		"NEXT",
		"STATE",
		"BATTERY",
	))
}

func appendThermostats(devs *fritz.Devicelist, table *console.Table) {
	for _, dev := range devs.Thermostats() {
		table.Append(thermostatColumns(dev))
	}
}

func thermostatColumns(dev fritz.Device) []string {
	var columnValues []string
	columnValues = appendMetadata(columnValues, dev)
	columnValues = appendRuntimeFlags(columnValues, dev)
	columnValues = appendTemperatureValues(columnValues, dev)
	columnValues = appendRuntimeWarnings(columnValues, dev)
	return columnValues
}

func appendMetadata(cols []string, dev fritz.Device) []string {
	return append(cols, dev.Name, fmt.Sprintf("%s %s", dev.Manufacturer, dev.Productname))
}

func appendRuntimeFlags(cols []string, dev fritz.Device) []string {
	return append(cols,
		console.IntToCheckmark(dev.Present),
		console.StringToCheckmark(dev.Thermostat.Lock)+"/"+console.StringToCheckmark(dev.Thermostat.DeviceLock))
}

func appendRuntimeWarnings(cols []string, dev fritz.Device) []string {
	return append(cols, errorCode(dev.Thermostat.ErrorCode),
		console.Stoc(dev.Thermostat.BatteryLow).Inverse().String())
}

func appendTemperatureValues(cols []string, dev fritz.Device) []string {
	return append(cols,
		dev.Thermostat.FmtMeasuredTemperature(),
		dev.Temperature.FmtOffset(),
		dev.Thermostat.FmtGoalTemperature(),
		dev.Thermostat.FmtSavingTemperature(),
		dev.Thermostat.FmtComfortTemperature(),
		fmtNextChange(dev.Thermostat.NextChange))
}
func fmtNextChange(n fritz.NextChange) string {
	return stringutils.DefaultIfEmpty(n.FmtTimestamp(time.Now()), "?") +
		" -> " +
		stringutils.DefaultIfEmpty(n.FmtGoalTemperature(), "?") +
		"°C"
}

func errorCode(ec string) string {
	checkMark := console.Stoc(ec).Inverse()
	return checkMark.String() + errorCodesVsDescriptions[ec]
}
<commit_msg>gofmt<commit_after>package cmd

import (
	"fmt"
	"os"
	"time"

	"github.com/bpicode/fritzctl/console"
	"github.com/bpicode/fritzctl/fritz"
	"github.com/bpicode/fritzctl/logger"
	"github.com/bpicode/fritzctl/stringutils"
	"github.com/spf13/cobra"
)

var listThermostatsCmd = &cobra.Command{
	Use:     "thermostats",
	Short:   "List the available smart home thermostats",
	Long:    "List the available smart home devices [thermostats] and associated data.",
	Example: "fritzctl list thermostats",
	RunE:    listThermostats,
}

func init() {
	listCmd.AddCommand(listThermostatsCmd)
}

func listThermostats(_ *cobra.Command, _ []string) error {
	c := homeAutoClient()
	devs, err := c.List()
	assertNoErr(err, "cannot obtain thermostats device data")
	logger.Success("Device data:")

	table := thermostatsTable()
	appendThermostats(devs, table)
	table.Print(os.Stdout)
	return nil
}

var errorCodesVsDescriptions = map[string]string{
	"":  "",
	"0": "",
	"1": " Thermostat adjustment not possible. Is the device mounted correctly?",
	"2": " Valve plunger cannot be driven far enough. Possible solutions: Open and close the plunger a couple of times by hand. Check if the battery is too weak.",
	"3": " Valve plunger cannot be moved. Is it blocked?",
	"4": " Preparing installation.",
	"5": " Device in mode 'INSTALLATION'. It can be mounted now.",
	"6": " Device is adjusting to the valve plunger.",
}

func thermostatsTable() *console.Table {
	return console.NewTable(console.Headers(
		"NAME",
		"PRODUCT",
		"PRESENT",
		"LOCK (BOX/DEV)",
		"MEASURED [°C]",
		"OFFSET [°C]",
		"WANT [°C]",
		"SAVING [°C]",
		"COMFORT [°C]",
		"NEXT",
		"STATE",
		"BATTERY",
	))
}

func appendThermostats(devs *fritz.Devicelist, table *console.Table) {
	for _, dev := range devs.Thermostats() {
		table.Append(thermostatColumns(dev))
	}
}

func thermostatColumns(dev fritz.Device) []string {
	var columnValues []string
	columnValues = appendMetadata(columnValues, dev)
	columnValues = appendRuntimeFlags(columnValues, dev)
	columnValues = appendTemperatureValues(columnValues, dev)
	columnValues = appendRuntimeWarnings(columnValues, dev)
	return columnValues
}

func appendMetadata(cols []string, dev fritz.Device) []string {
	return append(cols, dev.Name, fmt.Sprintf("%s %s", dev.Manufacturer, dev.Productname))
}

func appendRuntimeFlags(cols []string, dev fritz.Device) []string {
	return append(cols,
		console.IntToCheckmark(dev.Present),
		console.StringToCheckmark(dev.Thermostat.Lock)+"/"+console.StringToCheckmark(dev.Thermostat.DeviceLock))
}

func appendRuntimeWarnings(cols []string, dev fritz.Device) []string {
	return append(cols, errorCode(dev.Thermostat.ErrorCode),
		console.Stoc(dev.Thermostat.BatteryLow).Inverse().String())
}

func appendTemperatureValues(cols []string, dev fritz.Device) []string {
	return append(cols,
		dev.Thermostat.FmtMeasuredTemperature(),
		dev.Temperature.FmtOffset(),
		dev.Thermostat.FmtGoalTemperature(),
		dev.Thermostat.FmtSavingTemperature(),
		dev.Thermostat.FmtComfortTemperature(),
		fmtNextChange(dev.Thermostat.NextChange))
}
func fmtNextChange(n fritz.NextChange) string {
	return stringutils.DefaultIfEmpty(n.FmtTimestamp(time.Now()), "?") +
		" -> " +
		stringutils.DefaultIfEmpty(n.FmtGoalTemperature(), "?") +
		"°C"
}

func errorCode(ec string) string {
	checkMark := console.Stoc(ec).Inverse()
	return checkMark.String() + errorCodesVsDescriptions[ec]
}
<|endoftext|>


<commit_before>package clinamespace

import (
	"github.com/containerum/chkit/cmd/util"
	"github.com/sirupsen/logrus"
	"gopkg.in/urfave/cli.v2"
)

var Delete = &cli.Command{
	Name:    "namespace",
	Aliases: aliases,
	Action: func(ctx *cli.Context) error {
		logrus.Debugf("running command delete namespace")
		client := util.GetClient(ctx)
		namespace := util.GetNamespace(ctx)
		if ctx.NArg() == 0 {
			logrus.Debugf("show help")
			return cli.ShowSubcommandHelp(ctx)
		}
		depl := ctx.Args().First()
		logrus.Debugf("deleting deployment %q from %q", depl, namespace)
		err := client.DeleteNamespace(namespace)
		return err
	},
}
<commit_msg>fix delete namespace command<commit_after>package clinamespace

import (
	"github.com/containerum/chkit/cmd/util"
	"github.com/sirupsen/logrus"
	"gopkg.in/urfave/cli.v2"
)

var Delete = &cli.Command{
	Name:    "namespace",
	Aliases: aliases,
	Action: func(ctx *cli.Context) error {
		logrus.Debugf("running command delete namespace")
		client := util.GetClient(ctx)
		if ctx.NArg() == 0 {
			logrus.Debugf("show help")
			return cli.ShowSubcommandHelp(ctx)
		}
		namespace := ctx.Args().First()
		logrus.Debugf("deleting namespace %q", namespace)
		err := client.DeleteNamespace(namespace)
		return err
	},
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"github.com/Symantec/Dominator/lib/log"
	"github.com/Symantec/proxima/common"
	"github.com/Symantec/proxima/config"
	"github.com/Symantec/scotty/influx/qlutils"
	"github.com/Symantec/scotty/lib/pool"
	"github.com/Symantec/scotty/lib/yamlutil"
	"github.com/Symantec/tricorder/go/tricorder"
	"github.com/Symantec/tricorder/go/tricorder/units"
	"github.com/influxdata/influxdb/client/v2"
	"io"
	"strconv"
	"time"
)

var (
	kDatabasesTricorderPath = "/proc/databases"
)

var (
	kErrNoSuchDatabase = errors.New("No such database.")
)

type proximaResourceType struct {
	*pool.SingleResource
}

func NewProximaResource(p *common.Proxima) *proximaResourceType {
	return &proximaResourceType{
		SingleResource: pool.NewSingleResource(p),
	}
}

func (p *proximaResourceType) Get() (uint64, *common.Proxima) {
	id, val := p.SingleResource.Get()
	return id, val.(*common.Proxima)
}

func (p *proximaResourceType) Set(proxima *common.Proxima) {
	p.SingleResource.Set(proxima)
}

// executerType executes queries across multiple influx db instances.
// executerType instances are safe to use with multiple goroutines
type executerType struct {
	proxima *proximaResourceType
}

// newExecuter returns a new instance with no configuration. Querying it
// will always yield errNoBackends.
func newExecuter() *executerType {
	proxima, err := common.NewProxima(config.Proxima{})
	if err != nil {
		panic(err)
	}
	return &executerType{proxima: NewProximaResource(proxima)}
}

// SetupWithStream sets up this instance with config file contents in r.
func (e *executerType) SetupWithStream(r io.Reader) error {
	var proximaConfig config.Proxima
	if err := yamlutil.Read(r, &proximaConfig); err != nil {
		return err
	}
	proxima, err := common.NewProxima(proximaConfig)
	if err != nil {
		return err
	}
	if err := registerProxima(proximaConfig); err != nil {
		return err
	}
	e.proxima.Set(proxima)
	return nil
}

func registerInflux(
	influx config.Influx, dir *tricorder.DirectorySpec) error {
	if err := dir.RegisterMetric(
		"endpoint",
		&influx.HostAndPort,
		units.None,
		"endpoint of influx server"); err != nil {
		return err
	}
	if err := dir.RegisterMetric(
		"database",
		&influx.Database,
		units.None,
		"database in influx server"); err != nil {
		return err
	}
	if err := dir.RegisterMetric(
		"retentionPolicy",
		&influx.Duration,
		units.None,
		"retention policy of influx server"); err != nil {
		return err
	}
	return nil
}

func registerInfluxes(
	influxes []config.Influx, dir *tricorder.DirectorySpec) error {
	influxesDir, err := dir.RegisterDirectory("influxes")
	if err != nil {
		return err
	}
	for i := range influxes {
		influxDir, err := influxesDir.RegisterDirectory(strconv.Itoa(i))
		if err != nil {
			return err
		}
		if err := registerInflux(influxes[i], influxDir); err != nil {
			return err
		}
	}
	return nil
}

func registerScotty(
	scotty config.Scotty, dir *tricorder.DirectorySpec) error {
	if err := dir.RegisterMetric(
		"endpoint",
		&scotty.HostAndPort,
		units.None,
		"endpoint of scotty server"); err != nil {
		return err
	}
	return nil
}

func registerScotties(
	scotties []config.Scotty, dir *tricorder.DirectorySpec) error {
	scottiesDir, err := dir.RegisterDirectory("scotties")
	if err != nil {
		return err
	}
	for i := range scotties {
		scottyDir, err := scottiesDir.RegisterDirectory(strconv.Itoa(i))
		if err != nil {
			return err
		}
		if err := registerScotty(scotties[i], scottyDir); err != nil {
			return err
		}
	}
	return nil
}

func registerDatabase(
	db config.Database, dir *tricorder.DirectorySpec) error {
	databaseDir, err := dir.RegisterDirectory(db.Name)
	if err != nil {
		return err
	}
	if err := registerInfluxes(db.Influxes, databaseDir); err != nil {
		return err
	}
	if err := registerScotties(db.Scotties, databaseDir); err != nil {
		return err
	}
	return nil
}

func registerProxima(proximaConfig config.Proxima) error {
	tricorder.UnregisterPath(kDatabasesTricorderPath)
	databasesDir, err := tricorder.RegisterDirectory(kDatabasesTricorderPath)
	if err != nil {
		return err
	}
	for _, db := range proximaConfig.Dbs {
		if err := registerDatabase(db, databasesDir); err != nil {
			return err
		}
	}
	return nil
}

func (e *executerType) Names() []string {
	id, p := e.proxima.Get()
	defer e.proxima.Put(id)
	return p.Names()
}

// Query runs a query against multiple influx db instances merging the results
// Query uses the logger instance to report any influx instances that are
// down.
func (e *executerType) Query(
	queryStr, database, epoch string, logger log.Logger) (
	*client.Response, error) {
	id, p := e.proxima.Get()
	defer e.proxima.Put(id)
	now := time.Now()
	query, err := qlutils.NewQuery(queryStr, now)
	if err != nil {
		return nil, err
	}
	db := p.ByName(database)
	if db == nil {
		return nil, kErrNoSuchDatabase
	}
	return db.Query(query, epoch, now, logger)
}
<commit_msg>Correctly show metrics for new proxima config file format.<commit_after>package main

import (
	"errors"
	"github.com/Symantec/Dominator/lib/log"
	"github.com/Symantec/proxima/common"
	"github.com/Symantec/proxima/config"
	"github.com/Symantec/scotty/influx/qlutils"
	"github.com/Symantec/scotty/lib/pool"
	"github.com/Symantec/scotty/lib/yamlutil"
	"github.com/Symantec/tricorder/go/tricorder"
	"github.com/Symantec/tricorder/go/tricorder/units"
	"github.com/influxdata/influxdb/client/v2"
	"io"
	"strconv"
	"time"
)

var (
	kDatabasesTricorderPath = "/proc/databases"
)

var (
	kErrNoSuchDatabase = errors.New("No such database.")
)

type proximaResourceType struct {
	*pool.SingleResource
}

func NewProximaResource(p *common.Proxima) *proximaResourceType {
	return &proximaResourceType{
		SingleResource: pool.NewSingleResource(p),
	}
}

func (p *proximaResourceType) Get() (uint64, *common.Proxima) {
	id, val := p.SingleResource.Get()
	return id, val.(*common.Proxima)
}

func (p *proximaResourceType) Set(proxima *common.Proxima) {
	p.SingleResource.Set(proxima)
}

// executerType executes queries across multiple influx db instances.
// executerType instances are safe to use with multiple goroutines
type executerType struct {
	proxima *proximaResourceType
}

// newExecuter returns a new instance with no configuration. Querying it
// will always yield errNoBackends.
func newExecuter() *executerType {
	proxima, err := common.NewProxima(config.Proxima{})
	if err != nil {
		panic(err)
	}
	return &executerType{proxima: NewProximaResource(proxima)}
}

// SetupWithStream sets up this instance with config file contents in r.
func (e *executerType) SetupWithStream(r io.Reader) error {
	var proximaConfig config.Proxima
	if err := yamlutil.Read(r, &proximaConfig); err != nil {
		return err
	}
	proxima, err := common.NewProxima(proximaConfig)
	if err != nil {
		return err
	}
	if err := registerProxima(proximaConfig); err != nil {
		return err
	}
	e.proxima.Set(proxima)
	return nil
}

func registerInflux(
	influx config.Influx, dir *tricorder.DirectorySpec) error {
	if err := dir.RegisterMetric(
		"endpoint",
		&influx.HostAndPort,
		units.None,
		"endpoint of influx server"); err != nil {
		return err
	}
	if err := dir.RegisterMetric(
		"database",
		&influx.Database,
		units.None,
		"database in influx server"); err != nil {
		return err
	}
	if err := dir.RegisterMetric(
		"retentionPolicy",
		&influx.Duration,
		units.None,
		"retention policy of influx server"); err != nil {
		return err
	}
	return nil
}

func registerInfluxes(
	influxes []config.Influx, dir *tricorder.DirectorySpec) error {
	influxesDir, err := dir.RegisterDirectory("influxes")
	if err != nil {
		return err
	}
	for i := range influxes {
		influxDir, err := influxesDir.RegisterDirectory(strconv.Itoa(i))
		if err != nil {
			return err
		}
		if err := registerInflux(influxes[i], influxDir); err != nil {
			return err
		}
	}
	return nil
}

func registerScotty(
	scotty config.Scotty, dir *tricorder.DirectorySpec) error {
	if scotty.HostAndPort != "" {
		if err := dir.RegisterMetric(
			"endpoint",
			&scotty.HostAndPort,
			units.None,
			"endpoint of scotty server"); err != nil {
			return err
		}
		return nil
	}
	if len(scotty.Partials) != 0 {
		if err := registerScotties(scotty.Partials, "partials", dir); err != nil {
			return err
		}
		return nil
	}
	if len(scotty.Scotties) != 0 {
		if err := registerScotties(scotty.Scotties, "scotties", dir); err != nil {
			return err
		}
		return nil
	}
	return errors.New("Scotty must have hostAndPort, partials, or scotties")
}

func registerScotties(
	scotties []config.Scotty, name string, dir *tricorder.DirectorySpec) error {
	scottiesDir, err := dir.RegisterDirectory(name)
	if err != nil {
		return err
	}
	for i := range scotties {
		scottyDir, err := scottiesDir.RegisterDirectory(strconv.Itoa(i))
		if err != nil {
			return err
		}
		if err := registerScotty(scotties[i], scottyDir); err != nil {
			return err
		}
	}
	return nil
}

func registerDatabase(
	db config.Database, dir *tricorder.DirectorySpec) error {
	databaseDir, err := dir.RegisterDirectory(db.Name)
	if err != nil {
		return err
	}
	if err := registerInfluxes(db.Influxes, databaseDir); err != nil {
		return err
	}
	if err := registerScotties(db.Scotties, "scotties", databaseDir); err != nil {
		return err
	}
	return nil
}

func registerProxima(proximaConfig config.Proxima) error {
	tricorder.UnregisterPath(kDatabasesTricorderPath)
	databasesDir, err := tricorder.RegisterDirectory(kDatabasesTricorderPath)
	if err != nil {
		return err
	}
	for _, db := range proximaConfig.Dbs {
		if err := registerDatabase(db, databasesDir); err != nil {
			return err
		}
	}
	return nil
}

func (e *executerType) Names() []string {
	id, p := e.proxima.Get()
	defer e.proxima.Put(id)
	return p.Names()
}

// Query runs a query against multiple influx db instances merging the results
// Query uses the logger instance to report any influx instances that are
// down.
func (e *executerType) Query(
	queryStr, database, epoch string, logger log.Logger) (
	*client.Response, error) {
	id, p := e.proxima.Get()
	defer e.proxima.Put(id)
	now := time.Now()
	query, err := qlutils.NewQuery(queryStr, now)
	if err != nil {
		return nil, err
	}
	db := p.ByName(database)
	if db == nil {
		return nil, kErrNoSuchDatabase
	}
	return db.Query(query, epoch, now, logger)
}
<|endoftext|>


<commit_before>// Copyright 2015-2021 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// watch periodically executes the executable.
//
// Synopsis:
//     watch [-n] SEC [-t] cmd-exec
//
// Description:
//    cmd-exec is executed every n seconds
//    example, watch -n 5 dmesg >> log.txt
//    : executes dmesg every 5 sec and stores the log in log.txt
//
// Options:
//     -n: time in seconds
//     -t: do not print header

package main

import (
	"fmt"
	"flag"
	"os"
	"os/exec"
	"time"
	"bufio"

)

var (
	t       	= flag.Bool("t", false, "Don't print header")
	n       	= flag.Int64("n", 2, "Loop period in SEC, default 2")
)

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "watch [-n SEC] [-t] PROG ARGS\n")
		fmt.Fprintf(os.Stderr, "Run PROG Periodically\n")

		flag.PrintDefaults()
	}
}

func main() {
	flag.Parse()

	argRem := flag.Args()

	// argRem is the remaining args(non flag) after parsing.
	if len(argRem) == 0 {
		flag.Usage()
		os.Exit(0)
	}

	seconds := uint64(*n)
	if seconds <= 0 {
		seconds = 2
	}
	out := bufio.NewWriter(os.Stdout)

	for {
		fmt.Print("\033[0;0H")
		fmt.Print("\033[J")
		if !*t {
			fmt.Printf("Every %d : %v \n\n", seconds, argRem)
		}
		cmd := exec.Command(argRem[0], argRem[1:]...)
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		if err := cmd.Run(); err != nil {
			fmt.Println("Unable to run command", err)
			os.Exit(0)
		}

		out.Flush()
		time.Sleep(time.Second * time.Duration(seconds))
	}
}
<commit_msg>Check error returned<commit_after>// Copyright 2015-2021 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// watch periodically executes the executable.
//
// Synopsis:
//     watch [-n] SEC [-t] cmd-exec
//
// Description:
//    cmd-exec is executed every n seconds
//    example, watch -n 5 dmesg >> log.txt
//    : executes dmesg every 5 sec and stores the log in log.txt
//
// Options:
//     -n: time in seconds
//     -t: do not print header

package main

import (
	"fmt"
	"flag"
	"os"
	"os/exec"
	"time"
	"bufio"
	"strings"

)

var (
	t       	= flag.Bool("t", false, "Don't print header")
	n       	= flag.Int64("n", 2, "Loop period in SEC, default 2")
)

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "watch [-n SEC] [-t] PROG ARGS\n")
		fmt.Fprintf(os.Stderr, "Run PROG Periodically\n")

		flag.PrintDefaults()
	}
}

func main() {
	flag.Parse()

	argRem := flag.Args()

	// argRem is the remaining args(non flag) after parsing.
	if len(argRem) == 0 {
		flag.Usage()
		os.Exit(0)
	}

	seconds := uint64(*n)
	if seconds <= 0 {
		seconds = 2
	}
	out := bufio.NewWriter(os.Stdout)

	for {
		fmt.Print("\033[0;0H")
		fmt.Print("\033[J")
		if !*t {
			fmt.Printf("Every %d : %v \n\n", seconds, argRem)
		}
		cmd := exec.Command(argRem[0], argRem[1:]...)
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		if err := cmd.Run(); err != nil {
			if strings.Contains(err.Error(), "executable file not found") {
				fmt.Print(err)
			 }
		}

		out.Flush()
		time.Sleep(time.Second * time.Duration(seconds))
	}
}
<|endoftext|>


<commit_before>package rdf

// Triple represent a RDF Triple
//
// RDF Triple reference : https://www.w3.org/TR/2004/REC-rdf-concepts-20040210/#section-triples
type Triple struct {
	Subject   Node
	Predicate Node
	Object    Node
}

// NewTriple creates a new Triple.
func NewTriple(subject, predicate, object Node) Triple {
	return (Triple{subject, predicate, object})
}

// Equals is a function that compare two Triples and return True if they are equals, False otherwise.
func (t Triple) Equals(other Triple) (bool, error) {
	testSubj, err := t.Subject.Equals(other.Subject)
	if err != nil {
		return false, err
	}
	testPred, err := t.Predicate.Equals(other.Predicate)
	if err != nil {
		return false, err
	}
	testObj, err := t.Object.Equals(other.Object)
	if err != nil {
		return false, err
	}
	return testSubj && testPred && testObj, nil
}

// Equivalent is a function that determine if two Triple are equivalent.
// Two triples are equivalents if their nodes are equivalents between them.
// Otherwise, the result is always False.
func (t Triple) Equivalent(other Triple) (bool, error) {
	testSubj, err := t.Subject.Equivalent(other.Subject)
	if err != nil {
		return false, err
	}
	testPred, err := t.Predicate.Equivalent(other.Predicate)
	if err != nil {
		return false, err
	}
	testObj, err := t.Object.Equivalent(other.Object)
	if err != nil {
		return false, err
	}
	return testSubj && testPred && testObj, nil
}
<commit_msg>try something with the godoc<commit_after>// Copyright (c) 2016 Thomas Minier. All rights reserved.
// Use of this source code is governed by a MIT License
// license that can be found in the LICENSE file.

package rdf

// Triple represent a RDF Triple
//
// RDF Triple reference : https://www.w3.org/TR/2004/REC-rdf-concepts-20040210/#section-triples
type Triple struct {
	Subject   Node
	Predicate Node
	Object    Node
}

// NewTriple creates a new Triple.
func NewTriple(subject, predicate, object Node) Triple {
	return (Triple{subject, predicate, object})
}

// Equals is a function that compare two Triples and return True if they are equals, False otherwise.
func (t Triple) Equals(other Triple) (bool, error) {
	testSubj, err := t.Subject.Equals(other.Subject)
	if err != nil {
		return false, err
	}
	testPred, err := t.Predicate.Equals(other.Predicate)
	if err != nil {
		return false, err
	}
	testObj, err := t.Object.Equals(other.Object)
	if err != nil {
		return false, err
	}
	return testSubj && testPred && testObj, nil
}

// Equivalent is a function that determine if two Triple are equivalent.
// Two triples are equivalents if their nodes are equivalents between them.
// Otherwise, the result is always False.
func (t Triple) Equivalent(other Triple) (bool, error) {
	testSubj, err := t.Subject.Equivalent(other.Subject)
	if err != nil {
		return false, err
	}
	testPred, err := t.Predicate.Equivalent(other.Predicate)
	if err != nil {
		return false, err
	}
	testObj, err := t.Object.Equivalent(other.Object)
	if err != nil {
		return false, err
	}
	return testSubj && testPred && testObj, nil
}
<|endoftext|>


<commit_before>package recording

import (
	"bufio"
	"context"
	"errors"
	"log"
	"net/http"
	"os"
	"time"
)

var (
	ErrMismatchWrite = errors.New("recording: did not write the same number of bytes that were read")
)

// Recording ...
type Recording struct {
	ctx    context.Context
	url    string
	fname  string
	fout   *os.File
	cancel context.CancelFunc

	Debug bool
	Err   error
}

// New creates a new Recording of the given URL to the given filename for output.
func New(url, fname string) (*Recording, error) {
	fout, err := os.Create(fname)
	if err != nil {
		return nil, err
	}

	ctx, cancel := context.WithTimeout(context.Background(), 4*time.Hour)

	r := &Recording{
		ctx:    ctx,
		url:    url,
		fname:  fname,
		fout:   fout,
		cancel: cancel,
	}

	return r, nil
}

func (r *Recording) Cancel() {
	r.cancel()
}

func (r *Recording) Done() <-chan struct{} {
	return r.ctx.Done()
}

// OutputFilename gets the output filename originally passed into New.
func (r *Recording) OutputFilename() string {
	return r.fname
}

// Start blockingly starts the recording and returns the error if one is encountered while streaming.
// This should be stopped in another goroutine.
func (r *Recording) Start() error {
	resp, err := http.Get(r.url)
	if err != nil {
		return err
	}

	defer resp.Body.Close()
	defer r.fout.Close()
	defer r.cancel()

	reader := bufio.NewReader(resp.Body)

	c := time.NewTicker(5 * time.Second)
	defer c.Stop()

	buf := make([]byte, 65536)

	for {
		time.Sleep(250 * time.Millisecond)

		select {
		case <-r.ctx.Done():
			r.fout.Sync()
			return nil
		case <-c.C:
			if r.Debug {
				log.Println("Syncing file")
			}
			err := r.fout.Sync()
			if err != nil {
				r.Err = err
				return err
			}
		default:
			nr, err := reader.Read(buf)
			if err != nil {
				r.Err = err
				return err
			}

			if r.Debug {
				log.Printf("%d bytes read", nr)
			}

			buf = buf[:nr]

			_, err = r.fout.Write(buf)
			if err != nil {
				r.Err = err
				return err
			}
		}
	}
}
<commit_msg>recording: add started time to recording objects<commit_after>package recording

import (
	"bufio"
	"context"
	"errors"
	"log"
	"net/http"
	"os"
	"time"
)

var (
	ErrMismatchWrite = errors.New("recording: did not write the same number of bytes that were read")
)

// Recording ...
type Recording struct {
	ctx     context.Context
	url     string
	fname   string
	fout    *os.File
	cancel  context.CancelFunc
	started time.Time

	Debug bool
	Err   error
}

// New creates a new Recording of the given URL to the given filename for output.
func New(url, fname string) (*Recording, error) {
	fout, err := os.Create(fname)
	if err != nil {
		return nil, err
	}

	ctx, cancel := context.WithTimeout(context.Background(), 4*time.Hour)

	r := &Recording{
		ctx:     ctx,
		url:     url,
		fname:   fname,
		fout:    fout,
		cancel:  cancel,
		started: time.Now(),
	}

	return r, nil
}

func (r *Recording) Cancel() {
	r.cancel()
}

func (r *Recording) Done() <-chan struct{} {
	return r.ctx.Done()
}

// OutputFilename gets the output filename originally passed into New.
func (r *Recording) OutputFilename() string {
	return r.fname
}

// StartTime gets start time
func (r *Recording) StartTime() time.Time {
	return r.started
}

// Start blockingly starts the recording and returns the error if one is encountered while streaming.
// This should be stopped in another goroutine.
func (r *Recording) Start() error {
	resp, err := http.Get(r.url)
	if err != nil {
		return err
	}

	defer resp.Body.Close()
	defer r.fout.Close()
	defer r.cancel()

	reader := bufio.NewReader(resp.Body)

	c := time.NewTicker(5 * time.Second)
	defer c.Stop()

	buf := make([]byte, 65536)

	for {
		time.Sleep(250 * time.Millisecond)

		select {
		case <-r.ctx.Done():
			r.fout.Sync()
			return nil
		case <-c.C:
			if r.Debug {
				log.Println("Syncing file")
			}
			err := r.fout.Sync()
			if err != nil {
				r.Err = err
				return err
			}
		default:
			nr, err := reader.Read(buf)
			if err != nil {
				r.Err = err
				return err
			}

			if r.Debug {
				log.Printf("%d bytes read", nr)
			}

			buf = buf[:nr]

			_, err = r.fout.Write(buf)
			if err != nil {
				r.Err = err
				return err
			}
		}
	}
}
<|endoftext|>