<commit_before>package stats

import (
	"fmt"
	"math"
	"sort"
)

type Stats struct {
	Values []int
	sorted bool
	sum    int
}

func (stats *Stats) Add(i int) {
	stats.Values = append(stats.Values, i)
}

func (stats *Stats) Max() int {
	stats.Sort()
	if len(stats.Values) == 0 {
		return 0
	}
	return stats.Values[len(stats.Values)-1]
}

func (stats *Stats) Min() int {
	stats.Sort()
	if len(stats.Values) == 0 {
		return 0
	}
	return stats.Values[0]
}

func (stats *Stats) Sum() int {
	if stats.sum > 0 {
		return stats.sum
	}
	for _, i := range stats.Values {
		stats.sum += i
	}
	return stats.sum
}

func (stats *Stats) Sort() {
	if !stats.sorted {
		sort.Ints(stats.Values)
		stats.sorted = true
	}
}

func (stats *Stats) Avg() int {
	return int(math.Floor(float64(stats.Sum()) / float64(len(stats.Values))))
}

func (stats *Stats) Median() int {
	return stats.Perc(50)
}

func Percentile(values []int, perc int) (o int) {
	middle := float64(len(values)) * float64(perc) / 100.0
	floor := int(math.Floor(middle))
	return values[floor]
}

func (stats *Stats) Perc(perc int) int {
	stats.Sort()
	return Percentile(stats.Values, perc)
}

func (stats *Stats) String() string {
	return fmt.Sprintf("len: %d, avg: %d, med: %d, perc_95: %d, perc_99: %d, max: %d, min: %d",
		len(stats.Values), stats.Avg(), stats.Median(), stats.Perc(95), stats.Perc(99),
		stats.Max(),
		stats.Min(),
	)
}

func (stats *Stats) Len() int {
	return len(stats.Values)
}

func (stats *Stats) Reset() {
	stats.Values = stats.Values[:0]
	stats.sorted = false
}
<commit_msg>fix stats<commit_after>package stats

import (
	"fmt"
	"math"
	"sort"
)

type Stats struct {
	Values []int
	sorted bool
	sum    int
}

func (stats *Stats) Add(i int) {
	stats.sum += i
	stats.Values = append(stats.Values, i)
}

func (stats *Stats) Max() int {
	stats.Sort()
	if len(stats.Values) == 0 {
		return 0
	}
	return stats.Values[len(stats.Values)-1]
}

func (stats *Stats) Min() int {
	stats.Sort()
	if len(stats.Values) == 0 {
		return 0
	}
	return stats.Values[0]
}

func (stats *Stats) Sum() int {
	return stats.sum
}

func (stats *Stats) Sort() {
	if !stats.sorted {
		sort.Ints(stats.Values)
		stats.sorted = true
	}
}

func (stats *Stats) Avg() int {
	return int(math.Floor(float64(stats.Sum()) / float64(len(stats.Values))))
}

func (stats *Stats) Median() int {
	return stats.Perc(50)
}

func Percentile(values []int, perc int) (o int) {
	middle := float64(len(values)) * float64(perc) / 100.0
	floor := int(math.Floor(middle))
	return values[floor]
}

func (stats *Stats) Perc(perc int) int {
	stats.Sort()
	return Percentile(stats.Values, perc)
}

func (stats *Stats) String() string {
	return fmt.Sprintf("len: %d, avg: %d, med: %d, perc_95: %d, perc_99: %d, max: %d, min: %d",
		len(stats.Values), stats.Avg(), stats.Median(), stats.Perc(95), stats.Perc(99),
		stats.Max(),
		stats.Min(),
	)
}

func (stats *Stats) Len() int {
	return len(stats.Values)
}

func (stats *Stats) Reset() {
	stats.Values = stats.Values[:0]
	stats.sorted = false
}
<|endoftext|>


<commit_before>package stats

import (
	"bytes"
	"fmt"
	"log"
	"os"
	"strings"
	"sync"
	"sync/atomic"
	"text/tabwriter"
	"time"

	"github.com/bmizerany/perks/quantile"
)

var HammerStats Stats
var _csv_file *os.File

var slowThreshold uint64 // in ms

type GetProfileCSV func(total_time float64) string // to return current profile's stats for CSV
type GetProfileCSVHeader func() string             // to return current profile's header for CSV

var _profileCSV GetProfileCSV
var _profileCSVHeader GetProfileCSVHeader

var csv_header string
var csv_last_Line string

var printedLineNum int64

var IN_WARMUP bool
var silent bool
var runId string
var _num_of_workers int

var _env_no_serverStatus bool

var AllStats []Stats

const _stats_use_channel bool = false
const STAT_BATCH_SIZE uint64 = 1 // batch 20 at one time before submit

// Stats will be an atomic, to count the number of request handled
type Stats struct {
	quants        *quantile.Stream
	totalResp     uint64 // total # of request
	totalRespTime uint64 // total response time
	totalErr      uint64 // how many error
	totalResSlow  uint64 // how many slow response
	totalSend     uint64

	lastSend     uint64
	lastResp     uint64
	lastRespTime uint64

	lastTS uint64

	firstTS uint64

	monitor *time.Ticker
	// pstat   ProfileStats

	C_response chan uint64
	C_send     chan uint64
	once       sync.Once
}

// increase the count and record response time.
func (c *Stats) RecordRes(_time uint64, method string, worker_id int) {
	if _stats_use_channel {
		c.C_response <- _time
	} else {
		atomic.AddUint64(&c.totalResp, STAT_BATCH_SIZE)
		atomic.AddUint64(&c.totalRespTime, _time)
	}

	// if longer that 200ms, it is a slow response
	if _time > slowThreshold*1000000 {
		atomic.AddUint64(&c.totalResSlow, 1)
	}

	// record percentile
	c.quants.Insert(float64(_time))
}

func (c *Stats) RecordError(worker_id int) {
	atomic.AddUint64(&HammerStats.totalErr, 1)
}

func (c *Stats) RecordSend(worker_id int) uint64 {
	if HammerStats.firstTS == 0 {
		HammerStats.firstTS = uint64(time.Now().UnixNano())
	}

	if _stats_use_channel {
		HammerStats.C_send <- 1
	} else {
		sent := atomic.AddUint64(&c.totalSend, STAT_BATCH_SIZE)
		return sent
	}
	return c.totalSend // this is not accurate, but fairly close
}

func (c *Stats) monitorHammer() {
	ts := uint64(time.Now().UnixNano())

	_total_time := (float64(ts-c.firstTS) / 1.0e9)
	sendps := float64(c.totalSend) / _total_time
	respps := float64(c.totalResp) / _total_time

	// backlog := uint64(c.totalSend - c.totalResp - c.totalErr)
	// log.Println("total", c.totalSend, "resp", c.totalResp)

	avgT := float64(c.totalRespTime) / (float64(c.totalResp) * 1.0e9)
	avgLastT := float64(c.totalRespTime-c.lastRespTime) / (float64(c.totalResp-c.lastResp) * 1.0e9)

	lastSend := c.totalSend - c.lastSend
	atomic.StoreUint64(&c.lastResp, c.totalResp)
	atomic.StoreUint64(&c.lastSend, c.totalSend)
	atomic.StoreUint64(&c.lastRespTime, c.totalRespTime)

	if c.lastTS == 0 {
		c.lastTS = ts
		if !_env_no_serverStatus {
			HammerMongoStats.MonitorMongo()
		}
		return
	}

	c.lastTS = ts
	var s_print string
	var s_csv string

	s_print = ""
	s_print_title := ""
	s_csv = ""
	if !_env_no_serverStatus {
		s_print, s_csv = HammerMongoStats.MonitorMongo()
	}

	if !silent {
		var tabWriter *tabwriter.Writer
		var tabBuffer bytes.Buffer
		tabWriter = new(tabwriter.Writer)
		tabWriter.Init(&tabBuffer, 5, 0, 1, ' ', tabwriter.AlignRight)

		// print title
		fmt.Fprintln(tabWriter,
			"time \t",
			"Total send\t",
			"req/s\t",
			"ack/s\t",
			"avg(ms)\t",
			"p99\t",
			"p97\t",
			"p95\t",
			"p50\t",
			// " pending\t", backlog,
			//" err\t", c.totalErr,
			//"|", fmt.Sprintf("%2.2f%s", (float64(c.totalErr)*100.0/float64(c.totalErr+c.totalResp)), "%"),
			//" slow\t", fmt.Sprintf("%2.2f%s", (float64(c.totalResSlow)*100.0/float64(c.totalResp)), "%"),
			"Last avg(ms)\t",
			"Last send\t",
			s_print_title)

		// print data
		fmt.Fprintln(tabWriter,
			time.Now().Format(time.Stamp), "\t",
			fmt.Sprintf("%4d\t", c.totalSend),
			fmt.Sprintf("%4.1f\t", sendps),
			fmt.Sprintf("%4.1f\t", respps),
			fmt.Sprintf("%6.3f\t", avgT*1000), // adjust to MS
			fmt.Sprintf("%6.2f\t", c.quants.Query(0.99)/1.0e6),
			fmt.Sprintf("%6.2f\t", c.quants.Query(0.97)/1.0e6),
			fmt.Sprintf("%6.2f\t", c.quants.Query(0.95)/1.0e6),
			fmt.Sprintf("%6.2f\t", c.quants.Query(0.50)/1.0e6),
			fmt.Sprintf("%6.3f\t", avgLastT*1000),
			fmt.Sprintf("%d\t", lastSend),
			s_print)

		tabWriter.Flush()
		if printedLineNum == 0 {
			fmt.Println()
			fmt.Println(string(bytes.Split(tabBuffer.Bytes(), []byte{byte('\n')})[0]))
		}
		fmt.Println(string(bytes.Split(tabBuffer.Bytes(), []byte{byte('\n')})[1]))

		printedLineNum = printedLineNum + 1

		if printedLineNum == 5 {
			printedLineNum = 0
		}

		//WriteTo(os.Stdout)
	}

	// shall monitor Mongo Here to make sure display is consistant
	// mongoStats.monitorMongo()

	csv_last_Line = fmt.Sprint(
		time.Now().Format(time.Stamp),
		",", fmt.Sprintf("%d", c.totalSend), //Total send:
		",", fmt.Sprintf("%f", sendps), //req/s:
		",", fmt.Sprintf("%f", respps), //ack/s
		",", fmt.Sprintf("%f", avgT*1000), // total avg response time
		",", fmt.Sprintf("%f", c.quants.Query(0.99)/1.0e6),
		",", fmt.Sprintf("%f", c.quants.Query(0.97)/1.0e6),
		",", fmt.Sprintf("%f", c.quants.Query(0.95)/1.0e6),
		",", fmt.Sprintf("%f", c.quants.Query(0.50)/1.0e6),
		// ",", backlog, // backlog
		",", c.totalErr, // total error
		",", fmt.Sprintf("%2.2f", (float64(c.totalErr)*100.0/float64(c.totalErr+c.totalResp))), // error ratio (%)
		",", c.totalResSlow, // total slow
		",", fmt.Sprintf("%2.2f", (float64(c.totalResSlow)*100.0/float64(c.totalResp))), // slow ratio (%)
		",", fmt.Sprintf("%f", avgLastT*1000), // | Last avg:
		",", lastSend, // last Send
		",", s_csv, // output from MongoMonitor
		",", _profileCSV(_total_time))

	_csv_file.WriteString(csv_last_Line + "\n")
}

func PrettyPrint() {
	// to pretty print output
	fmt.Println("\nFinal results:\n")
	h := strings.Split(csv_header, ",")
	l := strings.Split(csv_last_Line, ",")

	for index, _ := range l {
		fmt.Printf("%40s : %s\n", h[index], l[index])
	}
}

func (c *Stats) StartMonitoring(monitor_channel *time.Ticker) {

	// this shall run only one
	f := func() {
		HammerStats.monitor = monitor_channel

		// this is the routine to pring stats
		go func() {
			// init percentile here
			HammerStats.quants = quantile.NewTargeted(0.50, 0.95, 0.97, 0.99)

			for {
				<-HammerStats.monitor.C // rate limit for monitor routine
				if !IN_WARMUP {
					HammerStats.monitorHammer()
				}
			}
		}()

	}

	HammerStats.once.Do(f)
}

func InitProfileStat(h GetProfileCSVHeader, c GetProfileCSV) {
	// init csv log file

	var err error
	var csvFileName string

	if runId == "" {
		csvFileName = "perf_test_data.csv"
	} else {
		csvFileName = "./test_reports/" + runId + "/perf_test_data.csv"
	}

	_csv_file, err = os.Create(csvFileName)
	if err != nil {
		log.Fatalln("Erro open output csv file, error -> ", err)
	}

	// FIXME: a hacky way, or not?, need improvement
	_profileCSV = c
	_profileCSVHeader = h

	// write header
	csv_header = fmt.Sprint("timestamp,total send,req/s,ack/s,avg(ms),p99,p97,p95,p50,total err,err ratio(%),total slow,slow ratio(%),last avg(ms),last sent,") +
		HammerMongoStats.CsvHeader() + "," + _profileCSVHeader()

	_csv_file.WriteString(csv_header + "\n")
}

func SetSilent(s bool) {
	silent = s
}

func SetRunId(_id string) {
	runId = _id
}

func SetNumWorker(_w int) {
	_num_of_workers = _w
	AllStats = make([]Stats, _num_of_workers)
}

func GetRunId() string {
	return runId
}

func init() {
	_env_no_serverStatus = true
	slowThreshold = 2000 // in ms

	// fmt.Println("Init Stats")

	s := os.Getenv("HT_MONGOD_MONITOR")
	if s != "" {
		_env_no_serverStatus = false
	}

	// init channel here
	HammerStats.C_send = make(chan uint64, 1000)     // need optimze this buffer size
	HammerStats.C_response = make(chan uint64, 1000) // need optimze this buffer size

	// gorouting to collect send
	go func() {
		for {
			<-HammerStats.C_send
			HammerStats.totalSend += STAT_BATCH_SIZE
		}
	}()

	go func() {
		var r uint64
		for {
			r = <-HammerStats.C_response
			HammerStats.totalResp += STAT_BATCH_SIZE
			HammerStats.totalRespTime += r
		}
	}()
}
<commit_msg>add more percentiles<commit_after>package stats

import (
	"bytes"
	"fmt"
	"log"
	"os"
	"strings"
	"sync"
	"sync/atomic"
	"text/tabwriter"
	"time"

	"github.com/bmizerany/perks/quantile"
)

var HammerStats Stats
var _csv_file *os.File

var slowThreshold uint64 // in ms

type GetProfileCSV func(total_time float64) string // to return current profile's stats for CSV
type GetProfileCSVHeader func() string             // to return current profile's header for CSV

var _profileCSV GetProfileCSV
var _profileCSVHeader GetProfileCSVHeader

var csv_header string
var csv_last_Line string

var printedLineNum int64

var IN_WARMUP bool
var silent bool
var runId string
var _num_of_workers int

var _env_no_serverStatus bool

var AllStats []Stats

const _stats_use_channel bool = false
const STAT_BATCH_SIZE uint64 = 1 // batch 20 at one time before submit

// Stats will be an atomic, to count the number of request handled
type Stats struct {
	quants        *quantile.Stream
	totalResp     uint64 // total # of request
	totalRespTime uint64 // total response time
	totalErr      uint64 // how many error
	totalResSlow  uint64 // how many slow response
	totalSend     uint64

	lastSend     uint64
	lastResp     uint64
	lastRespTime uint64

	lastTS uint64

	firstTS uint64

	monitor *time.Ticker
	// pstat   ProfileStats

	C_response chan uint64
	C_send     chan uint64
	once       sync.Once
}

// increase the count and record response time.
func (c *Stats) RecordRes(_time uint64, method string, worker_id int) {
	if _stats_use_channel {
		c.C_response <- _time
	} else {
		atomic.AddUint64(&c.totalResp, STAT_BATCH_SIZE)
		atomic.AddUint64(&c.totalRespTime, _time)
	}

	// if longer that 200ms, it is a slow response
	if _time > slowThreshold*1000000 {
		atomic.AddUint64(&c.totalResSlow, 1)
	}

	// record percentile
	c.quants.Insert(float64(_time))
}

func (c *Stats) RecordError(worker_id int) {
	atomic.AddUint64(&HammerStats.totalErr, 1)
}

func (c *Stats) RecordSend(worker_id int) uint64 {
	if HammerStats.firstTS == 0 {
		HammerStats.firstTS = uint64(time.Now().UnixNano())
	}

	if _stats_use_channel {
		HammerStats.C_send <- 1
	} else {
		sent := atomic.AddUint64(&c.totalSend, STAT_BATCH_SIZE)
		return sent
	}
	return c.totalSend // this is not accurate, but fairly close
}

func (c *Stats) monitorHammer() {
	ts := uint64(time.Now().UnixNano())

	_total_time := (float64(ts-c.firstTS) / 1.0e9)
	sendps := float64(c.totalSend) / _total_time
	respps := float64(c.totalResp) / _total_time

	// backlog := uint64(c.totalSend - c.totalResp - c.totalErr)
	// log.Println("total", c.totalSend, "resp", c.totalResp)

	avgT := float64(c.totalRespTime) / (float64(c.totalResp) * 1.0e9)
	avgLastT := float64(c.totalRespTime-c.lastRespTime) / (float64(c.totalResp-c.lastResp) * 1.0e9)

	lastSend := c.totalSend - c.lastSend
	atomic.StoreUint64(&c.lastResp, c.totalResp)
	atomic.StoreUint64(&c.lastSend, c.totalSend)
	atomic.StoreUint64(&c.lastRespTime, c.totalRespTime)

	if c.lastTS == 0 {
		c.lastTS = ts
		if !_env_no_serverStatus {
			HammerMongoStats.MonitorMongo()
		}
		return
	}

	c.lastTS = ts
	var s_print string
	var s_csv string

	s_print = ""
	s_print_title := ""
	s_csv = ""
	if !_env_no_serverStatus {
		s_print, s_csv = HammerMongoStats.MonitorMongo()
	}

	if !silent {
		var tabWriter *tabwriter.Writer
		var tabBuffer bytes.Buffer
		tabWriter = new(tabwriter.Writer)
		tabWriter.Init(&tabBuffer, 5, 0, 1, ' ', tabwriter.AlignRight)

		// print title
		fmt.Fprintln(tabWriter,
			"time \t",
			"Total send\t",
			"req/s\t",
			"ack/s\t",
			"avg(ms)\t",
			"p99\t",
			"p97\t",
			"p95\t",
			"p85\t",
			"p75\t",
			"p65\t",
			"p50\t",
			"p35\t",
			"p15\t",
			// " pending\t", backlog,
			//" err\t", c.totalErr,
			//"|", fmt.Sprintf("%2.2f%s", (float64(c.totalErr)*100.0/float64(c.totalErr+c.totalResp)), "%"),
			//" slow\t", fmt.Sprintf("%2.2f%s", (float64(c.totalResSlow)*100.0/float64(c.totalResp)), "%"),
			"Last avg(ms)\t",
			"Last send\t",
			s_print_title)

		// print data
		fmt.Fprintln(tabWriter,
			time.Now().Format(time.Stamp), "\t",
			fmt.Sprintf("%4d\t", c.totalSend),
			fmt.Sprintf("%4.1f\t", sendps),
			fmt.Sprintf("%4.1f\t", respps),
			fmt.Sprintf("%6.3f\t", avgT*1000), // adjust to MS
			fmt.Sprintf("%6.2f\t", c.quants.Query(0.99)/1.0e6),
			fmt.Sprintf("%6.2f\t", c.quants.Query(0.97)/1.0e6),
			fmt.Sprintf("%6.2f\t", c.quants.Query(0.95)/1.0e6),
			fmt.Sprintf("%6.2f\t", c.quants.Query(0.85)/1.0e6),
			fmt.Sprintf("%6.2f\t", c.quants.Query(0.75)/1.0e6),
			fmt.Sprintf("%6.2f\t", c.quants.Query(0.65)/1.0e6),
			fmt.Sprintf("%6.2f\t", c.quants.Query(0.50)/1.0e6),
			fmt.Sprintf("%6.2f\t", c.quants.Query(0.35)/1.0e6),
			fmt.Sprintf("%6.2f\t", c.quants.Query(0.15)/1.0e6),
			fmt.Sprintf("%6.3f\t", avgLastT*1000),
			fmt.Sprintf("%d\t", lastSend),
			s_print)

		tabWriter.Flush()
		if printedLineNum == 0 {
			fmt.Println()
			fmt.Println(string(bytes.Split(tabBuffer.Bytes(), []byte{byte('\n')})[0]))
		}
		fmt.Println(string(bytes.Split(tabBuffer.Bytes(), []byte{byte('\n')})[1]))

		printedLineNum = printedLineNum + 1

		if printedLineNum == 5 {
			printedLineNum = 0
		}

		//WriteTo(os.Stdout)
	}

	// shall monitor Mongo Here to make sure display is consistant
	// mongoStats.monitorMongo()

	csv_last_Line = fmt.Sprint(
		time.Now().Format(time.Stamp),
		",", fmt.Sprintf("%d", c.totalSend), //Total send:
		",", fmt.Sprintf("%f", sendps), //req/s:
		",", fmt.Sprintf("%f", respps), //ack/s
		",", fmt.Sprintf("%f", avgT*1000), // total avg response time
		",", fmt.Sprintf("%f", c.quants.Query(0.99)/1.0e6),
		",", fmt.Sprintf("%f", c.quants.Query(0.97)/1.0e6),
		",", fmt.Sprintf("%f", c.quants.Query(0.95)/1.0e6),
		",", fmt.Sprintf("%f", c.quants.Query(0.85)/1.0e6),
		",", fmt.Sprintf("%f", c.quants.Query(0.75)/1.0e6),
		",", fmt.Sprintf("%f", c.quants.Query(0.65)/1.0e6),
		",", fmt.Sprintf("%f", c.quants.Query(0.50)/1.0e6),
		// ",", backlog, // backlog
		",", c.totalErr, // total error
		",", fmt.Sprintf("%2.2f", (float64(c.totalErr)*100.0/float64(c.totalErr+c.totalResp))), // error ratio (%)
		",", c.totalResSlow, // total slow
		",", fmt.Sprintf("%2.2f", (float64(c.totalResSlow)*100.0/float64(c.totalResp))), // slow ratio (%)
		",", fmt.Sprintf("%f", avgLastT*1000), // | Last avg:
		",", lastSend, // last Send
		",", s_csv, // output from MongoMonitor
		",", _profileCSV(_total_time))

	_csv_file.WriteString(csv_last_Line + "\n")
}

func PrettyPrint() {
	// to pretty print output
	fmt.Println("\nFinal results:\n")
	h := strings.Split(csv_header, ",")
	l := strings.Split(csv_last_Line, ",")

	for index, _ := range l {
		fmt.Printf("%40s : %s\n", h[index], l[index])
	}
}

func (c *Stats) StartMonitoring(monitor_channel *time.Ticker) {

	// this shall run only one
	f := func() {
		HammerStats.monitor = monitor_channel

		// this is the routine to pring stats
		go func() {
			// init percentile here
			HammerStats.quants = quantile.NewTargeted(0.15, 0.35, 0.50, 0.85, 0.95, 0.97, 0.99)

			for {
				<-HammerStats.monitor.C // rate limit for monitor routine
				if !IN_WARMUP {
					HammerStats.monitorHammer()
				}
			}
		}()

	}

	HammerStats.once.Do(f)
}

func InitProfileStat(h GetProfileCSVHeader, c GetProfileCSV) {
	// init csv log file

	var err error
	var csvFileName string

	if runId == "" {
		csvFileName = "perf_test_data.csv"
	} else {
		csvFileName = "./test_reports/" + runId + "/perf_test_data.csv"
	}

	_csv_file, err = os.Create(csvFileName)
	if err != nil {
		log.Fatalln("Erro open output csv file, error -> ", err)
	}

	// FIXME: a hacky way, or not?, need improvement
	_profileCSV = c
	_profileCSVHeader = h

	// write header
	csv_header = fmt.Sprint("timestamp,total send,req/s,ack/s,avg(ms),p99,p97,p95,p85,p75,p65,p50,total err,err ratio(%),total slow,slow ratio(%),last avg(ms),last sent,") +
		HammerMongoStats.CsvHeader() + "," + _profileCSVHeader()

	_csv_file.WriteString(csv_header + "\n")
}

func SetSilent(s bool) {
	silent = s
}

func SetRunId(_id string) {
	runId = _id
}

func SetNumWorker(_w int) {
	_num_of_workers = _w
	AllStats = make([]Stats, _num_of_workers)
}

func GetRunId() string {
	return runId
}

func init() {
	_env_no_serverStatus = true
	slowThreshold = 2000 // in ms

	// fmt.Println("Init Stats")

	s := os.Getenv("HT_MONGOD_MONITOR")
	if s != "" {
		_env_no_serverStatus = false
	}

	// init channel here
	HammerStats.C_send = make(chan uint64, 1000)     // need optimze this buffer size
	HammerStats.C_response = make(chan uint64, 1000) // need optimze this buffer size

	// gorouting to collect send
	go func() {
		for {
			<-HammerStats.C_send
			HammerStats.totalSend += STAT_BATCH_SIZE
		}
	}()

	go func() {
		var r uint64
		for {
			r = <-HammerStats.C_response
			HammerStats.totalResp += STAT_BATCH_SIZE
			HammerStats.totalRespTime += r
		}
	}()
}
<|endoftext|>


<commit_before>package web

import (
	"database/sql"
	"strings"

	"github.com/cloudfoundry-incubator/notifications/metrics"
	"github.com/cloudfoundry-incubator/notifications/postal/strategies"
	"github.com/cloudfoundry-incubator/notifications/web/handlers"
	"github.com/cloudfoundry-incubator/notifications/web/middleware"
	"github.com/cloudfoundry-incubator/notifications/web/services"
	"github.com/gorilla/mux"
	"github.com/ryanmoran/stack"
)

type MotherInterface interface {
	Registrar() services.Registrar
	EmailStrategy() strategies.EmailStrategy
	UserStrategy() strategies.UserStrategy
	SpaceStrategy() strategies.SpaceStrategy
	OrganizationStrategy() strategies.OrganizationStrategy
	EveryoneStrategy() strategies.EveryoneStrategy
	UAAScopeStrategy() strategies.UAAScopeStrategy
	NotificationsFinder() services.NotificationsFinder
	NotificationsUpdater() services.NotificationsUpdater
	PreferencesFinder() *services.PreferencesFinder
	PreferenceUpdater() services.PreferenceUpdater
	MessageFinder() services.MessageFinder
	TemplateServiceObjects() (services.TemplateCreator, services.TemplateFinder, services.TemplateUpdater, services.TemplateDeleter, services.TemplateLister, services.TemplateAssigner, services.TemplateAssociationLister)
	Logging() middleware.RequestLogging
	ErrorWriter() handlers.ErrorWriter
	Authenticator(...string) middleware.Authenticator
	CORS() middleware.CORS
	SQLDatabase() *sql.DB
}

type Router struct {
	stacks map[string]stack.Stack
	router *mux.Router
}

func NewRouter(mother MotherInterface, config Config) Router {
	registrar := mother.Registrar()
	notificationsFinder := mother.NotificationsFinder()
	emailStrategy := mother.EmailStrategy()
	userStrategy := mother.UserStrategy()
	spaceStrategy := mother.SpaceStrategy()
	organizationStrategy := mother.OrganizationStrategy()
	everyoneStrategy := mother.EveryoneStrategy()
	uaaScopeStrategy := mother.UAAScopeStrategy()
	notify := handlers.NewNotify(mother.NotificationsFinder(), registrar)
	preferencesFinder := mother.PreferencesFinder()
	preferenceUpdater := mother.PreferenceUpdater()
	templateCreator, templateFinder, templateUpdater, templateDeleter, templateLister, templateAssigner, templateAssociationLister := mother.TemplateServiceObjects()
	notificationsUpdater := mother.NotificationsUpdater()
	messageFinder := mother.MessageFinder()
	logging := mother.Logging()
	errorWriter := mother.ErrorWriter()
	notificationsWriteAuthenticator := mother.Authenticator("notifications.write")
	notificationsManageAuthenticator := mother.Authenticator("notifications.manage")
	notificationPreferencesReadAuthenticator := mother.Authenticator("notification_preferences.read")
	notificationPreferencesWriteAuthenticator := mother.Authenticator("notification_preferences.write")
	notificationPreferencesAdminAuthenticator := mother.Authenticator("notification_preferences.admin")
	emailsWriteAuthenticator := mother.Authenticator("emails.write")
	notificationsTemplateWriteAuthenticator := mother.Authenticator("notification_templates.write")
	notificationsTemplateReadAuthenticator := mother.Authenticator("notification_templates.read")
	notificationsWriteOrEmailsWriteAuthenticator := mother.Authenticator("notifications.write", "emails.write")
	databaseAllocator := middleware.NewDatabaseAllocator(mother.SQLDatabase(), config.DBLoggingEnabled)
	cors := mother.CORS()
	router := mux.NewRouter()
	requestCounter := middleware.NewRequestCounter(router, metrics.DefaultLogger)

	return Router{
		router: router,
		stacks: map[string]stack.Stack{
			"GET /info":                                                         stack.NewStack(handlers.NewGetInfo()).Use(logging, requestCounter),
			"POST /users/{user_id}":                                             stack.NewStack(handlers.NewNotifyUser(notify, errorWriter, userStrategy)).Use(logging, requestCounter, notificationsWriteAuthenticator, databaseAllocator),
			"POST /spaces/{space_id}":                                           stack.NewStack(handlers.NewNotifySpace(notify, errorWriter, spaceStrategy)).Use(logging, requestCounter, notificationsWriteAuthenticator, databaseAllocator),
			"POST /organizations/{org_id}":                                      stack.NewStack(handlers.NewNotifyOrganization(notify, errorWriter, organizationStrategy)).Use(logging, requestCounter, notificationsWriteAuthenticator, databaseAllocator),
			"POST /everyone":                                                    stack.NewStack(handlers.NewNotifyEveryone(notify, errorWriter, everyoneStrategy)).Use(logging, requestCounter, notificationsWriteAuthenticator, databaseAllocator),
			"POST /uaa_scopes/{scope}":                                          stack.NewStack(handlers.NewNotifyUAAScope(notify, errorWriter, uaaScopeStrategy)).Use(logging, requestCounter, notificationsWriteAuthenticator, databaseAllocator),
			"POST /emails":                                                      stack.NewStack(handlers.NewNotifyEmail(notify, errorWriter, emailStrategy)).Use(logging, requestCounter, emailsWriteAuthenticator, databaseAllocator),
			"PUT /registration":                                                 stack.NewStack(handlers.NewRegisterNotifications(registrar, errorWriter)).Use(logging, requestCounter, notificationsWriteAuthenticator, databaseAllocator),
			"PUT /notifications":                                                stack.NewStack(handlers.NewRegisterClientWithNotifications(registrar, errorWriter)).Use(logging, requestCounter, notificationsWriteAuthenticator, databaseAllocator),
			"GET /notifications":                                                stack.NewStack(handlers.NewGetAllNotifications(notificationsFinder, errorWriter)).Use(logging, requestCounter, notificationsManageAuthenticator, databaseAllocator),
			"OPTIONS /user_preferences":                                         stack.NewStack(handlers.NewOptionsPreferences()).Use(logging, requestCounter, cors),
			"OPTIONS /user_preferences/{user_id}":                               stack.NewStack(handlers.NewOptionsPreferences()).Use(logging, requestCounter, cors),
			"GET /user_preferences":                                             stack.NewStack(handlers.NewGetPreferences(preferencesFinder, errorWriter)).Use(logging, requestCounter, cors, notificationPreferencesReadAuthenticator, databaseAllocator),
			"GET /user_preferences/{user_id}":                                   stack.NewStack(handlers.NewGetPreferencesForUser(preferencesFinder, errorWriter)).Use(logging, requestCounter, cors, notificationPreferencesAdminAuthenticator, databaseAllocator),
			"PATCH /user_preferences":                                           stack.NewStack(handlers.NewUpdatePreferences(preferenceUpdater, errorWriter)).Use(logging, requestCounter, cors, notificationPreferencesWriteAuthenticator, databaseAllocator),
			"PATCH /user_preferences/{user_id}":                                 stack.NewStack(handlers.NewUpdateSpecificUserPreferences(preferenceUpdater, errorWriter)).Use(logging, requestCounter, cors, notificationPreferencesAdminAuthenticator, databaseAllocator),
			"GET /default_template":                                             stack.NewStack(handlers.NewGetDefaultTemplate(templateFinder, errorWriter)).Use(logging, requestCounter, notificationsTemplateReadAuthenticator, databaseAllocator),
			"PUT /default_template":                                             stack.NewStack(handlers.NewUpdateDefaultTemplate(templateUpdater, errorWriter)).Use(logging, requestCounter, notificationsTemplateWriteAuthenticator, databaseAllocator),
			"POST /templates":                                                   stack.NewStack(handlers.NewCreateTemplate(templateCreator, errorWriter)).Use(logging, requestCounter, notificationsTemplateWriteAuthenticator, databaseAllocator),
			"GET /templates/{template_id}":                                      stack.NewStack(handlers.NewGetTemplates(templateFinder, errorWriter)).Use(logging, requestCounter, notificationsTemplateReadAuthenticator, databaseAllocator),
			"PUT /templates/{template_id}":                                      stack.NewStack(handlers.NewUpdateTemplates(templateUpdater, errorWriter)).Use(logging, requestCounter, notificationsTemplateWriteAuthenticator, databaseAllocator),
			"DELETE /templates/{template_id}":                                   stack.NewStack(handlers.NewDeleteTemplates(templateDeleter, errorWriter)).Use(logging, requestCounter, notificationsTemplateWriteAuthenticator, databaseAllocator),
			"GET /templates/{template_id}/associations":                         stack.NewStack(handlers.NewListTemplateAssociations(templateAssociationLister, errorWriter)).Use(logging, requestCounter, notificationsManageAuthenticator, databaseAllocator),
			"GET /templates":                                                    stack.NewStack(handlers.NewListTemplates(templateLister, errorWriter)).Use(logging, requestCounter, notificationsTemplateReadAuthenticator, databaseAllocator),
			"PUT /clients/{client_id}/template":                                 stack.NewStack(handlers.NewAssignClientTemplate(templateAssigner, errorWriter)).Use(logging, requestCounter, notificationsManageAuthenticator, databaseAllocator),
			"PUT /clients/{client_id}/notifications/{notification_id}":          stack.NewStack(handlers.NewUpdateNotifications(notificationsUpdater, errorWriter)).Use(logging, requestCounter, notificationsManageAuthenticator, databaseAllocator),
			"PUT /clients/{client_id}/notifications/{notification_id}/template": stack.NewStack(handlers.NewAssignNotificationTemplate(templateAssigner, errorWriter)).Use(logging, requestCounter, notificationsManageAuthenticator, databaseAllocator),
			"GET /messages/{message_id}":                                        stack.NewStack(handlers.NewGetMessages(messageFinder, errorWriter)).Use(logging, requestCounter, notificationsWriteOrEmailsWriteAuthenticator, databaseAllocator),
		},
	}
}

func (router Router) Routes() *mux.Router {
	for methodPath, stack := range router.stacks {
		var name = methodPath
		parts := strings.SplitN(methodPath, " ", 2)
		router.router.Handle(parts[1], stack).Methods(parts[0]).Name(name)
	}
	return router.router
}
<commit_msg>Breaks up stack in router into stacks based on paths<commit_after>package web

import (
	"database/sql"
	"strings"

	"github.com/cloudfoundry-incubator/notifications/metrics"
	"github.com/cloudfoundry-incubator/notifications/postal/strategies"
	"github.com/cloudfoundry-incubator/notifications/web/handlers"
	"github.com/cloudfoundry-incubator/notifications/web/middleware"
	"github.com/cloudfoundry-incubator/notifications/web/services"
	"github.com/gorilla/mux"
	"github.com/ryanmoran/stack"
)

type MotherInterface interface {
	Registrar() services.Registrar
	EmailStrategy() strategies.EmailStrategy
	UserStrategy() strategies.UserStrategy
	SpaceStrategy() strategies.SpaceStrategy
	OrganizationStrategy() strategies.OrganizationStrategy
	EveryoneStrategy() strategies.EveryoneStrategy
	UAAScopeStrategy() strategies.UAAScopeStrategy
	NotificationsFinder() services.NotificationsFinder
	NotificationsUpdater() services.NotificationsUpdater
	PreferencesFinder() *services.PreferencesFinder
	PreferenceUpdater() services.PreferenceUpdater
	MessageFinder() services.MessageFinder
	TemplateServiceObjects() (services.TemplateCreator, services.TemplateFinder, services.TemplateUpdater, services.TemplateDeleter, services.TemplateLister, services.TemplateAssigner, services.TemplateAssociationLister)
	Logging() middleware.RequestLogging
	ErrorWriter() handlers.ErrorWriter
	Authenticator(...string) middleware.Authenticator
	CORS() middleware.CORS
	SQLDatabase() *sql.DB
}

type Router struct {
	stacks map[string]stack.Stack
	router *mux.Router
}

func NewRouter(mother MotherInterface, config Config) Router {
	registrar := mother.Registrar()
	notificationsFinder := mother.NotificationsFinder()
	emailStrategy := mother.EmailStrategy()
	userStrategy := mother.UserStrategy()
	spaceStrategy := mother.SpaceStrategy()
	organizationStrategy := mother.OrganizationStrategy()
	everyoneStrategy := mother.EveryoneStrategy()
	uaaScopeStrategy := mother.UAAScopeStrategy()
	notify := handlers.NewNotify(mother.NotificationsFinder(), registrar)
	preferencesFinder := mother.PreferencesFinder()
	preferenceUpdater := mother.PreferenceUpdater()
	templateCreator, templateFinder, templateUpdater, templateDeleter, templateLister, templateAssigner, templateAssociationLister := mother.TemplateServiceObjects()
	notificationsUpdater := mother.NotificationsUpdater()
	messageFinder := mother.MessageFinder()
	logging := mother.Logging()
	errorWriter := mother.ErrorWriter()
	notificationsWriteAuthenticator := mother.Authenticator("notifications.write")
	notificationsManageAuthenticator := mother.Authenticator("notifications.manage")
	notificationPreferencesReadAuthenticator := mother.Authenticator("notification_preferences.read")
	notificationPreferencesWriteAuthenticator := mother.Authenticator("notification_preferences.write")
	notificationPreferencesAdminAuthenticator := mother.Authenticator("notification_preferences.admin")
	emailsWriteAuthenticator := mother.Authenticator("emails.write")
	notificationsTemplateWriteAuthenticator := mother.Authenticator("notification_templates.write")
	notificationsTemplateReadAuthenticator := mother.Authenticator("notification_templates.read")
	notificationsWriteOrEmailsWriteAuthenticator := mother.Authenticator("notifications.write", "emails.write")
	databaseAllocator := middleware.NewDatabaseAllocator(mother.SQLDatabase(), config.DBLoggingEnabled)
	cors := mother.CORS()
	router := mux.NewRouter()
	requestCounter := middleware.NewRequestCounter(router, metrics.DefaultLogger)

	infoStack := map[string]stack.Stack{
		"GET /info": stack.NewStack(handlers.NewGetInfo()).Use(logging, requestCounter),
	}

	notificationsStack := map[string]stack.Stack{
		"POST /users/{user_id}":        stack.NewStack(handlers.NewNotifyUser(notify, errorWriter, userStrategy)).Use(logging, requestCounter, notificationsWriteAuthenticator, databaseAllocator),
		"POST /spaces/{space_id}":      stack.NewStack(handlers.NewNotifySpace(notify, errorWriter, spaceStrategy)).Use(logging, requestCounter, notificationsWriteAuthenticator, databaseAllocator),
		"POST /organizations/{org_id}": stack.NewStack(handlers.NewNotifyOrganization(notify, errorWriter, organizationStrategy)).Use(logging, requestCounter, notificationsWriteAuthenticator, databaseAllocator),
		"POST /everyone":               stack.NewStack(handlers.NewNotifyEveryone(notify, errorWriter, everyoneStrategy)).Use(logging, requestCounter, notificationsWriteAuthenticator, databaseAllocator),
		"POST /uaa_scopes/{scope}":     stack.NewStack(handlers.NewNotifyUAAScope(notify, errorWriter, uaaScopeStrategy)).Use(logging, requestCounter, notificationsWriteAuthenticator, databaseAllocator),
		"POST /emails":                 stack.NewStack(handlers.NewNotifyEmail(notify, errorWriter, emailStrategy)).Use(logging, requestCounter, emailsWriteAuthenticator, databaseAllocator),
	}

	registrationStack := map[string]stack.Stack{
		"PUT /registration":  stack.NewStack(handlers.NewRegisterNotifications(registrar, errorWriter)).Use(logging, requestCounter, notificationsWriteAuthenticator, databaseAllocator),
		"PUT /notifications": stack.NewStack(handlers.NewRegisterClientWithNotifications(registrar, errorWriter)).Use(logging, requestCounter, notificationsWriteAuthenticator, databaseAllocator),
		"GET /notifications": stack.NewStack(handlers.NewGetAllNotifications(notificationsFinder, errorWriter)).Use(logging, requestCounter, notificationsManageAuthenticator, databaseAllocator),
	}

	userPreferencesStack := map[string]stack.Stack{
		"OPTIONS /user_preferences":           stack.NewStack(handlers.NewOptionsPreferences()).Use(logging, requestCounter, cors),
		"OPTIONS /user_preferences/{user_id}": stack.NewStack(handlers.NewOptionsPreferences()).Use(logging, requestCounter, cors),
		"GET /user_preferences":               stack.NewStack(handlers.NewGetPreferences(preferencesFinder, errorWriter)).Use(logging, requestCounter, cors, notificationPreferencesReadAuthenticator, databaseAllocator),
		"GET /user_preferences/{user_id}":     stack.NewStack(handlers.NewGetPreferencesForUser(preferencesFinder, errorWriter)).Use(logging, requestCounter, cors, notificationPreferencesAdminAuthenticator, databaseAllocator),
		"PATCH /user_preferences":             stack.NewStack(handlers.NewUpdatePreferences(preferenceUpdater, errorWriter)).Use(logging, requestCounter, cors, notificationPreferencesWriteAuthenticator, databaseAllocator),
		"PATCH /user_preferences/{user_id}":   stack.NewStack(handlers.NewUpdateSpecificUserPreferences(preferenceUpdater, errorWriter)).Use(logging, requestCounter, cors, notificationPreferencesAdminAuthenticator, databaseAllocator),
	}

	templatesStack := map[string]stack.Stack{
		"GET /default_template":                     stack.NewStack(handlers.NewGetDefaultTemplate(templateFinder, errorWriter)).Use(logging, requestCounter, notificationsTemplateReadAuthenticator, databaseAllocator),
		"PUT /default_template":                     stack.NewStack(handlers.NewUpdateDefaultTemplate(templateUpdater, errorWriter)).Use(logging, requestCounter, notificationsTemplateWriteAuthenticator, databaseAllocator),
		"POST /templates":                           stack.NewStack(handlers.NewCreateTemplate(templateCreator, errorWriter)).Use(logging, requestCounter, notificationsTemplateWriteAuthenticator, databaseAllocator),
		"GET /templates/{template_id}":              stack.NewStack(handlers.NewGetTemplates(templateFinder, errorWriter)).Use(logging, requestCounter, notificationsTemplateReadAuthenticator, databaseAllocator),
		"PUT /templates/{template_id}":              stack.NewStack(handlers.NewUpdateTemplates(templateUpdater, errorWriter)).Use(logging, requestCounter, notificationsTemplateWriteAuthenticator, databaseAllocator),
		"DELETE /templates/{template_id}":           stack.NewStack(handlers.NewDeleteTemplates(templateDeleter, errorWriter)).Use(logging, requestCounter, notificationsTemplateWriteAuthenticator, databaseAllocator),
		"GET /templates/{template_id}/associations": stack.NewStack(handlers.NewListTemplateAssociations(templateAssociationLister, errorWriter)).Use(logging, requestCounter, notificationsManageAuthenticator, databaseAllocator),
		"GET /templates":                            stack.NewStack(handlers.NewListTemplates(templateLister, errorWriter)).Use(logging, requestCounter, notificationsTemplateReadAuthenticator, databaseAllocator),
	}

	clientsStack := map[string]stack.Stack{
		"PUT /clients/{client_id}/template":                                 stack.NewStack(handlers.NewAssignClientTemplate(templateAssigner, errorWriter)).Use(logging, requestCounter, notificationsManageAuthenticator, databaseAllocator),
		"PUT /clients/{client_id}/notifications/{notification_id}":          stack.NewStack(handlers.NewUpdateNotifications(notificationsUpdater, errorWriter)).Use(logging, requestCounter, notificationsManageAuthenticator, databaseAllocator),
		"PUT /clients/{client_id}/notifications/{notification_id}/template": stack.NewStack(handlers.NewAssignNotificationTemplate(templateAssigner, errorWriter)).Use(logging, requestCounter, notificationsManageAuthenticator, databaseAllocator),
	}

	messagesStack := map[string]stack.Stack{
		"GET /messages/{message_id}": stack.NewStack(handlers.NewGetMessages(messageFinder, errorWriter)).Use(logging, requestCounter, notificationsWriteOrEmailsWriteAuthenticator, databaseAllocator),
	}

	stacks := make(map[string]stack.Stack)
	for _, s := range []map[string]stack.Stack{infoStack, notificationsStack, registrationStack, userPreferencesStack, templatesStack, clientsStack, messagesStack} {
		for route, handler := range s {
			stacks[route] = handler
		}
	}

	return Router{
		router: router,
		stacks: stacks,
	}
}

func (router Router) Routes() *mux.Router {
	for methodPath, stack := range router.stacks {
		var name = methodPath
		parts := strings.SplitN(methodPath, " ", 2)
		router.router.Handle(parts[1], stack).Methods(parts[0]).Name(name)
	}
	return router.router
}
<|endoftext|>


<commit_before>package web

import (
	"dreitafel"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"

	"fmt"
	"net/http"
	"sync"
)

func index(w http.ResponseWriter, r *http.Request) {
	txt := `

Welcome to Dreitafel!

Try this: <a href="/fmc/q?format=png&diagram=%5BActor%5D%20-%3E%20%28Storage%29

`
	fmt.Fprintf(w, txt)
}

func compileFmcBlockDiagramFromQueryString(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	// TODO input validation & error handling :]
	fmcSrcLines := make(chan *string) // lines are independent; statements don't span lines yet

	dotSrcLines := make(chan *string, 500)
	errors := make(chan error, 500) // errors are independent

	var wg sync.WaitGroup

	go dreitafel.CompileFmcBlockDiagramToDot(fmcSrcLines, dotSrcLines, errors)

	if r.Form["format"][0] == "png" {
		w.Header().Add("Content-Type", "image/png")
		writeDotGeneratedImage("png", w, dotSrcLines, errors, &wg)
	} else if r.Form["format"][0] == "svg" {
		w.Header().Add("Content-Type", "image/svg+xml")
		writeDotGeneratedImage("svg", w, dotSrcLines, errors, &wg)
	} else {
		writeDotSrcLines(w, dotSrcLines, errors, &wg)
	}

	fmcSrcLines <- &r.Form["diagram"][0]
	close(fmcSrcLines)

	wg.Wait()
}

func writeDotSrcLines(w io.Writer, dotSrcLines chan *string, errors chan error, wg *sync.WaitGroup) {
	wg.Add(1)
	go func() {
		for line := range dotSrcLines {
			fmt.Printf("%v\n", *line)
			fmt.Fprintf(w, "%v\n", *line)
		}

		wg.Done()
	}()

	wg.Add(1)
	go func() {
		for err := range errors {
			fmt.Fprintf(os.Stderr, "%v\n", err)
		}

		wg.Done()
	}()
}

func writeDotGeneratedImage(format string, w io.Writer, dotSrcLines chan *string, errors chan error, wg *sync.WaitGroup) {
	cmd := exec.Command("dot", "-T"+format)
	in, err := cmd.StdinPipe()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		return
	}
	out, err := cmd.StdoutPipe()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		return
	}
	errp, err := cmd.StderrPipe()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		return
	}

	wg.Add(1)
	go func() {
		for line := range dotSrcLines {
			fmt.Printf("%v\n", *line)
			fmt.Fprintf(in, "%v\n", *line)
		}

		in.Close()
		fmt.Println("Done with writing src to stdin of dot.")
		wg.Done()
	}()

	wg.Add(1)
	go func() {

		fmt.Printf("Starting to read.\n")
		b, err := ioutil.ReadAll(out)
		if err != nil {
			log.Fatal(err)
		}
		w.Write(b)
		fmt.Printf("Done with writing %v PNG bytes as response.\n", len(b))
		wg.Done()
	}()

	wg.Add(1)
	go func() {

		fmt.Printf("Starting to read stderr.\n")
		b, err := ioutil.ReadAll(errp)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Println(b)
		fmt.Printf("Done with writing %v stderr as response.\n", len(b))
		wg.Done()
	}()

	wg.Add(1)
	go func() {
		for err := range errors {
			fmt.Fprintf(os.Stderr, "%v\n", err)
		}

		wg.Done()
	}()

	err = cmd.Start()
	if err != nil {
		log.Fatal(err)
	}
}

func ListenAndServe() {
	http.HandleFunc("/fmc/", compileFmcBlockDiagramFromQueryString)
	http.HandleFunc("/", index)
	http.ListenAndServe(":8080", nil)
}
<commit_msg>Added parameter validation and updated index text<commit_after>package web

import (
	"dreitafel"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"

	"fmt"
	"net/http"
	"sync"
)

var responseFormats = map[string]string{
	"dot": "text/dot",
	"png": "image/png",
	"svg": "image/svg+xml",
}

var validFormats []string

func index(w http.ResponseWriter, r *http.Request) {
	txt := `

Welcome to Dreitafel viewer!

See this example:

[Actor] -> (Storage)

A diagram can be generated dynamically using this URL: /fmc/?format=svg&diagram=[Actor]->(Storage)

And here's the diagram:

(Storage)"/>

`
	fmt.Fprintf(w, txt)
}

func compileFmcBlockDiagramFromQueryString(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()

	if len(r.Form["format"]) == 0 {
		http.Error(w, fmt.Sprintf("Need 'format' query param. Supported formats are: %v", validFormats), http.StatusBadRequest)
		return
	}
	format := r.Form["format"][len(r.Form["format"])-1]

	contentType, ok := responseFormats[format]
	if !ok {
		http.Error(w, fmt.Sprintf("Supported values for 'format' param are: %v", validFormats), http.StatusBadRequest)
		return
	}

	if len(r.Form["diagram"]) == 0 {
		http.Error(w, "Need 'diagram' query param with Dreitafel source code", http.StatusBadRequest)
		return
	}
	diagram := r.Form["diagram"][len(r.Form["diagram"])-1]

	fmcSrcLines := make(chan *string) // lines are independent; statements don't span lines yet

	dotSrcLines := make(chan *string, 500)
	errors := make(chan error, 500) // errors are independent

	var wg sync.WaitGroup

	go dreitafel.CompileFmcBlockDiagramToDot(fmcSrcLines, dotSrcLines, errors)

	w.Header().Add("Content-Type", contentType)
	if format == "dot" {
		writeDotSrcLines(w, dotSrcLines, errors, &wg)
	} else {
		writeDotGeneratedImage(format, w, dotSrcLines, errors, &wg)
	}

	fmcSrcLines <- &diagram
	close(fmcSrcLines)

	wg.Wait()
}

func writeDotSrcLines(w io.Writer, dotSrcLines chan *string, errors chan error, wg *sync.WaitGroup) {
	wg.Add(1)
	go func() {
		for line := range dotSrcLines {
			fmt.Printf("%v\n", *line)
			fmt.Fprintf(w, "%v\n", *line)
		}

		wg.Done()
	}()

	wg.Add(1)
	go func() {
		for err := range errors {
			fmt.Fprintf(os.Stderr, "%v\n", err)
		}

		wg.Done()
	}()
}

func writeDotGeneratedImage(format string, w io.Writer, dotSrcLines chan *string, errors chan error, wg *sync.WaitGroup) {
	cmd := exec.Command("dot", "-T"+format)
	in, err := cmd.StdinPipe()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		return
	}
	out, err := cmd.StdoutPipe()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		return
	}
	errp, err := cmd.StderrPipe()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		return
	}

	wg.Add(1)
	go func() {
		for line := range dotSrcLines {
			fmt.Printf("%v\n", *line)
			fmt.Fprintf(in, "%v\n", *line)
		}

		in.Close()
		fmt.Println("Done with writing src to stdin of dot.")
		wg.Done()
	}()

	wg.Add(1)
	go func() {

		fmt.Printf("Starting to read.\n")
		b, err := ioutil.ReadAll(out)
		if err != nil {
			log.Fatal(err)
		}
		w.Write(b)
		fmt.Printf("Done with writing %v PNG bytes as response.\n", len(b))
		wg.Done()
	}()

	wg.Add(1)
	go func() {

		fmt.Printf("Starting to read stderr.\n")
		b, err := ioutil.ReadAll(errp)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Println(b)
		fmt.Printf("Done with writing %v stderr as response.\n", len(b))
		wg.Done()
	}()

	wg.Add(1)
	go func() {
		for err := range errors {
			fmt.Fprintf(os.Stderr, "%v\n", err)
		}

		wg.Done()
	}()

	err = cmd.Start()
	if err != nil {
		log.Fatal(err)
	}
}

func ListenAndServe() {
	validFormats = make([]string, len(responseFormats))
	i := 0
	for key, _ := range responseFormats {
		validFormats[i] = key
		i += 1
	}

	http.HandleFunc("/fmc/", compileFmcBlockDiagramFromQueryString)
	http.HandleFunc("/", index)
	http.ListenAndServe(":8080", nil)
}
<|endoftext|>


<commit_before>package web

import (
	"encoding/json"
	"fmt"
	"github.com/gorilla/mux"
	"github.com/tgermain/grandRepositorySky/node"
	"github.com/tgermain/grandRepositorySky/shared"
	"net/http"

	//"time" // to set a timer
)

//Objects parts ---------------------------------------------------------

type NodeJson struct {
	Id          string
	Ip          string
	Port        string
	Successor   *shared.DistantNode
	Predecessor *shared.DistantNode
	Fingers     []*node.FingerEntry
}

type MyServer struct {
	r *mux.Router
}

var node1 *node.DHTnode

//Method parts ----------------------------------------------------------

// Hello Handler
func HelloHandler(w http.ResponseWriter, req *http.Request) {
	fmt.Println("GET /")
	fmt.Fprintf(w, "Hello World too")
}

//TODO refactor handler url (not really a get nodes)
//TODO send datas informations too
func NodesHandler(w http.ResponseWriter, req *http.Request) {
	fmt.Println("GET /noeuds")
	// res, _ := json.Marshal(node1.GetFingerTable())
	// shared.Logger.Warning("%s", res)
	// //gives the local node infos and fingertable
	// fingers := node1.GetFingerTable()
	// fingersJSON := make([]FingerJSON, len(fingers))
	// for key, value := range fingers {
	// 	if value != nil {
	// 		nodeResp := value.NodeResp
	// 		var nodeRespJSON DistantNodeJSON
	// 		if nodeResp != nil {
	// 			nodeRespJSON = DistantNodeJSON{nodeResp.Id, nodeResp.Ip, nodeResp.Port}
	// 		} else {
	// 			nodeRespJSON = DistantNodeJSON{}
	// 		}
	// 		entry := FingerJSON{value.IdKey, nodeRespJSON}
	// 		fingersJSON[key] = entry
	// 	} else {
	// 		entry := FingerJSON{}
	// 		fingersJSON[key] = entry
	// 	}
	// }
	// succ := node1.GetSuccesor()
	// succJSON := DistantNodeJSON{succ.Id, succ.Ip, succ.Port}
	// pred := node1.GetPredecessor()
	// predJSON := DistantNodeJSON{pred.Id, pred.Ip, pred.Port}

	node1Json := NodeJson{shared.LocalId, shared.LocalIp, shared.LocalPort, node1.GetSuccesor(), node1.GetPredecessor(), node1.GetFingerTable()}
	js, err := json.Marshal(node1Json)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Write(js)
}

//TODO? launch lookup request

//TODO post data

//TODO update data

//TODO remove data

//TODO:POSTPONE disconnect a node

//TODO:POSTPONE manual areYouAlive request

//wrap server handler function to activate CORS
func (s *MyServer) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	//if origin := req.Header.Get("Origin"); origin != "" {
	rw.Header().Set("Content-Type", "application/json")
	rw.Header().Set("Accept", "application/json")
	rw.Header().Set("Access-Control-Allow-Credentials", "true")
	rw.Header().Set("Access-Control-Allow-Origin", "*")
	rw.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE")
	rw.Header().Set("Access-Control-Allow-Headers",
		"Accept, Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization")
	//}
	// Stop here if its Preflighted OPTIONS request
	if req.Method == "OPTIONS" {
		return
	}
	//the real library handler
	s.r.ServeHTTP(rw, req)
}

//create a functionnal server
func MakeServer(ip string, port string, nod *node.DHTnode) {
	receive := ip + ":" + port
	node1 = nod
	fmt.Printf("server listen on : %s\n", receive)

	r := mux.NewRouter()
	// r.HandleFunc("/", HelloHandler)
	//serv staticly index.html
	fs := http.FileServer(http.Dir("web/client"))
	r.Handle("/", fs)

	r.HandleFunc("/nodes", NodesHandler)
	http.Handle("/", &MyServer{r})

	http.ListenAndServe(receive, nil)
}
<commit_msg>add charset header<commit_after>package web

import (
	"encoding/json"
	"fmt"
	"github.com/gorilla/mux"
	"github.com/tgermain/grandRepositorySky/node"
	"github.com/tgermain/grandRepositorySky/shared"
	"net/http"

	//"time" // to set a timer
)

//Objects parts ---------------------------------------------------------

type NodeJson struct {
	Id          string
	Ip          string
	Port        string
	Successor   *shared.DistantNode
	Predecessor *shared.DistantNode
	Fingers     []*node.FingerEntry
}

type MyServer struct {
	r *mux.Router
}

var node1 *node.DHTnode

//Method parts ----------------------------------------------------------

// Hello Handler
func HelloHandler(w http.ResponseWriter, req *http.Request) {
	fmt.Println("GET /")
	fmt.Fprintf(w, "Hello World too")
}

//TODO refactor handler url (not really a get nodes)
//TODO send datas informations too
func NodesHandler(w http.ResponseWriter, req *http.Request) {
	fmt.Println("GET /noeuds")
	// res, _ := json.Marshal(node1.GetFingerTable())
	// shared.Logger.Warning("%s", res)
	// //gives the local node infos and fingertable
	// fingers := node1.GetFingerTable()
	// fingersJSON := make([]FingerJSON, len(fingers))
	// for key, value := range fingers {
	// 	if value != nil {
	// 		nodeResp := value.NodeResp
	// 		var nodeRespJSON DistantNodeJSON
	// 		if nodeResp != nil {
	// 			nodeRespJSON = DistantNodeJSON{nodeResp.Id, nodeResp.Ip, nodeResp.Port}
	// 		} else {
	// 			nodeRespJSON = DistantNodeJSON{}
	// 		}
	// 		entry := FingerJSON{value.IdKey, nodeRespJSON}
	// 		fingersJSON[key] = entry
	// 	} else {
	// 		entry := FingerJSON{}
	// 		fingersJSON[key] = entry
	// 	}
	// }
	// succ := node1.GetSuccesor()
	// succJSON := DistantNodeJSON{succ.Id, succ.Ip, succ.Port}
	// pred := node1.GetPredecessor()
	// predJSON := DistantNodeJSON{pred.Id, pred.Ip, pred.Port}

	node1Json := NodeJson{shared.LocalId, shared.LocalIp, shared.LocalPort, node1.GetSuccesor(), node1.GetPredecessor(), node1.GetFingerTable()}
	js, err := json.Marshal(node1Json)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Write(js)
}

//TODO? launch lookup request

//TODO post data

//TODO update data

//TODO remove data

//TODO:POSTPONE disconnect a node

//TODO:POSTPONE manual areYouAlive request

//wrap server handler function to activate CORS
func (s *MyServer) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	//if origin := req.Header.Get("Origin"); origin != "" {
	rw.Header().Set("Content-Type", "application/json")
	rw.Header().Set("Accept", "application/json")
	rw.Header().Set("Accept-Charset", "utf-8")
	rw.Header().Set("Access-Control-Allow-Credentials", "true")
	rw.Header().Set("Access-Control-Allow-Origin", "*")
	rw.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE")
	rw.Header().Set("Access-Control-Allow-Headers",
		"Accept, Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization")
	//}
	// Stop here if its Preflighted OPTIONS request
	if req.Method == "OPTIONS" {
		return
	}
	//the real library handler
	s.r.ServeHTTP(rw, req)
}

//create a functionnal server
func MakeServer(ip string, port string, nod *node.DHTnode) {
	receive := ip + ":" + port
	node1 = nod
	fmt.Printf("server listen on : %s\n", receive)

	r := mux.NewRouter()
	// r.HandleFunc("/", HelloHandler)
	//serv staticly index.html
	fs := http.FileServer(http.Dir("web/client"))
	r.Handle("/", fs)

	r.HandleFunc("/nodes", NodesHandler)
	http.Handle("/", &MyServer{r})

	http.ListenAndServe(receive, nil)
}
<|endoftext|>


<commit_before>package web

import (
	"encoding/json"
	"fmt"
	"github.com/gorilla/mux"
	"github.com/tgermain/grandRepositorySky/node"
	"github.com/tgermain/grandRepositorySky/shared"
	"net/http"

	//"time" // to set a timer
)

//Objects parts ---------------------------------------------------------

type NodeJson struct {
	Id          string
	Ip          string
	Port        string
	Successor   *shared.DistantNode
	Predecessor *shared.DistantNode
	Fingers     []*node.FingerEntry
}

type MyServer struct {
	r *mux.Router
}

var node1 *node.DHTnode

//Method parts ----------------------------------------------------------

// Hello Handler
func HelloHandler(w http.ResponseWriter, req *http.Request) {
	fmt.Println("GET /")
	fmt.Fprintf(w, "Hello World too")
}

//TODO refactor handler url (not really a get nodes)
//TODO send datas informations too
func NodesHandler(w http.ResponseWriter, req *http.Request) {
	fmt.Println("GET /noeuds")
	// res, _ := json.Marshal(node1.GetFingerTable())
	// shared.Logger.Warning("%s", res)
	// //gives the local node infos and fingertable
	// fingers := node1.GetFingerTable()
	// fingersJSON := make([]FingerJSON, len(fingers))
	// for key, value := range fingers {
	// 	if value != nil {
	// 		nodeResp := value.NodeResp
	// 		var nodeRespJSON DistantNodeJSON
	// 		if nodeResp != nil {
	// 			nodeRespJSON = DistantNodeJSON{nodeResp.Id, nodeResp.Ip, nodeResp.Port}
	// 		} else {
	// 			nodeRespJSON = DistantNodeJSON{}
	// 		}
	// 		entry := FingerJSON{value.IdKey, nodeRespJSON}
	// 		fingersJSON[key] = entry
	// 	} else {
	// 		entry := FingerJSON{}
	// 		fingersJSON[key] = entry
	// 	}
	// }
	// succ := node1.GetSuccesor()
	// succJSON := DistantNodeJSON{succ.Id, succ.Ip, succ.Port}
	// pred := node1.GetPredecessor()
	// predJSON := DistantNodeJSON{pred.Id, pred.Ip, pred.Port}

	node1Json := NodeJson{shared.LocalId, shared.LocalIp, shared.LocalPort, node1.GetSuccesor(), node1.GetPredecessor(), node1.GetFingerTable()}
	js, err := json.Marshal(node1Json)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Write(js)
}

//TODO? launch lookup request

//TODO post data

//TODO update data

//TODO remove data

//TODO:POSTPONE disconnect a node

//TODO:POSTPONE manual areYouAlive request

//wrap server handler function to activate CORS
func (s *MyServer) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	//if origin := req.Header.Get("Origin"); origin != "" {
	rw.Header().Set("Content-Type", "application/json")
	rw.Header().Set("Accept", "application/json")
	rw.Header().Set("Access-Control-Allow-Credentials", "true")
	rw.Header().Set("Access-Control-Allow-Origin", "*")
	rw.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE")
	rw.Header().Set("Access-Control-Allow-Headers",
		"Accept, Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization")
	//}
	// Stop here if its Preflighted OPTIONS request
	if req.Method == "OPTIONS" {
		return
	}
	//the real library handler
	s.r.ServeHTTP(rw, req)
}

//create a functionnal server
func MakeServer(ip string, port string, nod *node.DHTnode) {
	receive := ip + ":" + port
	node1 = nod
	fmt.Printf("server listen on : %s\n", receive)

	r := mux.NewRouter()
	r.HandleFunc("/", HelloHandler)
	r.HandleFunc("/nodes", NodesHandler)
	http.Handle("/", &MyServer{r})

	http.ListenAndServe(receive, nil)
}
<commit_msg>serving staticaly the page<commit_after>package web

import (
	"encoding/json"
	"fmt"
	"github.com/gorilla/mux"
	"github.com/tgermain/grandRepositorySky/node"
	"github.com/tgermain/grandRepositorySky/shared"
	"net/http"

	//"time" // to set a timer
)

//Objects parts ---------------------------------------------------------

type NodeJson struct {
	Id          string
	Ip          string
	Port        string
	Successor   *shared.DistantNode
	Predecessor *shared.DistantNode
	Fingers     []*node.FingerEntry
}

type MyServer struct {
	r *mux.Router
}

var node1 *node.DHTnode

//Method parts ----------------------------------------------------------

// Hello Handler
func HelloHandler(w http.ResponseWriter, req *http.Request) {
	fmt.Println("GET /")
	fmt.Fprintf(w, "Hello World too")
}

//TODO refactor handler url (not really a get nodes)
//TODO send datas informations too
func NodesHandler(w http.ResponseWriter, req *http.Request) {
	fmt.Println("GET /noeuds")
	// res, _ := json.Marshal(node1.GetFingerTable())
	// shared.Logger.Warning("%s", res)
	// //gives the local node infos and fingertable
	// fingers := node1.GetFingerTable()
	// fingersJSON := make([]FingerJSON, len(fingers))
	// for key, value := range fingers {
	// 	if value != nil {
	// 		nodeResp := value.NodeResp
	// 		var nodeRespJSON DistantNodeJSON
	// 		if nodeResp != nil {
	// 			nodeRespJSON = DistantNodeJSON{nodeResp.Id, nodeResp.Ip, nodeResp.Port}
	// 		} else {
	// 			nodeRespJSON = DistantNodeJSON{}
	// 		}
	// 		entry := FingerJSON{value.IdKey, nodeRespJSON}
	// 		fingersJSON[key] = entry
	// 	} else {
	// 		entry := FingerJSON{}
	// 		fingersJSON[key] = entry
	// 	}
	// }
	// succ := node1.GetSuccesor()
	// succJSON := DistantNodeJSON{succ.Id, succ.Ip, succ.Port}
	// pred := node1.GetPredecessor()
	// predJSON := DistantNodeJSON{pred.Id, pred.Ip, pred.Port}

	node1Json := NodeJson{shared.LocalId, shared.LocalIp, shared.LocalPort, node1.GetSuccesor(), node1.GetPredecessor(), node1.GetFingerTable()}
	js, err := json.Marshal(node1Json)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Write(js)
}

//TODO? launch lookup request

//TODO post data

//TODO update data

//TODO remove data

//TODO:POSTPONE disconnect a node

//TODO:POSTPONE manual areYouAlive request

//wrap server handler function to activate CORS
func (s *MyServer) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	//if origin := req.Header.Get("Origin"); origin != "" {
	rw.Header().Set("Content-Type", "application/json")
	rw.Header().Set("Accept", "application/json")
	rw.Header().Set("Access-Control-Allow-Credentials", "true")
	rw.Header().Set("Access-Control-Allow-Origin", "*")
	rw.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE")
	rw.Header().Set("Access-Control-Allow-Headers",
		"Accept, Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization")
	//}
	// Stop here if its Preflighted OPTIONS request
	if req.Method == "OPTIONS" {
		return
	}
	//the real library handler
	s.r.ServeHTTP(rw, req)
}

//create a functionnal server
func MakeServer(ip string, port string, nod *node.DHTnode) {
	receive := ip + ":" + port
	node1 = nod
	fmt.Printf("server listen on : %s\n", receive)

	r := mux.NewRouter()
	// r.HandleFunc("/", HelloHandler)
	//serv staticly index.html
	fs := http.FileServer(http.Dir("web/client"))
	r.Handle("/", fs)

	r.HandleFunc("/nodes", NodesHandler)
	http.Handle("/", &MyServer{r})

	http.ListenAndServe(receive, nil)
}
<|endoftext|>


<commit_before>// Package carve implements the seam carving algorithm for dyanmic image resizing.
package carve

import (
	"errors"
	"fmt"
	"image"
	"math"

	"github.com/disintegration/gift"
	"github.com/disintegration/imaging"
)

// ReduceHeight uses seam carving to reduce height of given image by n pixels.
func ReduceHeight(im image.Image, n int) (image.Image, error) {
	height := im.Bounds().Max.Y - im.Bounds().Min.Y
	if height < n {
		return im, errors.New(
			fmt.Sprintf("Cannot resize image of height %d by %d pixels", height, n))
	}

	for x := 0; x < n; x++ {
		energy := GenerateEnergyMap(im)
		seam := GenerateSeam(energy)
		im = RemoveSeam(im, seam)
	}
	return im, nil
}

// ReduceHeight uses seam carving to reduce height of given image by n pixels.
func ReduceWidth(im image.Image, n int) (image.Image, error) {
	width := im.Bounds().Max.Y - im.Bounds().Min.Y
	if width < n {
		return im, errors.New(
			fmt.Sprintf("Cannot resize image of width %d by %d pixels", width, n))
	}

	i := imaging.Rotate90(im)
	out, err := ReduceHeight(i, n)
	return imaging.Rotate270(out), err
}

// GenerateEnergyMap applies grayscale and sobel filters to the
// input image to create an energy map.
func GenerateEnergyMap(im image.Image) image.Image {
	g := gift.New(gift.Grayscale(), gift.Sobel())
	res := image.NewRGBA(im.Bounds())
	g.Draw(res, im)
	return res
}

// GenerateSeam returns the optimal horizontal seam for removal.
func GenerateSeam(im image.Image) Seam {
	mat := GenerateCostMatrix(im)
	return FindLowestCostSeam(mat)
}

// RemoveSeam creates a copy of the provided image, with the pixels at
// the points in the provided seam removed.
func RemoveSeam(im image.Image, seam Seam) image.Image {
	b := im.Bounds()
	out := image.NewRGBA(image.Rect(0, 0, b.Dx(), b.Dy()-1))
	min, max := b.Min, b.Max

	for _, point := range seam {
		x := point.X

		for y := min.Y; y < max.Y; y++ {
			if y == point.Y {
				continue
			}

			if y > point.Y {
				out.Set(x, y-1, im.At(x, y))
			} else {
				out.Set(x, y, im.At(x, y))
			}
		}
	}

	return out
}

// Seam defines a sequence of pixels through an image to be removed.
type Seam []Point

// Point defines an X Y point in an image.
type Point struct {
	X, Y int
}

// GenerateCostMatrix creates a matrix indicating the cumulative energy of the
// lowest cost seam from the left of the image to each pixel.
//
// mat[x][y] is the cumulative energy of the seam that runs from the left of
// the image to the pixel at column x, row y.
func GenerateCostMatrix(im image.Image) [][]float64 {
	min, max := im.Bounds().Min, im.Bounds().Max
	height, width := max.Y-min.Y, max.X-min.X

	mat := make([][]float64, width)
	for x := min.X; x < max.X; x++ {
		mat[x-min.X] = make([]float64, height)
	}

	for y := min.Y; y < max.Y; y++ {
		e, _, _, a := im.At(0, y).RGBA()
		mat[0][y-min.Y] = float64(e) / float64(a)
	}

	updatePoint := func(x, y int) {
		e, _, _, a := im.At(x, y).RGBA()

		up, down := math.MaxFloat64, math.MaxFloat64
		left := mat[x-1][y]
		if y != min.Y {
			up = mat[x-1][y-1]
		}
		if y < max.Y-1 {
			down = mat[x-1][y+1]
		}
		val := math.Min(float64(left), math.Min(float64(up), float64(down)))
		mat[x][y] = val + (float64(e) / float64(a))
	}

	// Calculate the remaining columns iteratively
	for x := min.X + 1; x < max.X; x++ {
		for y := min.Y; y < max.Y; y++ {
			updatePoint(x, y)
		}
	}

	return mat
}

// FindLowestCostSeam uses a cumulative cost matrix to identify the seam with
// the lowest total cumulative energy.
func FindLowestCostSeam(mat [][]float64) Seam {
	width, height := len(mat), len(mat[0])
	seam := make([]Point, width)

	min, y := math.MaxFloat64, 0
	for ind, val := range mat[width-1] {
		if val < min {
			min = val
			y = ind
		}
	}

	seam[width-1] = Point{X: width - 1, Y: y}
	for x := width - 2; x >= 0; x-- {
		left := mat[x][y]
		up, down := math.MaxFloat64, math.MaxFloat64
		if y > 0 {
			up = mat[x][y-1]
		}
		if y < height-1 {
			down = mat[x][y+1]
		}

		if up <= left && up <= down {
			seam[x] = Point{X: x, Y: y - 1}
			y = y - 1
		} else if left <= up && left <= down {
			seam[x] = Point{X: x, Y: y}
		} else {
			seam[x] = Point{X: x, Y: y + 1}
			y = y + 1
		}
	}

	return seam
}
<commit_msg>Fix doc comment, remove redundant errors.New()<commit_after>// Package carve implements the seam carving algorithm for dyanmic image resizing.
package carve

import (
	"errors"
	"fmt"
	"image"
	"math"

	"github.com/disintegration/gift"
	"github.com/disintegration/imaging"
)

// ReduceHeight uses seam carving to reduce height of given image by n pixels.
func ReduceHeight(im image.Image, n int) (image.Image, error) {
	height := im.Bounds().Max.Y - im.Bounds().Min.Y
	if height < n {
		return im, fmt.Errorf("Cannot resize image of height %d by %d pixels", height, n)
	}

	for x := 0; x < n; x++ {
		energy := GenerateEnergyMap(im)
		seam := GenerateSeam(energy)
		im = RemoveSeam(im, seam)
	}
	return im, nil
}

// ReduceWidth uses seam carving to reduce width of given image by n pixels.
func ReduceWidth(im image.Image, n int) (image.Image, error) {
	width := im.Bounds().Max.Y - im.Bounds().Min.Y
	if width < n {
		return im, fmt.Errorf("Cannot resize image of width %d by %d pixels", width, n)
	}

	i := imaging.Rotate90(im)
	out, err := ReduceHeight(i, n)
	return imaging.Rotate270(out), err
}

// GenerateEnergyMap applies grayscale and sobel filters to the
// input image to create an energy map.
func GenerateEnergyMap(im image.Image) image.Image {
	g := gift.New(gift.Grayscale(), gift.Sobel())
	res := image.NewRGBA(im.Bounds())
	g.Draw(res, im)
	return res
}

// GenerateSeam returns the optimal horizontal seam for removal.
func GenerateSeam(im image.Image) Seam {
	mat := GenerateCostMatrix(im)
	return FindLowestCostSeam(mat)
}

// RemoveSeam creates a copy of the provided image, with the pixels at
// the points in the provided seam removed.
func RemoveSeam(im image.Image, seam Seam) image.Image {
	b := im.Bounds()
	out := image.NewRGBA(image.Rect(0, 0, b.Dx(), b.Dy()-1))
	min, max := b.Min, b.Max

	for _, point := range seam {
		x := point.X

		for y := min.Y; y < max.Y; y++ {
			if y == point.Y {
				continue
			}

			if y > point.Y {
				out.Set(x, y-1, im.At(x, y))
			} else {
				out.Set(x, y, im.At(x, y))
			}
		}
	}

	return out
}

// Seam defines a sequence of pixels through an image to be removed.
type Seam []Point

// Point defines an X Y point in an image.
type Point struct {
	X, Y int
}

// GenerateCostMatrix creates a matrix indicating the cumulative energy of the
// lowest cost seam from the left of the image to each pixel.
//
// mat[x][y] is the cumulative energy of the seam that runs from the left of
// the image to the pixel at column x, row y.
func GenerateCostMatrix(im image.Image) [][]float64 {
	min, max := im.Bounds().Min, im.Bounds().Max
	height, width := max.Y-min.Y, max.X-min.X

	mat := make([][]float64, width)
	for x := min.X; x < max.X; x++ {
		mat[x-min.X] = make([]float64, height)
	}

	for y := min.Y; y < max.Y; y++ {
		e, _, _, a := im.At(0, y).RGBA()
		mat[0][y-min.Y] = float64(e) / float64(a)
	}

	updatePoint := func(x, y int) {
		e, _, _, a := im.At(x, y).RGBA()

		up, down := math.MaxFloat64, math.MaxFloat64
		left := mat[x-1][y]
		if y != min.Y {
			up = mat[x-1][y-1]
		}
		if y < max.Y-1 {
			down = mat[x-1][y+1]
		}
		val := math.Min(float64(left), math.Min(float64(up), float64(down)))
		mat[x][y] = val + (float64(e) / float64(a))
	}

	// Calculate the remaining columns iteratively
	for x := min.X + 1; x < max.X; x++ {
		for y := min.Y; y < max.Y; y++ {
			updatePoint(x, y)
		}
	}

	return mat
}

// FindLowestCostSeam uses a cumulative cost matrix to identify the seam with
// the lowest total cumulative energy.
func FindLowestCostSeam(mat [][]float64) Seam {
	width, height := len(mat), len(mat[0])
	seam := make([]Point, width)

	min, y := math.MaxFloat64, 0
	for ind, val := range mat[width-1] {
		if val < min {
			min = val
			y = ind
		}
	}

	seam[width-1] = Point{X: width - 1, Y: y}
	for x := width - 2; x >= 0; x-- {
		left := mat[x][y]
		up, down := math.MaxFloat64, math.MaxFloat64
		if y > 0 {
			up = mat[x][y-1]
		}
		if y < height-1 {
			down = mat[x][y+1]
		}

		if up <= left && up <= down {
			seam[x] = Point{X: x, Y: y - 1}
			y = y - 1
		} else if left <= up && left <= down {
			seam[x] = Point{X: x, Y: y}
		} else {
			seam[x] = Point{X: x, Y: y + 1}
			y = y + 1
		}
	}

	return seam
}
<|endoftext|>


<commit_before>package cf

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"time"

	helpersCF "github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-cf/cf-rabbitmq-smoke-tests/lifecycle_tests/cf_helpers"
)

func Api(endpoint string, skipSSLValidation bool) *gexec.Session {
	apiCmd := []string{"api", endpoint}

	if skipSSLValidation {
		apiCmd = append(apiCmd, "--skip-ssl-validation")
	}

	return eventually(apiCmd...)
}

func Auth(username, password string) *gexec.Session {
	return eventually("auth", username, password)
}

func Target(orgName, spaceName string) *gexec.Session {
	return eventually("target", "-o", orgName, "-s", spaceName)
}

func CreateAndBindSecurityGroup(securityGroupName, orgName, spaceName string) *gexec.Session {
	sgs := []struct {
		Protocol    string `json:"protocol"`
		Destination string `json:"destination"`
		Ports       string `json:"ports"`
	}{
		{"tcp", "0.0.0.0/0", "5671,5672,1883,8883,61613,61614,15672,15674"},
	}

	sgFile, err := ioutil.TempFile("", "smoke-test-security-group-")
	Expect(err).NotTo(HaveOccurred())
	defer sgFile.Close()
	defer os.Remove(sgFile.Name())

	err = json.NewEncoder(sgFile).Encode(sgs)
	Expect(err).NotTo(HaveOccurred(), `{"FailReason": "Failed to encode security groups"}`)

	eventually("create-security-group", securityGroupName, sgFile.Name())
	return eventually("bind-security-group", securityGroupName, orgName, spaceName)
}

func CreateAndSetQuota(quotaName, orgName string) *gexec.Session {
	eventually("create-quota", quotaName, "-m", "10G", "-r", "1000", "-s", "100", "--allow-paid-service-plans")

	return eventually("set-quota", orgName, quotaName)
}

func EnableServiceAccess(serviceOffering, testPlan, orgName string) *gexec.Session {
	return eventually("enable-service-access", serviceOffering, "-p", testPlan, "-o", orgName)
}

func CreateOrg(orgName string) *gexec.Session {
	return eventually("create-org", orgName)
}

func CreateSpace(orgName, spaceName string) *gexec.Session {
	return eventually("create-space", spaceName, "-o", orgName)
}

func DeleteOrg(orgName string) *gexec.Session {
	return eventuallyWithTimeout(cf_helpers.ThirtySecondTimeout, "delete-org", orgName, "-f")
}

func DeleteSpace(spaceName string) *gexec.Session {
	return eventuallyWithTimeout(cf_helpers.ThirtySecondTimeout, "delete-space", spaceName, "-f")
}

func DeleteApp(appName string) *gexec.Session {
	return eventuallyWithTimeout(cf_helpers.ThirtySecondTimeout, "delete", appName, "-f", "-r")
}

func DeleteSecurityGroup(securityGroupName string) *gexec.Session {
	return eventuallyWithTimeout(cf_helpers.ThirtySecondTimeout, "delete-security-group", securityGroupName, "-f")
}

func DeleteQuota(quotaName string) *gexec.Session {
	return eventuallyWithTimeout(cf_helpers.ThirtySecondTimeout, "delete-quota", quotaName, "-f")
}

func CreateService(serviceOffering, servicePlan, serviceName, arbitraryParams string) *gexec.Session {
	args := []string{"create-service", serviceOffering, servicePlan, serviceName}
	if arbitraryParams != "" {
		args = append(args, "-c", arbitraryParams)
	}

	session := eventuallyWithTimeout(cf_helpers.FiveMinuteTimeout, args...)
	cf_helpers.AwaitServiceCreation(serviceName)
	return session
}

func UpdateService(serviceName, planName string) *gexec.Session {
	return eventuallyWithTimeout(cf_helpers.FiveMinuteTimeout, "update-service", serviceName, "-p", planName)
}

func UnbindService(appName, serviceName string) *gexec.Session {
	return eventuallyWithTimeout(cf_helpers.FiveMinuteTimeout, "unbind-service", appName, serviceName)
}

func AssertProgress(serviceName, operation string) {
	session := eventuallyWithTimeout(cf_helpers.FiveMinuteTimeout, "service", serviceName)
	Eventually(session).Should(gbytes.Say(operation + " in progress"))
}

func DeleteService(serviceName string) {
	eventuallyWithTimeout(cf_helpers.TenMinuteTimeout, "delete-service", serviceName, "-f")

	cf_helpers.AwaitServiceDeletion(serviceName)
}

func eventually(args ...string) *gexec.Session {
	return eventuallyWithTimeout(cf_helpers.FiveSecondTimeout, args...)
}

func eventuallyWithTimeout(timeout time.Duration, args ...string) *gexec.Session {
	session := helpersCF.Cf(args...)
	Eventually(session, timeout).Should(gexec.Exit(0))
	return session
}
<commit_msg>Increase default timeout to 30 seconds<commit_after>package cf

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"time"

	helpersCF "github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-cf/cf-rabbitmq-smoke-tests/lifecycle_tests/cf_helpers"
)

func Api(endpoint string, skipSSLValidation bool) *gexec.Session {
	apiCmd := []string{"api", endpoint}

	if skipSSLValidation {
		apiCmd = append(apiCmd, "--skip-ssl-validation")
	}

	return eventually(apiCmd...)
}

func Auth(username, password string) *gexec.Session {
	return eventually("auth", username, password)
}

func Target(orgName, spaceName string) *gexec.Session {
	return eventually("target", "-o", orgName, "-s", spaceName)
}

func CreateAndBindSecurityGroup(securityGroupName, orgName, spaceName string) *gexec.Session {
	sgs := []struct {
		Protocol    string `json:"protocol"`
		Destination string `json:"destination"`
		Ports       string `json:"ports"`
	}{
		{"tcp", "0.0.0.0/0", "5671,5672,1883,8883,61613,61614,15672,15674"},
	}

	sgFile, err := ioutil.TempFile("", "smoke-test-security-group-")
	Expect(err).NotTo(HaveOccurred())
	defer sgFile.Close()
	defer os.Remove(sgFile.Name())

	err = json.NewEncoder(sgFile).Encode(sgs)
	Expect(err).NotTo(HaveOccurred(), `{"FailReason": "Failed to encode security groups"}`)

	eventually("create-security-group", securityGroupName, sgFile.Name())
	return eventually("bind-security-group", securityGroupName, orgName, spaceName)
}

func CreateAndSetQuota(quotaName, orgName string) *gexec.Session {
	eventually("create-quota", quotaName, "-m", "10G", "-r", "1000", "-s", "100", "--allow-paid-service-plans")

	return eventually("set-quota", orgName, quotaName)
}

func EnableServiceAccess(serviceOffering, testPlan, orgName string) *gexec.Session {
	return eventually("enable-service-access", serviceOffering, "-p", testPlan, "-o", orgName)
}

func CreateOrg(orgName string) *gexec.Session {
	return eventually("create-org", orgName)
}

func CreateSpace(orgName, spaceName string) *gexec.Session {
	return eventually("create-space", spaceName, "-o", orgName)
}

func DeleteOrg(orgName string) *gexec.Session {
	return eventuallyWithTimeout(cf_helpers.ThirtySecondTimeout, "delete-org", orgName, "-f")
}

func DeleteSpace(spaceName string) *gexec.Session {
	return eventuallyWithTimeout(cf_helpers.ThirtySecondTimeout, "delete-space", spaceName, "-f")
}

func DeleteApp(appName string) *gexec.Session {
	return eventuallyWithTimeout(cf_helpers.ThirtySecondTimeout, "delete", appName, "-f", "-r")
}

func DeleteSecurityGroup(securityGroupName string) *gexec.Session {
	return eventuallyWithTimeout(cf_helpers.ThirtySecondTimeout, "delete-security-group", securityGroupName, "-f")
}

func DeleteQuota(quotaName string) *gexec.Session {
	return eventuallyWithTimeout(cf_helpers.ThirtySecondTimeout, "delete-quota", quotaName, "-f")
}

func CreateService(serviceOffering, servicePlan, serviceName, arbitraryParams string) *gexec.Session {
	args := []string{"create-service", serviceOffering, servicePlan, serviceName}
	if arbitraryParams != "" {
		args = append(args, "-c", arbitraryParams)
	}

	session := eventuallyWithTimeout(cf_helpers.FiveMinuteTimeout, args...)
	cf_helpers.AwaitServiceCreation(serviceName)
	return session
}

func UpdateService(serviceName, planName string) *gexec.Session {
	return eventuallyWithTimeout(cf_helpers.FiveMinuteTimeout, "update-service", serviceName, "-p", planName)
}

func UnbindService(appName, serviceName string) *gexec.Session {
	return eventuallyWithTimeout(cf_helpers.FiveMinuteTimeout, "unbind-service", appName, serviceName)
}

func AssertProgress(serviceName, operation string) {
	session := eventuallyWithTimeout(cf_helpers.FiveMinuteTimeout, "service", serviceName)
	Eventually(session).Should(gbytes.Say(operation + " in progress"))
}

func DeleteService(serviceName string) {
	eventuallyWithTimeout(cf_helpers.TenMinuteTimeout, "delete-service", serviceName, "-f")

	cf_helpers.AwaitServiceDeletion(serviceName)
}

func eventually(args ...string) *gexec.Session {
	return eventuallyWithTimeout(cf_helpers.ThirtySecondTimeout, args...)
}

func eventuallyWithTimeout(timeout time.Duration, args ...string) *gexec.Session {
	session := helpersCF.Cf(args...)
	Eventually(session, timeout).Should(gexec.Exit(0))
	return session
}
<|endoftext|>


<commit_before>// Copyright 2016 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package web

import (
	"fmt"
	"time"

	"github.com/control-center/serviced/dao"
	"github.com/control-center/serviced/health"
	"github.com/control-center/serviced/metrics"
	"github.com/control-center/serviced/node"
	"github.com/zenoss/glog"
	"github.com/zenoss/go-json-rest"
)

// ConciseServiceStatus contains the information necessary to display the
// status of a service in the UI. It joins running instances with health checks
// with memory stats. It does not include fields unnecessary for status
// display, such as config files or monitoring profiles.
type ConciseServiceStatus struct {
	DockerID        string
	HostID          string
	ID              string
	InSync          bool
	InstanceID      int
	Instances       int
	Name            string
	ParentServiceID string
	PoolID          string
	ServiceID       string
	StartedAt       time.Time
	RAMMax          int64
	RAMLast         int64
	RAMAverage      int64
	HealthChecks    map[string]health.HealthStatus
}

func memoryKey(serviceID, instanceID string) string {
	return fmt.Sprintf("%s-%s", serviceID, instanceID)
}

// convertInstancesToMetric converts dao.RunningServices into the structure the
// metrics API wants
func convertInstancesToMetric(instances []dao.RunningService) []metrics.ServiceInstance {
	svcInstances := make([]metrics.ServiceInstance, len(instances))
	for i, inst := range instances {
		if inst.ServiceID == "isvc-internalservices" {
			continue
		}
		svcInstances[i] = metrics.ServiceInstance{
			inst.ServiceID,
			inst.InstanceID,
		}
	}
	return svcInstances
}

func getAllServiceStatuses(client *node.ControlClient) (statuses []*ConciseServiceStatus, err error) {
	// Get all running service instances
	var instances []dao.RunningService
	if err := client.GetRunningServices(&empty, &instances); err != nil {
		return nil, err
	}
	if instances == nil {
		glog.V(3).Info("Nil service list returned")
		instances = []dao.RunningService{}
	}

	// Look up memory stats for the last hour
	// Memory stats for isvcs don't work the same way, so we don't append
	// those services to this list yet
	memoryStats := make(map[string]metrics.MemoryUsageStats)
	if len(instances) > 0 {
		response := []metrics.MemoryUsageStats{}
		query := dao.MetricRequest{
			StartTime: time.Now().Add(-time.Hour * 24),
			Instances: convertInstancesToMetric(instances),
		}
		if err := client.GetInstanceMemoryStats(query, &response); err != nil {
			glog.Errorf("Unable to look up instance memory stats (%s)", err)
		}
		for _, mus := range response {
			memoryStats[memoryKey(mus.ServiceID, mus.InstanceID)] = mus
		}
	}

	// Append internal services to the list
	instances = append(instances, getIRS()...)

	// Look up health check statuses
	healthChecks := make(map[string]map[string]health.HealthStatus)
	if len(instances) > 0 {
		results := make(map[string]map[int]map[string]health.HealthStatus)
		if err := client.GetServicesHealth(0, &results); err != nil {
			glog.Errorf("Unable to look up health check results (%s)", err)
		}
		for svcid, insts := range results {
			for instid, checks := range insts {
				healthChecks[memoryKey(svcid, string(instid))] = checks
			}
		}
	}

	// Create the concise service statuses
	for _, instance := range instances {
		stat := &ConciseServiceStatus{
			DockerID:        instance.DockerID,
			HostID:          instance.HostID,
			ID:              instance.ID,
			InSync:          instance.InSync,
			InstanceID:      instance.InstanceID,
			Instances:       instance.Instances,
			Name:            instance.Name,
			ParentServiceID: instance.ParentServiceID,
			PoolID:          instance.PoolID,
			ServiceID:       instance.ServiceID,
			StartedAt:       instance.StartedAt,
		}
		key := memoryKey(instance.ServiceID, string(instance.InstanceID))
		if mem, ok := memoryStats[key]; ok {
			stat.RAMMax = mem.Max
			stat.RAMLast = mem.Last
			stat.RAMAverage = mem.Average
		}
		if hc, ok := healthChecks[key]; ok {
			stat.HealthChecks = hc
		}
		statuses = append(statuses, stat)
	}
	return
}

func restGetConciseServiceStatus(w *rest.ResponseWriter, r *rest.Request, client *node.ControlClient) {
	statuses, err := getAllServiceStatuses(client)
	if err != nil {
		glog.Errorf("Could not get services: %v", err)
		restServerError(w, err)
	}
	w.WriteJson(statuses)
}
<commit_msg>Don't both filtering out isvcs, as they are not included<commit_after>// Copyright 2016 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package web

import (
	"fmt"
	"time"

	"github.com/control-center/serviced/dao"
	"github.com/control-center/serviced/health"
	"github.com/control-center/serviced/metrics"
	"github.com/control-center/serviced/node"
	"github.com/zenoss/glog"
	"github.com/zenoss/go-json-rest"
)

// ConciseServiceStatus contains the information necessary to display the
// status of a service in the UI. It joins running instances with health checks
// with memory stats. It does not include fields unnecessary for status
// display, such as config files or monitoring profiles.
type ConciseServiceStatus struct {
	DockerID        string
	HostID          string
	ID              string
	InSync          bool
	InstanceID      int
	Instances       int
	Name            string
	ParentServiceID string
	PoolID          string
	ServiceID       string
	StartedAt       time.Time
	RAMMax          int64
	RAMLast         int64
	RAMAverage      int64
	HealthChecks    map[string]health.HealthStatus
}

func memoryKey(serviceID, instanceID string) string {
	return fmt.Sprintf("%s-%s", serviceID, instanceID)
}

// convertInstancesToMetric converts dao.RunningServices into the structure the
// metrics API wants
func convertInstancesToMetric(instances []dao.RunningService) []metrics.ServiceInstance {
	svcInstances := make([]metrics.ServiceInstance, len(instances))
	for i, inst := range instances {
		svcInstances[i] = metrics.ServiceInstance{
			inst.ServiceID,
			inst.InstanceID,
		}
	}
	return svcInstances
}

func getAllServiceStatuses(client *node.ControlClient) (statuses []*ConciseServiceStatus, err error) {
	// Get all running service instances
	var instances []dao.RunningService
	if err := client.GetRunningServices(&empty, &instances); err != nil {
		return nil, err
	}
	if instances == nil {
		glog.V(3).Info("Nil service list returned")
		instances = []dao.RunningService{}
	}

	// Look up memory stats for the last hour
	// Memory stats for isvcs don't work the same way, so we don't append
	// those services to this list yet
	memoryStats := make(map[string]metrics.MemoryUsageStats)
	if len(instances) > 0 {
		response := []metrics.MemoryUsageStats{}
		query := dao.MetricRequest{
			StartTime: time.Now().Add(-time.Hour * 24),
			Instances: convertInstancesToMetric(instances),
		}
		if err := client.GetInstanceMemoryStats(query, &response); err != nil {
			glog.Errorf("Unable to look up instance memory stats (%s)", err)
		}
		for _, mus := range response {
			memoryStats[memoryKey(mus.ServiceID, mus.InstanceID)] = mus
		}
	}

	// Append internal services to the list
	instances = append(instances, getIRS()...)

	// Look up health check statuses
	healthChecks := make(map[string]map[string]health.HealthStatus)
	if len(instances) > 0 {
		results := make(map[string]map[int]map[string]health.HealthStatus)
		if err := client.GetServicesHealth(0, &results); err != nil {
			glog.Errorf("Unable to look up health check results (%s)", err)
		}
		for svcid, insts := range results {
			for instid, checks := range insts {
				healthChecks[memoryKey(svcid, string(instid))] = checks
			}
		}
	}

	// Create the concise service statuses
	for _, instance := range instances {
		stat := &ConciseServiceStatus{
			DockerID:        instance.DockerID,
			HostID:          instance.HostID,
			ID:              instance.ID,
			InSync:          instance.InSync,
			InstanceID:      instance.InstanceID,
			Instances:       instance.Instances,
			Name:            instance.Name,
			ParentServiceID: instance.ParentServiceID,
			PoolID:          instance.PoolID,
			ServiceID:       instance.ServiceID,
			StartedAt:       instance.StartedAt,
		}
		key := memoryKey(instance.ServiceID, string(instance.InstanceID))
		if mem, ok := memoryStats[key]; ok {
			stat.RAMMax = mem.Max
			stat.RAMLast = mem.Last
			stat.RAMAverage = mem.Average
		}
		if hc, ok := healthChecks[key]; ok {
			stat.HealthChecks = hc
		}
		statuses = append(statuses, stat)
	}
	return
}

func restGetConciseServiceStatus(w *rest.ResponseWriter, r *rest.Request, client *node.ControlClient) {
	statuses, err := getAllServiceStatuses(client)
	if err != nil {
		glog.Errorf("Could not get services: %v", err)
		restServerError(w, err)
	}
	w.WriteJson(statuses)
}
<|endoftext|>


<commit_before>package store

import (
    "fmt"
    "log"
    "errors"
    "strings"
    "strconv"
    "encoding/json"
    "huabot-sched/sched"
    "github.com/garyburd/redigo/redis"
)

const REDIS_PREFIX = "huabot-sched:job:"

type RedisStore struct {
    pool *redis.Pool
}

func NewRedisStore(server string) RedisStore {
    parts := strings.SplitN(server, "://", 2)
    pool := redis.NewPool(func() (conn redis.Conn, err error) {
        conn, err = redis.Dial("tcp", parts[1])
        return
    }, 3)

    return RedisStore{pool: pool,}
}

func (r RedisStore) get(jobId int64) (job sched.Job, err error) {
    var data []byte
    var conn = r.pool.Get()
    defer conn.Close()
    var key = REDIS_PREFIX + strconv.FormatInt(job.Id, 10)
    data, err = redis.Bytes(conn.Do("GET", key))
    if err != nil {
        return
    }
    err = json.Unmarshal(data, &job)
    return
}

func (r RedisStore) Save(job sched.Job) (err error) {
    var key string
    var prefix = REDIS_PREFIX + job.Func + ":"
    var conn = r.pool.Get()
    defer conn.Close()
    if job.Id > 0 {
        old, e := r.get(job.Id)
        key = REDIS_PREFIX + strconv.FormatInt(job.Id, 10)
        if e != nil || old.Id < 1 {
            err = errors.New(fmt.Sprintf("Update Job %d fail, the old job is not exists.", job.Id))
            return
        }
        if old.Name != job.Name {
            if _, e := conn.Do("ZERM", prefix + "name", old.Name); e != nil {
                log.Printf("DelIndex Error: %s %s\n", prefix + "name", old.Name)
            }
        }
    } else {
        job.Id, err = redis.Int64(conn.Do("INCRBY", REDIS_PREFIX + "sequence", 1))
        if err != nil {
            return
        }
    }
    idx, _ := redis.Int64(conn.Do("ZSCORE", prefix + "name", job.Name))
    if idx > 0 && idx != job.Id {
        err = errors.New("Duplicate Job name: " + job.Name)
        return
    }
    key = REDIS_PREFIX + strconv.FormatInt(job.Id, 10)
    var data []byte
    data, err = json.Marshal(job)
    if err != nil {
        return
    }
    _, err = conn.Do("SET", key, data)
    if err == nil {
        if _, e := conn.Do("ZADD", prefix + "name", job.Name, job.Id); e != nil {
            log.Printf("DelIndex Error: %s %s\n",  prefix + "name", job.Name)
        }
        if _, e := conn.Do("ZADD", REDIS_PREFIX + "ID", strconv.FormatInt(job.Id, 10), job.Id); e != nil {
            log.Printf("DelIndex Error: %s %s\n",  prefix + "name", job.Name)
        }
    }
    return
}

func (r RedisStore) Delete(jobId int64) (err error) {
    var key = REDIS_PREFIX + strconv.FormatInt(jobId, 10)
    job, e := r.get(jobId)
    if e != nil {
        return e
    }
    var prefix = REDIS_PREFIX + job.Func + ":"

    var conn = r.pool.Get()
    defer conn.Close()

    _, err = conn.Do("DEL", key)
    conn.Do("ZREM", prefix + "name", job.Name)
    conn.Do("ZREM", REDIS_PREFIX + "ID", strconv.FormatInt(job.Id, 10))
    return
}

func (r RedisStore) Get(jobId int64) (job sched.Job, err error) {
    job, err = r.get(jobId)
    return
}

func (r RedisStore) GetOne(Func string, jobName string) (job sched.Job, err error) {
    var conn = r.pool.Get()
    defer conn.Close()
    jobId, _ := redis.Int64(conn.Do("ZSCORE", REDIS_PREFIX + Func + ":name", jobName))
    if jobId > 0 {
        return r.get(jobId)
    }
    return
}

func (r RedisStore) NewIterator(Func []byte) sched.JobIterator {
    return &RedisIterator{
        Func: Func,
        cursor: 0,
        cacheJob: make([]sched.Job, 0),
        start: 0,
        limit: 20,
        err: nil,
        r: r,
    }
}

type RedisIterator struct {
    Func   []byte
    cursor int
    err    error
    cacheJob []sched.Job
    start  int
    limit  int
    r      RedisStore
}

func (iter *RedisIterator) Next() bool {
    iter.cursor += 1
    if len(iter.cacheJob) > 0 && len(iter.cacheJob) > iter.cursor {
        return true
    }
    start := iter.start
    stop := iter.start + iter.limit - 1
    iter.start = iter.start + iter.limit
    var err error

    var conn = iter.r.pool.Get()
    defer conn.Close()
    var key string
    if iter.Func == nil {
        key = REDIS_PREFIX + "ID"
    } else {
        key = REDIS_PREFIX + string(iter.Func) + ":name"
    }

    reply, err := redis.Values(conn.Do("ZRANGE", key, start, stop, "WITHSCORES"))
    if err != nil || len(reply) == 0 {
        return false
    }
    var jobId int64
    jobs := make([]sched.Job, len(reply)/2)
    for k, v := range reply {
        if k % 2 == 1 {
            jobId, _ = strconv.ParseInt(string(v.([]byte)), 10, 0)
            jobs[(k - 1) / 2], _ = iter.r.get(jobId)
        }
    }
    iter.cacheJob = jobs
    iter.cursor = 0
    return true
}

func (iter *RedisIterator) Value() sched.Job {
    return iter.cacheJob[iter.cursor]
}

func (iter *RedisIterator) Error() error {
    return iter.err
}

func (iter *RedisIterator) Close() {

}
<commit_msg>Fix. get job<commit_after>package store

import (
    "fmt"
    "log"
    "errors"
    "strings"
    "strconv"
    "encoding/json"
    "huabot-sched/sched"
    "github.com/garyburd/redigo/redis"
)

const REDIS_PREFIX = "huabot-sched:job:"

type RedisStore struct {
    pool *redis.Pool
}

func NewRedisStore(server string) RedisStore {
    parts := strings.SplitN(server, "://", 2)
    pool := redis.NewPool(func() (conn redis.Conn, err error) {
        conn, err = redis.Dial("tcp", parts[1])
        return
    }, 3)

    return RedisStore{pool: pool,}
}

func (r RedisStore) get(jobId int64) (job sched.Job, err error) {
    var data []byte
    var conn = r.pool.Get()
    defer conn.Close()
    var key = REDIS_PREFIX + strconv.FormatInt(jobId, 10)
    data, err = redis.Bytes(conn.Do("GET", key))
    if err != nil {
        return
    }
    err = json.Unmarshal(data, &job)
    return
}

func (r RedisStore) Save(job sched.Job) (err error) {
    var key string
    var prefix = REDIS_PREFIX + job.Func + ":"
    var conn = r.pool.Get()
    defer conn.Close()
    if job.Id > 0 {
        old, e := r.get(job.Id)
        key = REDIS_PREFIX + strconv.FormatInt(job.Id, 10)
        if e != nil || old.Id < 1 {
            err = errors.New(fmt.Sprintf("Update Job %d fail, the old job is not exists.", job.Id))
            return
        }
        if old.Name != job.Name {
            if _, e := conn.Do("ZERM", prefix + "name", old.Name); e != nil {
                log.Printf("DelIndex Error: %s %s\n", prefix + "name", old.Name)
            }
        }
    } else {
        job.Id, err = redis.Int64(conn.Do("INCRBY", REDIS_PREFIX + "sequence", 1))
        if err != nil {
            return
        }
    }
    idx, _ := redis.Int64(conn.Do("ZSCORE", prefix + "name", job.Name))
    if idx > 0 && idx != job.Id {
        err = errors.New("Duplicate Job name: " + job.Name)
        return
    }
    key = REDIS_PREFIX + strconv.FormatInt(job.Id, 10)
    var data []byte
    data, err = json.Marshal(job)
    if err != nil {
        return
    }
    _, err = conn.Do("SET", key, data)
    if err == nil {
        if _, e := conn.Do("ZADD", prefix + "name", job.Name, job.Id); e != nil {
            log.Printf("DelIndex Error: %s %s\n",  prefix + "name", job.Name)
        }
        if _, e := conn.Do("ZADD", REDIS_PREFIX + "ID", strconv.FormatInt(job.Id, 10), job.Id); e != nil {
            log.Printf("DelIndex Error: %s %s\n",  prefix + "name", job.Name)
        }
    }
    return
}

func (r RedisStore) Delete(jobId int64) (err error) {
    var key = REDIS_PREFIX + strconv.FormatInt(jobId, 10)
    job, e := r.get(jobId)
    if e != nil {
        return e
    }
    var prefix = REDIS_PREFIX + job.Func + ":"

    var conn = r.pool.Get()
    defer conn.Close()

    _, err = conn.Do("DEL", key)
    conn.Do("ZREM", prefix + "name", job.Name)
    conn.Do("ZREM", REDIS_PREFIX + "ID", strconv.FormatInt(job.Id, 10))
    return
}

func (r RedisStore) Get(jobId int64) (job sched.Job, err error) {
    job, err = r.get(jobId)
    return
}

func (r RedisStore) GetOne(Func string, jobName string) (job sched.Job, err error) {
    var conn = r.pool.Get()
    defer conn.Close()
    jobId, _ := redis.Int64(conn.Do("ZSCORE", REDIS_PREFIX + Func + ":name", jobName))
    if jobId > 0 {
        return r.get(jobId)
    }
    return
}

func (r RedisStore) NewIterator(Func []byte) sched.JobIterator {
    return &RedisIterator{
        Func: Func,
        cursor: 0,
        cacheJob: make([]sched.Job, 0),
        start: 0,
        limit: 20,
        err: nil,
        r: r,
    }
}

type RedisIterator struct {
    Func   []byte
    cursor int
    err    error
    cacheJob []sched.Job
    start  int
    limit  int
    r      RedisStore
}

func (iter *RedisIterator) Next() bool {
    iter.cursor += 1
    if len(iter.cacheJob) > 0 && len(iter.cacheJob) > iter.cursor {
        return true
    }
    start := iter.start
    stop := iter.start + iter.limit - 1
    iter.start = iter.start + iter.limit
    var err error

    var conn = iter.r.pool.Get()
    defer conn.Close()
    var key string
    if iter.Func == nil {
        key = REDIS_PREFIX + "ID"
    } else {
        key = REDIS_PREFIX + string(iter.Func) + ":name"
    }

    reply, err := redis.Values(conn.Do("ZRANGE", key, start, stop, "WITHSCORES"))
    if err != nil || len(reply) == 0 {
        return false
    }
    var jobId int64
    jobs := make([]sched.Job, len(reply)/2)
    for k, v := range reply {
        if k % 2 == 1 {
            jobId, _ = strconv.ParseInt(string(v.([]byte)), 10, 0)
            jobs[(k - 1) / 2], _ = iter.r.get(jobId)
        }
    }
    iter.cacheJob = jobs
    iter.cursor = 0
    return true
}

func (iter *RedisIterator) Value() sched.Job {
    return iter.cacheJob[iter.cursor]
}

func (iter *RedisIterator) Error() error {
    return iter.err
}

func (iter *RedisIterator) Close() {

}
<|endoftext|>