<commit_before>package git

import (
	"bytes"
	"compress/zlib"
	"errors"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
)

func readIdxFile(path string) (*idxFile, error) {
	ifile := &idxFile{}
	ifile.indexpath = path
	ifile.packpath = path[0:len(path)-3] + "pack"
	idx, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}

	if !bytes.HasPrefix(idx, []byte{255, 't', 'O', 'c'}) {
		return nil, errors.New("Not version 2 index file")
	}
	pos := 8
	var fanout [256]uint32
	for i := 0; i < 256; i++ {
		// TODO: use range
		fanout[i] = uint32(idx[pos])<<24 + uint32(idx[pos+1])<<16 + uint32(idx[pos+2])<<8 + uint32(idx[pos+3])
		pos += 4
	}
	numObjects := int(fanout[255])
	ids := make([]sha1, numObjects)

	for i := 0; i < numObjects; i++ {
		for j := 0; j < 20; j++ {
			ids[i][j] = idx[pos+j]
		}
		pos = pos + 20
	}
	// skip crc32 and offsetValues4
	pos += 8 * numObjects

	excessLen := len(idx) - 258*4 - 28*numObjects - 40
	var offsetValues8 []uint64
	if excessLen > 0 {
		// We have an index table, so let's read it first
		offsetValues8 = make([]uint64, excessLen/8)
		for i := 0; i < excessLen/8; i++ {
			offsetValues8[i] = uint64(idx[pos])<<070 + uint64(idx[pos+1])<<060 + uint64(idx[pos+2])<<050 + uint64(idx[pos+3])<<040 + uint64(idx[pos+4])<<030 + uint64(idx[pos+5])<<020 + uint64(idx[pos+6])<<010 + uint64(idx[pos+7])
			pos = pos + 8
		}
	}
	ifile.offsetValues = make(map[sha1]uint64, numObjects)
	pos = 258*4 + 24*numObjects
	for i := 0; i < numObjects; i++ {
		offset := uint32(idx[pos])<<24 + uint32(idx[pos+1])<<16 + uint32(idx[pos+2])<<8 + uint32(idx[pos+3])
		offset32ndbit := offset & 0x80000000
		offset31bits := offset & 0x7FFFFFFF
		if offset32ndbit == 0x80000000 {
			// it's an index entry
			ifile.offsetValues[ids[i]] = offsetValues8[offset31bits]
		} else {
			ifile.offsetValues[ids[i]] = uint64(offset31bits)
		}
		pos = pos + 4
	}
	// sha1Packfile := idx[pos : pos+20]
	// sha1Index := idx[pos+21 : pos+40]
	fi, err := os.Open(ifile.packpath)
	if err != nil {
		return nil, err
	}
	defer fi.Close()

	packVersion := make([]byte, 8)
	_, err = fi.Read(packVersion)
	if err != nil {
		return nil, err
	}
	if !bytes.HasPrefix(packVersion, []byte{'P', 'A', 'C', 'K'}) {
		return nil, errors.New("Pack file does not start with 'PACK'")
	}
	ifile.packversion = uint32(packVersion[4])<<24 + uint32(packVersion[5])<<16 + uint32(packVersion[6])<<8 + uint32(packVersion[7])
	return ifile, nil
}

// If the object is stored in its own file (i.e not in a pack file),
// this function returns the full path to the object file.
// It does not test if the file exists.
func filepathFromSHA1(rootdir, sha1 string) string {
	return filepath.Join(rootdir, "objects", sha1[:2], sha1[2:])
}

// The object length in a packfile is a bit more difficult than
// just reading the bytes. The first byte has the length in its
// lowest four bits, and if bit 7 is set, it means 'more' bytes
// will follow. These are added to the »left side« of the length
func readLenInPackFile(buf []byte) (length int, advance int) {
	advance = 0
	shift := [...]byte{0, 4, 11, 18, 25, 32, 39, 46, 53, 60}
	length = int(buf[advance] & 0x0F)
	for buf[advance]&0x80 > 0 {
		advance += 1
		length += (int(buf[advance]&0x7F) << shift[advance])
	}
	advance++
	return
}

// Read from a pack file (given by path) at position offset. If this is a
// non-delta object, the (inflated) bytes are just returned, if the object
// is a deltafied-object, we have to apply the delta to base objects
// before hand.
func readObjectBytes(path string, offset uint64, sizeonly bool) (ot ObjectType, length int64, dataRc io.ReadCloser, err error) {
	offsetInt := int64(offset)
	file, err := os.Open(path)
	if err != nil {
		return
	}

	defer func() {
		if err != nil || sizeonly {
			if file != nil {
				file.Close()
			}
		}
	}()

	pos, err := file.Seek(offsetInt, os.SEEK_SET)
	if err != nil {
		return
	}

	if pos != offsetInt {
		err = errors.New("Seek went wrong")
		return
	}

	buf := make([]byte, 1024)
	n, err := file.Read(buf)
	if err != nil {
		return
	}

	if n == 0 {
		err = errors.New("Nothing read from pack file")
		return
	}

	ot = ObjectType(buf[0] & 0x70)

	l, p := readLenInPackFile(buf)
	pos = int64(p)
	length = int64(l)

	var baseObjectOffset uint64
	switch ot {
	case ObjectCommit, ObjectTree, ObjectBlob, ObjectTag:
		if sizeonly {
			// if we are only interested in the size of the object,
			// we don't need to do more expensive stuff
			return
		}

		readerSeek(file, offsetInt+pos)
		dataRc, err = readerDecompressed(file, length)
		if err != nil {
			return
		}
		return
		// data, err = readCompressedDataFromFile(file, offsetInt+pos, length)

	case 0x60:
		// DELTA_ENCODED object w/ offset to base
		// Read the offset first, then calculate the starting point
		// of the base object
		num := int64(buf[pos]) & 0x7f
		for buf[pos]&0x80 > 0 {
			pos = pos + 1
			num = ((num + 1) << 7) | int64(buf[pos]&0x7f)
		}
		baseObjectOffset = uint64(offsetInt - num)
		pos = pos + 1

	case 0x70:
		// DELTA_ENCODED object w/ base BINARY_OBJID
		var id sha1
		id, err = NewId(buf[pos : pos+20])
		if err != nil {
			return
		}

		pos = pos + 20

		var f *idxFile
		f, err = readIdxFile(path[0:len(path)-4] + "idx")
		if err != nil {
			return
		}
		var ok bool
		if baseObjectOffset, ok = f.offsetValues[id]; !ok {
			log.Fatal("not implemented yet")
			err = errors.New("base object is not exist")
			return
		}
	}

	var (
		base   []byte
		baseRc io.ReadCloser
	)
	ot, _, baseRc, err = readObjectBytes(path, baseObjectOffset, false)
	if err != nil {
		return
	}

	defer func() {
		baseRc.Close()
	}()

	base, err = ioutil.ReadAll(baseRc)
	if err != nil {
		return
	}

	readerSeek(file, offsetInt+pos)
	rc, err := readerDecompressed(file, length)
	if err != nil {
		return
	}

	zpos := 0
	// This is the length of the base object. Do we need to know it?
	_, bytesRead := readerLittleEndianBase128Number(rc)
	log.Println(zpos, bytesRead)
	zpos += bytesRead

	resultObjectLength, bytesRead := readerLittleEndianBase128Number(rc)
	zpos += bytesRead

	if sizeonly {
		// if we are only interested in the size of the object,
		// we don't need to do more expensive stuff
		length = resultObjectLength
		return
	}

	br := &readAter{base}
	data, err := readerApplyDelta(br, rc, resultObjectLength)

	dataRc = newBufReadCloser(data)
	return
}

// Return length as integer from zero terminated string
// and the beginning of the real object
func getLengthZeroTerminated(b []byte) (int64, int64) {
	i := 0
	var pos int
	for b[i] != 0 {
		i++
	}
	pos = i
	i--
	var length int64
	var pow int64
	pow = 1
	for i >= 0 {
		length = length + (int64(b[i])-48)*pow
		pow = pow * 10
		i--
	}
	return length, int64(pos) + 1
}

// Read the contents of the object file at path.
// Return the content type, the contents of the file and error, if any
func readObjectFile(path string, sizeonly bool) (ot ObjectType, length int64, dataRc io.ReadCloser, err error) {
	f, err := os.Open(path)
	if err != nil {
		return
	}

	defer func() {
		if err != nil || sizeonly {
			if f != nil {
				f.Close()
			}
		}
	}()

	r, err := zlib.NewReader(f)
	if err != nil {
		return
	}

	firstBufferSize := int64(1024)

	buf := make([]byte, firstBufferSize)
	_, err = r.Read(buf)
	if err != nil {
		return
	}

	spacePos := int64(bytes.IndexByte(buf, ' '))

	// "tree", "commit", "blob", ...
	switch string(buf[:spacePos]) {
	case "blob":
		ot = ObjectBlob
	case "tree":
		ot = ObjectTree
	case "commit":
		ot = ObjectCommit
	case "tag":
		ot = ObjectTag
	}

	// length starts at the position after the space
	var objstart int64
	length, objstart = getLengthZeroTerminated(buf[spacePos+1:])

	if sizeonly {
		return
	}

	objstart += spacePos + 1

	readerSeek(f, 0)

	rc, err := readerDecompressed(f, length+objstart)
	if err != nil {
		return
	}
	readerSkip(rc, objstart)

	dataRc = newReadCloser(readerLimit(rc, length), rc)
	return
}
<commit_msg>Delete verbose log<commit_after>package git

import (
	"bytes"
	"compress/zlib"
	"errors"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
)

func readIdxFile(path string) (*idxFile, error) {
	ifile := &idxFile{}
	ifile.indexpath = path
	ifile.packpath = path[0:len(path)-3] + "pack"
	idx, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}

	if !bytes.HasPrefix(idx, []byte{255, 't', 'O', 'c'}) {
		return nil, errors.New("Not version 2 index file")
	}
	pos := 8
	var fanout [256]uint32
	for i := 0; i < 256; i++ {
		// TODO: use range
		fanout[i] = uint32(idx[pos])<<24 + uint32(idx[pos+1])<<16 + uint32(idx[pos+2])<<8 + uint32(idx[pos+3])
		pos += 4
	}
	numObjects := int(fanout[255])
	ids := make([]sha1, numObjects)

	for i := 0; i < numObjects; i++ {
		for j := 0; j < 20; j++ {
			ids[i][j] = idx[pos+j]
		}
		pos = pos + 20
	}
	// skip crc32 and offsetValues4
	pos += 8 * numObjects

	excessLen := len(idx) - 258*4 - 28*numObjects - 40
	var offsetValues8 []uint64
	if excessLen > 0 {
		// We have an index table, so let's read it first
		offsetValues8 = make([]uint64, excessLen/8)
		for i := 0; i < excessLen/8; i++ {
			offsetValues8[i] = uint64(idx[pos])<<070 + uint64(idx[pos+1])<<060 + uint64(idx[pos+2])<<050 + uint64(idx[pos+3])<<040 + uint64(idx[pos+4])<<030 + uint64(idx[pos+5])<<020 + uint64(idx[pos+6])<<010 + uint64(idx[pos+7])
			pos = pos + 8
		}
	}
	ifile.offsetValues = make(map[sha1]uint64, numObjects)
	pos = 258*4 + 24*numObjects
	for i := 0; i < numObjects; i++ {
		offset := uint32(idx[pos])<<24 + uint32(idx[pos+1])<<16 + uint32(idx[pos+2])<<8 + uint32(idx[pos+3])
		offset32ndbit := offset & 0x80000000
		offset31bits := offset & 0x7FFFFFFF
		if offset32ndbit == 0x80000000 {
			// it's an index entry
			ifile.offsetValues[ids[i]] = offsetValues8[offset31bits]
		} else {
			ifile.offsetValues[ids[i]] = uint64(offset31bits)
		}
		pos = pos + 4
	}
	// sha1Packfile := idx[pos : pos+20]
	// sha1Index := idx[pos+21 : pos+40]
	fi, err := os.Open(ifile.packpath)
	if err != nil {
		return nil, err
	}
	defer fi.Close()

	packVersion := make([]byte, 8)
	_, err = fi.Read(packVersion)
	if err != nil {
		return nil, err
	}
	if !bytes.HasPrefix(packVersion, []byte{'P', 'A', 'C', 'K'}) {
		return nil, errors.New("Pack file does not start with 'PACK'")
	}
	ifile.packversion = uint32(packVersion[4])<<24 + uint32(packVersion[5])<<16 + uint32(packVersion[6])<<8 + uint32(packVersion[7])
	return ifile, nil
}

// If the object is stored in its own file (i.e not in a pack file),
// this function returns the full path to the object file.
// It does not test if the file exists.
func filepathFromSHA1(rootdir, sha1 string) string {
	return filepath.Join(rootdir, "objects", sha1[:2], sha1[2:])
}

// The object length in a packfile is a bit more difficult than
// just reading the bytes. The first byte has the length in its
// lowest four bits, and if bit 7 is set, it means 'more' bytes
// will follow. These are added to the »left side« of the length
func readLenInPackFile(buf []byte) (length int, advance int) {
	advance = 0
	shift := [...]byte{0, 4, 11, 18, 25, 32, 39, 46, 53, 60}
	length = int(buf[advance] & 0x0F)
	for buf[advance]&0x80 > 0 {
		advance += 1
		length += (int(buf[advance]&0x7F) << shift[advance])
	}
	advance++
	return
}

// Read from a pack file (given by path) at position offset. If this is a
// non-delta object, the (inflated) bytes are just returned, if the object
// is a deltafied-object, we have to apply the delta to base objects
// before hand.
func readObjectBytes(path string, offset uint64, sizeonly bool) (ot ObjectType, length int64, dataRc io.ReadCloser, err error) {
	offsetInt := int64(offset)
	file, err := os.Open(path)
	if err != nil {
		return
	}

	defer func() {
		if err != nil || sizeonly {
			if file != nil {
				file.Close()
			}
		}
	}()

	pos, err := file.Seek(offsetInt, os.SEEK_SET)
	if err != nil {
		return
	}

	if pos != offsetInt {
		err = errors.New("Seek went wrong")
		return
	}

	buf := make([]byte, 1024)
	n, err := file.Read(buf)
	if err != nil {
		return
	}

	if n == 0 {
		err = errors.New("Nothing read from pack file")
		return
	}

	ot = ObjectType(buf[0] & 0x70)

	l, p := readLenInPackFile(buf)
	pos = int64(p)
	length = int64(l)

	var baseObjectOffset uint64
	switch ot {
	case ObjectCommit, ObjectTree, ObjectBlob, ObjectTag:
		if sizeonly {
			// if we are only interested in the size of the object,
			// we don't need to do more expensive stuff
			return
		}

		readerSeek(file, offsetInt+pos)
		dataRc, err = readerDecompressed(file, length)
		if err != nil {
			return
		}
		return
		// data, err = readCompressedDataFromFile(file, offsetInt+pos, length)

	case 0x60:
		// DELTA_ENCODED object w/ offset to base
		// Read the offset first, then calculate the starting point
		// of the base object
		num := int64(buf[pos]) & 0x7f
		for buf[pos]&0x80 > 0 {
			pos = pos + 1
			num = ((num + 1) << 7) | int64(buf[pos]&0x7f)
		}
		baseObjectOffset = uint64(offsetInt - num)
		pos = pos + 1

	case 0x70:
		// DELTA_ENCODED object w/ base BINARY_OBJID
		var id sha1
		id, err = NewId(buf[pos : pos+20])
		if err != nil {
			return
		}

		pos = pos + 20

		var f *idxFile
		f, err = readIdxFile(path[0:len(path)-4] + "idx")
		if err != nil {
			return
		}
		var ok bool
		if baseObjectOffset, ok = f.offsetValues[id]; !ok {
			log.Fatal("not implemented yet")
			err = errors.New("base object is not exist")
			return
		}
	}

	var (
		base   []byte
		baseRc io.ReadCloser
	)
	ot, _, baseRc, err = readObjectBytes(path, baseObjectOffset, false)
	if err != nil {
		return
	}

	defer func() {
		baseRc.Close()
	}()

	base, err = ioutil.ReadAll(baseRc)
	if err != nil {
		return
	}

	readerSeek(file, offsetInt+pos)
	rc, err := readerDecompressed(file, length)
	if err != nil {
		return
	}

	zpos := 0
	// This is the length of the base object. Do we need to know it?
	_, bytesRead := readerLittleEndianBase128Number(rc)
	//log.Println(zpos, bytesRead)
	zpos += bytesRead

	resultObjectLength, bytesRead := readerLittleEndianBase128Number(rc)
	zpos += bytesRead

	if sizeonly {
		// if we are only interested in the size of the object,
		// we don't need to do more expensive stuff
		length = resultObjectLength
		return
	}

	br := &readAter{base}
	data, err := readerApplyDelta(br, rc, resultObjectLength)

	dataRc = newBufReadCloser(data)
	return
}

// Return length as integer from zero terminated string
// and the beginning of the real object
func getLengthZeroTerminated(b []byte) (int64, int64) {
	i := 0
	var pos int
	for b[i] != 0 {
		i++
	}
	pos = i
	i--
	var length int64
	var pow int64
	pow = 1
	for i >= 0 {
		length = length + (int64(b[i])-48)*pow
		pow = pow * 10
		i--
	}
	return length, int64(pos) + 1
}

// Read the contents of the object file at path.
// Return the content type, the contents of the file and error, if any
func readObjectFile(path string, sizeonly bool) (ot ObjectType, length int64, dataRc io.ReadCloser, err error) {
	f, err := os.Open(path)
	if err != nil {
		return
	}

	defer func() {
		if err != nil || sizeonly {
			if f != nil {
				f.Close()
			}
		}
	}()

	r, err := zlib.NewReader(f)
	if err != nil {
		return
	}

	firstBufferSize := int64(1024)

	buf := make([]byte, firstBufferSize)
	_, err = r.Read(buf)
	if err != nil {
		return
	}

	spacePos := int64(bytes.IndexByte(buf, ' '))

	// "tree", "commit", "blob", ...
	switch string(buf[:spacePos]) {
	case "blob":
		ot = ObjectBlob
	case "tree":
		ot = ObjectTree
	case "commit":
		ot = ObjectCommit
	case "tag":
		ot = ObjectTag
	}

	// length starts at the position after the space
	var objstart int64
	length, objstart = getLengthZeroTerminated(buf[spacePos+1:])

	if sizeonly {
		return
	}

	objstart += spacePos + 1

	readerSeek(f, 0)

	rc, err := readerDecompressed(f, length+objstart)
	if err != nil {
		return
	}
	readerSkip(rc, objstart)

	dataRc = newReadCloser(readerLimit(rc, length), rc)
	return
}
<|endoftext|>


<commit_before>package api

import (
	"github.com/go-macaron/binding"
	"github.com/grafana/metrictank/api/middleware"
	"github.com/grafana/metrictank/api/models"
	"github.com/raintank/gziper"
	"gopkg.in/macaron.v1"
)

func (s *Server) RegisterRoutes() {
	r := s.Macaron
	if useGzip {
		r.Use(gziper.Gziper())
	}
	r.Use(middleware.RequestStats())
	r.Use(middleware.Tracer(s.Tracer))
	r.Use(macaron.Renderer())
	r.Use(middleware.OrgMiddleware(multiTenant))
	r.Use(middleware.CorsHandler())

	bind := binding.Bind
	withOrg := middleware.RequireOrg()
	cBody := middleware.CaptureBody
	ready := middleware.NodeReady()

	r.Get("/", s.appStatus)
	r.Get("/node", s.getNodeStatus)
	r.Post("/node", bind(models.NodeStatus{}), s.setNodeStatus)
	r.Get("/debug/pprof/block", blockHandler)
	r.Get("/debug/pprof/mutex", mutexHandler)

	r.Get("/cluster", s.getClusterStatus)
	r.Post("/cluster", bind(models.ClusterMembers{}), s.postClusterMembers)

	r.Combo("/getdata", ready, bind(models.GetData{})).Get(s.getData).Post(s.getData)

	r.Combo("/index/find", ready, bind(models.IndexFind{})).Get(s.indexFind).Post(s.indexFind)
	r.Combo("/index/list", ready, bind(models.IndexList{})).Get(s.indexList).Post(s.indexList)
	r.Combo("/index/delete", ready, bind(models.IndexDelete{})).Get(s.indexDelete).Post(s.indexDelete)
	r.Combo("/index/get", ready, bind(models.IndexGet{})).Get(s.indexGet).Post(s.indexGet)
	r.Combo("/index/tags", ready, bind(models.IndexTags{})).Get(s.indexTags).Post(s.indexTags)
	r.Combo("/index/tags/findSeries", ready, bind(models.IndexTagFindSeries{})).Get(s.indexTagFindSeries).Post(s.indexTagFindSeries)
	r.Combo("/index/tags/:tag([0-9a-zA-Z]+)", ready, bind(models.IndexTagDetails{})).Get(s.indexTagDetails).Post(s.indexTagDetails)

	r.Options("/*", func(ctx *macaron.Context) {
		ctx.Write(nil)
	})

	// Graphite endpoints
	r.Combo("/render", cBody, withOrg, ready, bind(models.GraphiteRender{})).Get(s.renderMetrics).Post(s.renderMetrics)
	r.Combo("/metrics/find", withOrg, ready, bind(models.GraphiteFind{})).Get(s.metricsFind).Post(s.metricsFind)
	r.Get("/metrics/index.json", withOrg, ready, s.metricsIndex)
	r.Post("/metrics/delete", withOrg, ready, bind(models.MetricsDelete{}), s.metricsDelete)
	r.Combo("/tags", withOrg, ready, bind(models.GraphiteTags{})).Get(s.graphiteTags).Post(s.graphiteTags)
	r.Combo("/tags/:tag([0-9a-zA-Z]+)", withOrg, ready, bind(models.GraphiteTagDetails{})).Get(s.graphiteTagDetails).Post(s.graphiteTagDetails)
	r.Combo("/tags/findSeries", withOrg, ready, bind(models.GraphiteTagFindSeries{})).Get(s.graphiteTagFindSeries).Post(s.graphiteTagFindSeries)
}
<commit_msg>move endpoints to /metrics<commit_after>package api

import (
	"github.com/go-macaron/binding"
	"github.com/grafana/metrictank/api/middleware"
	"github.com/grafana/metrictank/api/models"
	"github.com/raintank/gziper"
	"gopkg.in/macaron.v1"
)

func (s *Server) RegisterRoutes() {
	r := s.Macaron
	if useGzip {
		r.Use(gziper.Gziper())
	}
	r.Use(middleware.RequestStats())
	r.Use(middleware.Tracer(s.Tracer))
	r.Use(macaron.Renderer())
	r.Use(middleware.OrgMiddleware(multiTenant))
	r.Use(middleware.CorsHandler())

	bind := binding.Bind
	withOrg := middleware.RequireOrg()
	cBody := middleware.CaptureBody
	ready := middleware.NodeReady()

	r.Get("/", s.appStatus)
	r.Get("/node", s.getNodeStatus)
	r.Post("/node", bind(models.NodeStatus{}), s.setNodeStatus)
	r.Get("/debug/pprof/block", blockHandler)
	r.Get("/debug/pprof/mutex", mutexHandler)

	r.Get("/cluster", s.getClusterStatus)
	r.Post("/cluster", bind(models.ClusterMembers{}), s.postClusterMembers)

	r.Combo("/getdata", ready, bind(models.GetData{})).Get(s.getData).Post(s.getData)

	r.Combo("/index/find", ready, bind(models.IndexFind{})).Get(s.indexFind).Post(s.indexFind)
	r.Combo("/index/list", ready, bind(models.IndexList{})).Get(s.indexList).Post(s.indexList)
	r.Combo("/index/delete", ready, bind(models.IndexDelete{})).Get(s.indexDelete).Post(s.indexDelete)
	r.Combo("/index/get", ready, bind(models.IndexGet{})).Get(s.indexGet).Post(s.indexGet)
	r.Combo("/index/tags", ready, bind(models.IndexTags{})).Get(s.indexTags).Post(s.indexTags)
	r.Combo("/index/tags/findSeries", ready, bind(models.IndexTagFindSeries{})).Get(s.indexTagFindSeries).Post(s.indexTagFindSeries)
	r.Combo("/index/tags/:tag([0-9a-zA-Z]+)", ready, bind(models.IndexTagDetails{})).Get(s.indexTagDetails).Post(s.indexTagDetails)

	r.Options("/*", func(ctx *macaron.Context) {
		ctx.Write(nil)
	})

	// Graphite endpoints
	r.Combo("/render", cBody, withOrg, ready, bind(models.GraphiteRender{})).Get(s.renderMetrics).Post(s.renderMetrics)
	r.Combo("/metrics/find", withOrg, ready, bind(models.GraphiteFind{})).Get(s.metricsFind).Post(s.metricsFind)
	r.Get("/metrics/index.json", withOrg, ready, s.metricsIndex)
	r.Post("/metrics/delete", withOrg, ready, bind(models.MetricsDelete{}), s.metricsDelete)
	r.Combo("/metrics/tags", withOrg, ready, bind(models.GraphiteTags{})).Get(s.graphiteTags).Post(s.graphiteTags)
	r.Combo("/metrics/tags/:tag([0-9a-zA-Z]+)", withOrg, ready, bind(models.GraphiteTagDetails{})).Get(s.graphiteTagDetails).Post(s.graphiteTagDetails)
	r.Combo("/metrics/tags/findSeries", withOrg, ready, bind(models.GraphiteTagFindSeries{})).Get(s.graphiteTagFindSeries).Post(s.graphiteTagFindSeries)
}
<|endoftext|>


<commit_before>package cmdr

import (
	"flag"
	"sort"
)

type (
	// Commands is a mapping of strings (the command name) to commands.
	Commands map[string]Command
	// Command is a command that can be invoked.
	Command interface {
		// Help is the help message for a command. To be a command, all you need
		// is a help message.
		Help() string
	}
	// CanExecute means the command can itself be executed to do something.
	Executor interface {
		Execute(args []string) Result
	}
	// HasSubcommands means this command has subcommands.
	Subcommander interface {
		// Subcommands returns a map of command names to Commands.
		Subcommands() Commands
	}
	// AddsFlags means this command has flags to add. These flags will be
	// available not only to this command, but will still be read, and have
	// impact on this command, even if the user applies them to deeper
	// subcommands.
	AddsFlags interface {
		// AddFlags will be passed a flag.FlagSet already named correctly for
		// this command. All you need to do is add whatever flag definitions
		// apply to this command.
		AddFlags(*flag.FlagSet)
	}
)

func (cs Commands) SortedKeys() []string {
	keys := make([]string, len(cs))
	i := 0
	for k := range cs {
		keys[i] = k
		i++
	}
	sort.Strings(keys)
	return keys
}
<commit_msg>Partially restored description of cmdr help strings.<commit_after>package cmdr

import (
	"flag"
	"sort"
)

type (
	// Commands is a mapping of strings (the command name) to commands.
	Commands map[string]Command
	// Command is a command that can be invoked.
	Command interface {
		// Help is the help message for a command. To be a command, all you need
		// is a help message.
		// Help messages must follow this convention:
		// The first line must be 50 characters or fewer, and describe
		// succinctly what the command does.
		// The second line must be blank.
		// The remainder of the string is free-form text.
		Help() string
	}
	// CanExecute means the command can itself be executed to do something.
	Executor interface {
		Execute(args []string) Result
	}
	// HasSubcommands means this command has subcommands.
	Subcommander interface {
		// Subcommands returns a map of command names to Commands.
		Subcommands() Commands
	}
	// AddsFlags means this command has flags to add. These flags will be
	// available not only to this command, but will still be read, and have
	// impact on this command, even if the user applies them to deeper
	// subcommands.
	AddsFlags interface {
		// AddFlags will be passed a flag.FlagSet already named correctly for
		// this command. All you need to do is add whatever flag definitions
		// apply to this command.
		AddFlags(*flag.FlagSet)
	}
)

func (cs Commands) SortedKeys() []string {
	keys := make([]string, len(cs))
	i := 0
	for k := range cs {
		keys[i] = k
		i++
	}
	sort.Strings(keys)
	return keys
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strconv"

	"github.com/icza/gowut/gwu"
)

type account struct {
	Id       int    `json:"id,omitempty"`
	Username string `json:"username"`
	Name     string `json:"name"`
	Email    string `json:"email"`
}

type accounts []account

type myBtnDelete struct {
	id     int
	parent gwu.Panel
	panel  gwu.Panel
}

func (h *myBtnDelete) HandleEvent(e gwu.Event) {
	if _, isButton := e.Src().(gwu.Button); isButton {

		fmt.Println("Delete called")
		fmt.Println("Id:", h.id)
		fmt.Println("Parent:", h.parent.Id().String())
		fmt.Println("Panel:", h.panel.Id().String())

		//TODO Call Add REST API

		h.parent.Remove(h.panel)

		e.MarkDirty(h.parent)
	}
}

type myBtnAdd struct {
	usernames []gwu.TextBox
	names     []gwu.TextBox
	emails    []gwu.TextBox
}

func (h *myBtnAdd) HandleEvent(e gwu.Event) {
	if _, isButton := e.Src().(gwu.Button); isButton {

		fmt.Println("Add called")

		for i := 0; i < 3; i++ {
			fmt.Println("Username: " + h.usernames[i].Text())
			fmt.Println("Name: " + h.names[i].Text())
			fmt.Println("Email: " + h.emails[i].Text())

			if len(h.usernames[i].Text()) == 0 ||
				len(h.names[i].Text()) == 0 ||
				len(h.emails[i].Text()) == 0 {
				continue
			}

			//TODO Call Add REST API

			h.usernames[i].SetText("")
			h.names[i].SetText("")
			h.emails[i].SetText("")

			e.MarkDirty(h.usernames[i])
			e.MarkDirty(h.names[i])
			e.MarkDirty(h.emails[i])
		}
	}
}

func refresh(address string, port int, parent gwu.Panel, panels []gwu.Panel) {
	for i := 0; i < len(panels); i++ {
		parent.Remove(panels[i])
	}

	url := "http://" + address + ":" + strconv.Itoa(port) + "/user"
	fmt.Println("URL:>", url)

	req, err := http.NewRequest("GET", url, nil)
	//req.Header.Set("X-Custom-Header", "myvalue")
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(io.LimitReader(resp.Body, 1048576))
	if err != nil {
		panic(err)
	}
	if err := resp.Body.Close(); err != nil {
		panic(err)
	}

	fmt.Println("response Status:", resp.Status)
	fmt.Println("response Headers:", resp.Header)
	fmt.Println("response Body:", string(body))

	var accts accounts
	err = json.Unmarshal(body, &accts)

	for i := 0; i < len(accts); i++ {
		fmt.Println("Id:", accts[i].Id)
		fmt.Println("Username:", accts[i].Username)
		fmt.Println("Name:", accts[i].Name)
		fmt.Println("Email:", accts[i].Email)

		p := gwu.NewHorizontalPanel()
		p.SetCellPadding(2)
		p.Add(gwu.NewLabel("Username: " + accts[i].Username))
		p.Add(gwu.NewLabel("Name: " + accts[i].Name))
		p.Add(gwu.NewLabel("Email: " + accts[i].Email))

		btndelete := gwu.NewButton("Delete")
		btndelete.SetAttr("align", "center")
		btndelete.AddEHandler(&myBtnDelete{accts[i].Id, parent, p}, gwu.ETypeClick)
		p.Add(btndelete)

		fmt.Println("Panel Id:", p.Id().String())

		panels = append(panels, p)

		parent.Add(p)
	}
}

type myBtnRefresh struct {
	address string
	port    int
	parent  gwu.Panel
	panels  []gwu.Panel
}

func (h *myBtnRefresh) HandleEvent(e gwu.Event) {
	if _, isButton := e.Src().(gwu.Button); isButton {

		fmt.Println("Refresh called")
		fmt.Println("address:", h.address)
		fmt.Println("port:", h.port)
		fmt.Println("parent:", h.parent.Id().String())
		for i := 0; i < len(h.panels); i++ {
			fmt.Println("panel:", h.panels[i].Id().String())
		}

		refresh(h.address, h.port, h.parent, h.panels)

		e.MarkDirty(h.parent)
	}
}

func main() {
	//define flags
	var port int
	flag.IntVar(&port, "port", 9000, "the REST server in which to bind to")
	var address string
	flag.StringVar(&address, "address", "127.0.0.1", "the REST server in which to bind to")
	//parse
	flag.Parse()

	// Create and build a window
	win := gwu.NewWindow("", "3-Tier App Demo")
	win.Style().SetFullWidth()
	win.SetHAlign(gwu.HACenter)
	win.SetCellPadding(2)

	//Add users
	addform := gwu.NewVerticalPanel()
	addform.Style().SetBorder2(1, gwu.BrdStyleSolid, gwu.ClrBlack)
	addform.SetCellPadding(10)
	addform.SetAttr("width", "400")
	addform.Add(gwu.NewLabel("Add New Accounts"))

	var usernames []gwu.TextBox
	var names []gwu.TextBox
	var emails []gwu.TextBox

	for i := 0; i < 3; i++ {
		p := gwu.NewHorizontalPanel()
		p.SetCellPadding(2)
		p.Add(gwu.NewLabel("Username:"))
		tbusername := gwu.NewTextBox("")
		p.Add(tbusername)
		p.Add(gwu.NewLabel("Name:"))
		tbname := gwu.NewTextBox("")
		p.Add(tbname)
		p.Add(gwu.NewLabel("Email:"))
		tbemail := gwu.NewTextBox("")
		p.Add(tbemail)

		usernames = append(usernames, tbusername)
		names = append(names, tbname)
		emails = append(emails, tbemail)

		addform.Add(p)
	}

	btnadd := gwu.NewButton("Add")
	btnadd.SetAttr("align", "center")
	btnadd.AddEHandler(&myBtnAdd{usernames, names, emails}, gwu.ETypeClick)
	addform.Add(btnadd)

	win.Add(addform)

	//Display users...
	acctlist := gwu.NewVerticalPanel()
	acctlist.Style().SetBorder2(1, gwu.BrdStyleSolid, gwu.ClrBlack)
	acctlist.SetCellPadding(10)
	acctlist.SetAttr("width", "400")

	var panels []gwu.Panel

	title2 := gwu.NewHorizontalPanel()
	title2.SetCellPadding(2)
	title2.Add(gwu.NewLabel("Current List of Accounts"))
	btnrefresh := gwu.NewButton("Refresh")
	btnrefresh.SetAttr("align", "center")
	btnrefresh.AddEHandler(&myBtnRefresh{address, port, acctlist, panels}, gwu.ETypeClick)
	title2.Add(btnrefresh)

	acctlist.Add(title2)

	//get current list of accounts
	refresh(address, port, acctlist, panels)

	win.Add(acctlist)

	// Create and start a GUI server (omitting error check)
	server := gwu.NewServer("", "127.0.0.1:8000")
	server.SetText("Test GUI App")
	server.AddWin(win)
	server.Start("") // Also opens windows list in browser
}
<commit_msg>Reorganize src and add a width<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strconv"

	"github.com/icza/gowut/gwu"
)

type account struct {
	Id       int    `json:"id,omitempty"`
	Username string `json:"username"`
	Name     string `json:"name"`
	Email    string `json:"email"`
}

type accounts []account

func adduserform() gwu.Panel {
	addform := gwu.NewVerticalPanel()
	addform.Style().SetBorder2(1, gwu.BrdStyleSolid, gwu.ClrBlack)
	addform.SetCellPadding(10)
	addform.SetAttr("width", "800")
	addform.Add(gwu.NewLabel("Add New Accounts"))

	var usernames []gwu.TextBox
	var names []gwu.TextBox
	var emails []gwu.TextBox

	for i := 0; i < 3; i++ {
		p := gwu.NewHorizontalPanel()
		p.SetCellPadding(2)
		p.Add(gwu.NewLabel("Username:"))
		tbusername := gwu.NewTextBox("")
		p.Add(tbusername)
		p.Add(gwu.NewLabel("Name:"))
		tbname := gwu.NewTextBox("")
		p.Add(tbname)
		p.Add(gwu.NewLabel("Email:"))
		tbemail := gwu.NewTextBox("")
		p.Add(tbemail)

		usernames = append(usernames, tbusername)
		names = append(names, tbname)
		emails = append(emails, tbemail)

		addform.Add(p)
	}

	btnadd := gwu.NewButton("Add")
	btnadd.SetAttr("align", "center")
	btnadd.AddEHandler(&myBtnAdd{usernames, names, emails}, gwu.ETypeClick)
	addform.Add(btnadd)

	return addform
}

type myBtnDelete struct {
	id     int
	parent gwu.Panel
	panel  gwu.Panel
}

func (h *myBtnDelete) HandleEvent(e gwu.Event) {
	if _, isButton := e.Src().(gwu.Button); isButton {

		fmt.Println("Delete called")
		fmt.Println("Id:", h.id)
		fmt.Println("Parent:", h.parent.Id().String())
		fmt.Println("Panel:", h.panel.Id().String())

		//TODO Call Add REST API

		h.parent.Remove(h.panel)

		e.MarkDirty(h.parent)
	}
}

type myBtnAdd struct {
	usernames []gwu.TextBox
	names     []gwu.TextBox
	emails    []gwu.TextBox
}

func (h *myBtnAdd) HandleEvent(e gwu.Event) {
	if _, isButton := e.Src().(gwu.Button); isButton {

		fmt.Println("Add called")

		for i := 0; i < 3; i++ {
			fmt.Println("Username: " + h.usernames[i].Text())
			fmt.Println("Name: " + h.names[i].Text())
			fmt.Println("Email: " + h.emails[i].Text())

			if len(h.usernames[i].Text()) == 0 ||
				len(h.names[i].Text()) == 0 ||
				len(h.emails[i].Text()) == 0 {
				continue
			}

			//TODO Call Add REST API

			h.usernames[i].SetText("")
			h.names[i].SetText("")
			h.emails[i].SetText("")

			e.MarkDirty(h.usernames[i])
			e.MarkDirty(h.names[i])
			e.MarkDirty(h.emails[i])
		}
	}
}

func refresh(address string, port int, parent gwu.Panel, panels []gwu.Panel) {
	for i := 0; i < len(panels); i++ {
		parent.Remove(panels[i])
	}

	url := "http://" + address + ":" + strconv.Itoa(port) + "/user"
	fmt.Println("URL:>", url)

	req, err := http.NewRequest("GET", url, nil)
	//req.Header.Set("X-Custom-Header", "myvalue")
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(io.LimitReader(resp.Body, 1048576))
	if err != nil {
		panic(err)
	}
	if err := resp.Body.Close(); err != nil {
		panic(err)
	}

	fmt.Println("response Status:", resp.Status)
	fmt.Println("response Headers:", resp.Header)
	fmt.Println("response Body:", string(body))

	var accts accounts
	err = json.Unmarshal(body, &accts)

	for i := 0; i < len(accts); i++ {
		fmt.Println("Id:", accts[i].Id)
		fmt.Println("Username:", accts[i].Username)
		fmt.Println("Name:", accts[i].Name)
		fmt.Println("Email:", accts[i].Email)

		p := gwu.NewHorizontalPanel()
		p.SetCellPadding(2)
		p.Add(gwu.NewLabel("Username: " + accts[i].Username))
		p.Add(gwu.NewLabel("Name: " + accts[i].Name))
		p.Add(gwu.NewLabel("Email: " + accts[i].Email))

		btndelete := gwu.NewButton("Delete")
		btndelete.SetAttr("align", "center")
		btndelete.AddEHandler(&myBtnDelete{accts[i].Id, parent, p}, gwu.ETypeClick)
		p.Add(btndelete)

		fmt.Println("Panel Id:", p.Id().String())

		panels = append(panels, p)

		parent.Add(p)
	}
}

type myBtnRefresh struct {
	address string
	port    int
	parent  gwu.Panel
	panels  []gwu.Panel
}

func (h *myBtnRefresh) HandleEvent(e gwu.Event) {
	if _, isButton := e.Src().(gwu.Button); isButton {

		fmt.Println("Refresh called")
		fmt.Println("address:", h.address)
		fmt.Println("port:", h.port)
		fmt.Println("parent:", h.parent.Id().String())
		for i := 0; i < len(h.panels); i++ {
			fmt.Println("panel:", h.panels[i].Id().String())
		}

		refresh(h.address, h.port, h.parent, h.panels)

		e.MarkDirty(h.parent)
	}
}

func listuserform(address string, port int) gwu.Panel {
	acctlist := gwu.NewVerticalPanel()
	acctlist.Style().SetBorder2(1, gwu.BrdStyleSolid, gwu.ClrBlack)
	acctlist.SetCellPadding(10)
	acctlist.SetAttr("width", "800")

	var panels []gwu.Panel

	title2 := gwu.NewHorizontalPanel()
	title2.SetCellPadding(2)
	title2.Add(gwu.NewLabel("Current List of Accounts"))
	btnrefresh := gwu.NewButton("Refresh")
	btnrefresh.SetAttr("align", "center")
	btnrefresh.AddEHandler(&myBtnRefresh{address, port, acctlist, panels}, gwu.ETypeClick)
	title2.Add(btnrefresh)

	acctlist.Add(title2)

	//get current list of accounts
	refresh(address, port, acctlist, panels)

	return acctlist
}

func main() {
	//define flags
	var port int
	flag.IntVar(&port, "port", 9000, "the REST server in which to bind to")
	var address string
	flag.StringVar(&address, "address", "127.0.0.1", "the REST server in which to bind to")
	//parse
	flag.Parse()

	// Create and build a window
	win := gwu.NewWindow("main", "3-Tier App Demo")
	win.Style().SetFullWidth()
	win.SetHAlign(gwu.HACenter)
	win.SetCellPadding(2)

	//Add users
	win.Add(adduserform())

	//Display users...
	win.Add(listuserform(address, port))

	// Create and start a GUI server (omitting error check)
	server := gwu.NewServer("", "127.0.0.1:8000")
	server.SetText("Test GUI App")
	server.AddWin(win)
	server.Start("") // Also opens windows list in browser
}
<|endoftext|>


<commit_before>package susigo

import (
    "crypto/tls"
    "encoding/json"
    "errors"
    "log"
    "net"
    "strconv"
    "time"
)

type Callback func(map[string]interface{})

type Susi struct {
    conn       net.Conn
    encoder    *json.Encoder
    decoder    *json.Decoder
    callbacks  map[string]Callback
    consumers  map[string]map[int64]Callback
    processors map[string]map[int64]Callback
}

func NewSusi(addr, certFile, keyFile string) (*Susi, error) {
    susi := new(Susi)
    cert, err := tls.LoadX509KeyPair(certFile, keyFile)
    if err != nil {
        return nil, err
    }
    conn, err := tls.Dial("tcp", addr, &tls.Config{
        Certificates:       []tls.Certificate{cert},
        InsecureSkipVerify: true,
    })
    if err != nil {
        return nil, err
    }
    susi.conn = conn
    susi.callbacks = make(map[string]Callback)
    susi.consumers = make(map[string]map[int64]Callback)
    susi.processors = make(map[string]map[int64]Callback)
    susi.encoder = json.NewEncoder(susi.conn)
    susi.decoder = json.NewDecoder(susi.conn)
    go susi.backend()
    return susi, nil
}

func (susi *Susi) Publish(topic string, payload interface{}, callback Callback) error {
    id := strconv.FormatInt(time.Now().UnixNano(), 10)
    packet := map[string]interface{}{
        "type": "publish",
        "data": map[string]interface{}{
            "topic":   topic,
            "payload": payload,
            "id":      id,
        },
    }
    susi.callbacks[id] = callback
    return susi.encoder.Encode(packet)
}

func (susi *Susi) RegisterConsumer(topic string, callback Callback) (int64, error) {
    if susi.consumers[topic] == nil {
        susi.consumers[topic] = make(map[int64]Callback)
    }
    consumers := susi.consumers[topic]
    id := time.Now().UnixNano()
    consumers[id] = callback
    if len(consumers) == 1 {
        packet := map[string]interface{}{
            "type": "registerConsumer",
            "data": map[string]interface{}{
                "topic": topic,
            },
        }
        return id, susi.encoder.Encode(packet)
    }
    return -1, nil
}

func (susi *Susi) RegisterProcessor(topic string, callback Callback) (int64, error) {
    if susi.processors[topic] == nil {
        susi.processors[topic] = make(map[int64]Callback)
    }
    processors := susi.processors[topic]
    id := time.Now().UnixNano()
    processors[id] = callback
    if len(processors) == 1 {
        packet := map[string]interface{}{
            "type": "registerProcessor",
            "data": map[string]interface{}{
                "topic": topic,
            },
        }
        return id, susi.encoder.Encode(packet)
    }
    return -1, nil
}

func (susi *Susi) UnregisterConsumer(id int64) error {
    for _, consumers := range susi.consumers {
        if topic, ok := consumers[id]; ok {
            delete(consumers, id)
            if len(consumers) == 0 {
                packet := map[string]interface{}{
                    "type": "unregisterConsumer",
                    "data": map[string]interface{}{
                        "topic": topic,
                    },
                }
                return susi.encoder.Encode(packet)
            } else {
                return nil
            }
        }
    }
    return errors.New("no such consumer")
}

func (susi *Susi) UnregisterProcessor(id int64) error {
    for _, processors := range susi.processors {
        if topic, ok := processors[id]; ok {
            delete(processors, id)
            if len(processors) == 0 {
                packet := map[string]interface{}{
                    "type": "unregisterProcessor",
                    "data": map[string]interface{}{
                        "topic": topic,
                    },
                }
                return susi.encoder.Encode(packet)
            } else {
                return nil
            }
        }
    }
    return errors.New("no such processor")
}

func (susi *Susi) backend() {
    packet := make(map[string]interface{})
    for {
        err := susi.decoder.Decode(&packet)
        if err != nil {
            log.Println("Error reading from susi json decoder: ", err)
            continue
        }
        switch packet["type"] {
        case "ack":
            {
                event := packet["data"].(map[string]interface{})
                id := event["id"].(string)
                callback := susi.callbacks[id]
                callback(event)
                delete(susi.callbacks, id)
            }
        case "consumerEvent":
            {
                event := packet["data"].(map[string]interface{})
                topic := event["topic"].(string)
                consumers := susi.consumers[topic]
                for _, consumer := range consumers {
                    consumer(event)
                }
            }
        case "processorEvent":
            {
                event := packet["data"].(map[string]interface{})
                topic := event["topic"].(string)
                processors := susi.processors[topic]
                for _, processor := range processors {
                    processor(event)
                }
                packet["type"] = "ack"
                packet["data"] = event
                if err = susi.encoder.Encode(packet); err != nil {
                    log.Println("can not ack event back to susi: ", err)
                }
            }
        }
    }
}
<commit_msg>added Event datatype;<commit_after>package susigo

import (
    "crypto/tls"
    "encoding/json"
    "errors"
    "log"
    "net"
    "strconv"
    "time"
)

type Callback func(map[string]interface{})

type Susi struct {
    conn       net.Conn
    encoder    *json.Encoder
    decoder    *json.Decoder
    callbacks  map[string]Callback
    consumers  map[string]map[int64]Callback
    processors map[string]map[int64]Callback
}

type Event struct {
    Topic     string              `json:"topic"`
    Payload   interface{}         `json:"payload"`
    Headers   []map[string]string `json:"headers"`
    Id        string              `json:"id"`
    SessionId string              `json:"sessionid"`
}

func NewSusi(addr, certFile, keyFile string) (*Susi, error) {
    susi := new(Susi)
    cert, err := tls.LoadX509KeyPair(certFile, keyFile)
    if err != nil {
        return nil, err
    }
    conn, err := tls.Dial("tcp", addr, &tls.Config{
        Certificates:       []tls.Certificate{cert},
        InsecureSkipVerify: true,
    })
    if err != nil {
        return nil, err
    }
    susi.conn = conn
    susi.callbacks = make(map[string]Callback)
    susi.consumers = make(map[string]map[int64]Callback)
    susi.processors = make(map[string]map[int64]Callback)
    susi.encoder = json.NewEncoder(susi.conn)
    susi.decoder = json.NewDecoder(susi.conn)
    go susi.backend()
    return susi, nil
}

func (susi *Susi) Publish(event Event, callback Callback) error {
    id := strconv.FormatInt(time.Now().UnixNano(), 10)
    event.Id = id
    packet := map[string]interface{}{
        "type": "publish",
        "data": event,
    }
    susi.callbacks[id] = callback
    return susi.encoder.Encode(packet)
}

func (susi *Susi) RegisterConsumer(topic string, callback Callback) (int64, error) {
    if susi.consumers[topic] == nil {
        susi.consumers[topic] = make(map[int64]Callback)
    }
    consumers := susi.consumers[topic]
    id := time.Now().UnixNano()
    consumers[id] = callback
    if len(consumers) == 1 {
        packet := map[string]interface{}{
            "type": "registerConsumer",
            "data": map[string]interface{}{
                "topic": topic,
            },
        }
        return id, susi.encoder.Encode(packet)
    }
    return -1, nil
}

func (susi *Susi) RegisterProcessor(topic string, callback Callback) (int64, error) {
    if susi.processors[topic] == nil {
        susi.processors[topic] = make(map[int64]Callback)
    }
    processors := susi.processors[topic]
    id := time.Now().UnixNano()
    processors[id] = callback
    if len(processors) == 1 {
        packet := map[string]interface{}{
            "type": "registerProcessor",
            "data": map[string]interface{}{
                "topic": topic,
            },
        }
        return id, susi.encoder.Encode(packet)
    }
    return -1, nil
}

func (susi *Susi) UnregisterConsumer(id int64) error {
    for _, consumers := range susi.consumers {
        if topic, ok := consumers[id]; ok {
            delete(consumers, id)
            if len(consumers) == 0 {
                packet := map[string]interface{}{
                    "type": "unregisterConsumer",
                    "data": map[string]interface{}{
                        "topic": topic,
                    },
                }
                return susi.encoder.Encode(packet)
            } else {
                return nil
            }
        }
    }
    return errors.New("no such consumer")
}

func (susi *Susi) UnregisterProcessor(id int64) error {
    for _, processors := range susi.processors {
        if topic, ok := processors[id]; ok {
            delete(processors, id)
            if len(processors) == 0 {
                packet := map[string]interface{}{
                    "type": "unregisterProcessor",
                    "data": map[string]interface{}{
                        "topic": topic,
                    },
                }
                return susi.encoder.Encode(packet)
            } else {
                return nil
            }
        }
    }
    return errors.New("no such processor")
}

func (susi *Susi) backend() {
    packet := make(map[string]interface{})
    for {
        err := susi.decoder.Decode(&packet)
        if err != nil {
            log.Println("Error reading from susi json decoder: ", err)
            continue
        }
        switch packet["type"] {
        case "ack":
            {
                event := packet["data"].(map[string]interface{})
                id := event["id"].(string)
                callback := susi.callbacks[id]
                callback(event)
                delete(susi.callbacks, id)
            }
        case "consumerEvent":
            {
                event := packet["data"].(map[string]interface{})
                topic := event["topic"].(string)
                consumers := susi.consumers[topic]
                for _, consumer := range consumers {
                    consumer(event)
                }
            }
        case "processorEvent":
            {
                event := packet["data"].(map[string]interface{})
                topic := event["topic"].(string)
                processors := susi.processors[topic]
                for _, processor := range processors {
                    processor(event)
                }
                packet["type"] = "ack"
                packet["data"] = event
                if err = susi.encoder.Encode(packet); err != nil {
                    log.Println("can not ack event back to susi: ", err)
                }
            }
        }
    }
}
<|endoftext|>


<commit_before>// Copyright 2011 Numrotron Inc.
// Use of this source code is governed by an MIT-style license
// that can be found in the LICENSE file.
//
// Developed at www.stathat.com by Patrick Crosby
// Contact us on twitter with any questions:  twitter.com/stat_hat

// amzses is a Go package to send emails using Amazon's Simple Email Service.

package amzses

import (
	"os"
	"crypto/hmac"
	"encoding/base64"
	"fmt"
	"github.com/stathat/jconfig"
	"io/ioutil"
	"http"
	"url"
	"time"
)

const (
	endpoint = "https://email.us-east-1.amazonaws.com"
)

var accessKey, secretKey string

type AmazonResponse struct {
	MessageId string `xml:"SendEmailResult>MessageId>"`
	RequestId string `xml:"ResponseMetadata>RequestId>"`
}

func init() {
	config := jconfig.LoadConfig("aws.conf")
	accessKey = config.GetString("aws_access_key")
	secretKey = config.GetString("aws_secret_key")
}

func SendMail(from, to, subject, text, html string) (string, os.Error) {
	data := make(url.Values)
	data.Add("Action", "SendEmail")
	data.Add("Source", from)
	data.Add("Destination.ToAddresses.member.1", to)
	data.Add("Message.Subject.Data", subject)
	data.Add("Message.Body.Text.Data", text)
	data.Add("Message.Body.Html.Data", html)
	return sesGet(data)
}

func VerifyEmail(email string) (string, os.Error) {
    data := make(url.Values)
    data.Add("Action", "VerifyEmailAddress")
    data.Add("EmailAddress", email)
    return sesGet(data)
}

func DeleteEmail(email string) (string, os.Error){
    data := make(url.Values)
    data.Add("Action", "DeleteVerifiedEmailAddress")
    data.Add("EmailAddress", email)
    return sesGet(data)
}

func GetSendQuota() (string, os.Error){
    data := make(url.Values)
    data.Add("Action", "GetSendQuota")
    return sesGet(data)
}

func GetSendStatistics() (string, os.Error){
    data := make(url.Values)
    data.Add("Action", "GetSendStatistics")
    return sesGet(data)
}

func ListVerifiedEmail() (string, os.Error){
    data := make(url.Values)
    data.Add("Action", "ListVerifiedEmailAddresses")
    return sesGet(data)
}

func authorizationHeader(date string) []string {
	h := hmac.NewSHA256([]uint8(secretKey))
	h.Write([]uint8(date))
	signature := base64.StdEncoding.EncodeToString(h.Sum())
	auth := fmt.Sprintf("AWS3-HTTPS AWSAccessKeyId=%s, Algorithm=HmacSHA256, Signature=%s", accessKey, signature)
	return []string{auth}
}

func sesGet(data url.Values) (string, os.Error) {
	data.Add("AWSAccessKeyId", accessKey)
	urlstr := fmt.Sprintf("%s?%s", endpoint, data.Encode())
	endpointURL, _ := url.Parse(urlstr)
	headers := map[string][]string{}

	now := time.UTC()
	date := now.Format("Mon, 02 Jan 2006 15:04:05 -0700")
	headers["Date"] = []string{date}

	h := hmac.NewSHA256([]uint8(secretKey))
	h.Write([]uint8(date))
	signature := base64.StdEncoding.EncodeToString(h.Sum())
	auth := fmt.Sprintf("AWS3-HTTPS AWSAccessKeyId=%s, Algorithm=HmacSHA256, Signature=%s", accessKey, signature)
	headers["X-Amzn-Authorization"] = []string{auth}

	req := http.Request{
		URL:        endpointURL,
		Method:     "GET",
		ProtoMajor: 1,
		ProtoMinor: 1,
		Close:      true,
		Header:     headers,
	}

	r, err := http.DefaultClient.Do(&req)
	if err != nil {
		return "", err
	}

	resultbody, _ := ioutil.ReadAll(r.Body)
	r.Body.Close()

	if r.StatusCode != 200 {
		return "", os.NewError(string(resultbody))
	}

	return string(resultbody), nil
}

<commit_msg>gofmt<commit_after>// Copyright 2011 Numrotron Inc.
// Use of this source code is governed by an MIT-style license
// that can be found in the LICENSE file.
//
// Developed at www.stathat.com by Patrick Crosby
// Contact us on twitter with any questions:  twitter.com/stat_hat

// amzses is a Go package to send emails using Amazon's Simple Email Service.

package amzses

import (
	"os"
	"crypto/hmac"
	"encoding/base64"
	"fmt"
	"github.com/stathat/jconfig"
	"io/ioutil"
	"http"
	"url"
	"time"
)

const (
	endpoint = "https://email.us-east-1.amazonaws.com"
)

var accessKey, secretKey string

type AmazonResponse struct {
	MessageId string `xml:"SendEmailResult>MessageId>"`
	RequestId string `xml:"ResponseMetadata>RequestId>"`
}

func init() {
	config := jconfig.LoadConfig("aws.conf")
	accessKey = config.GetString("aws_access_key")
	secretKey = config.GetString("aws_secret_key")
}

func SendMail(from, to, subject, text, html string) (string, os.Error) {
	data := make(url.Values)
	data.Add("Action", "SendEmail")
	data.Add("Source", from)
	data.Add("Destination.ToAddresses.member.1", to)
	data.Add("Message.Subject.Data", subject)
	data.Add("Message.Body.Text.Data", text)
	data.Add("Message.Body.Html.Data", html)
	return sesGet(data)
}

func VerifyEmail(email string) (string, os.Error) {
	data := make(url.Values)
	data.Add("Action", "VerifyEmailAddress")
	data.Add("EmailAddress", email)
	return sesGet(data)
}

func DeleteEmail(email string) (string, os.Error) {
	data := make(url.Values)
	data.Add("Action", "DeleteVerifiedEmailAddress")
	data.Add("EmailAddress", email)
	return sesGet(data)
}

func GetSendQuota() (string, os.Error) {
	data := make(url.Values)
	data.Add("Action", "GetSendQuota")
	return sesGet(data)
}

func GetSendStatistics() (string, os.Error) {
	data := make(url.Values)
	data.Add("Action", "GetSendStatistics")
	return sesGet(data)
}

func ListVerifiedEmail() (string, os.Error) {
	data := make(url.Values)
	data.Add("Action", "ListVerifiedEmailAddresses")
	return sesGet(data)
}

func authorizationHeader(date string) []string {
	h := hmac.NewSHA256([]uint8(secretKey))
	h.Write([]uint8(date))
	signature := base64.StdEncoding.EncodeToString(h.Sum())
	auth := fmt.Sprintf("AWS3-HTTPS AWSAccessKeyId=%s, Algorithm=HmacSHA256, Signature=%s", accessKey, signature)
	return []string{auth}
}

func sesGet(data url.Values) (string, os.Error) {
	data.Add("AWSAccessKeyId", accessKey)
	urlstr := fmt.Sprintf("%s?%s", endpoint, data.Encode())
	endpointURL, _ := url.Parse(urlstr)
	headers := map[string][]string{}

	now := time.UTC()
	date := now.Format("Mon, 02 Jan 2006 15:04:05 -0700")
	headers["Date"] = []string{date}

	h := hmac.NewSHA256([]uint8(secretKey))
	h.Write([]uint8(date))
	signature := base64.StdEncoding.EncodeToString(h.Sum())
	auth := fmt.Sprintf("AWS3-HTTPS AWSAccessKeyId=%s, Algorithm=HmacSHA256, Signature=%s", accessKey, signature)
	headers["X-Amzn-Authorization"] = []string{auth}

	req := http.Request{
		URL:        endpointURL,
		Method:     "GET",
		ProtoMajor: 1,
		ProtoMinor: 1,
		Close:      true,
		Header:     headers,
	}

	r, err := http.DefaultClient.Do(&req)
	if err != nil {
		return "", err
	}

	resultbody, _ := ioutil.ReadAll(r.Body)
	r.Body.Close()

	if r.StatusCode != 200 {
		return "", os.NewError(string(resultbody))
	}

	return string(resultbody), nil
}
<|endoftext|>


<commit_before>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package e2e

import (
	"fmt"
	"os"
	"strings"
	"testing"
	"time"

	"github.com/coreos/etcd/pkg/flags"
	"github.com/coreos/etcd/pkg/testutil"
	"github.com/coreos/etcd/version"
)

func TestCtlV3Version(t *testing.T) { testCtl(t, versionTest) }

func versionTest(cx ctlCtx) {
	if err := ctlV3Version(cx); err != nil {
		cx.t.Fatalf("versionTest ctlV3Version error (%v)", err)
	}
}

func ctlV3Version(cx ctlCtx) error {
	cmdArgs := append(cx.PrefixArgs(), "version")
	return spawnWithExpect(cmdArgs, version.Version)
}

// TestCtlV3DialWithHTTPScheme ensures that client handles endpoints with HTTPS scheme.
func TestCtlV3DialWithHTTPScheme(t *testing.T) {
	testCtl(t, dialWithSchemeTest, withCfg(configClientTLS))
}

func dialWithSchemeTest(cx ctlCtx) {
	cmdArgs := append(cx.prefixArgs(cx.epc.endpoints()), "put", "foo", "bar")
	if err := spawnWithExpect(cmdArgs, "OK"); err != nil {
		cx.t.Fatal(err)
	}
}

type ctlCtx struct {
	t                 *testing.T
	cfg               etcdProcessClusterConfig
	quotaBackendBytes int64
	noStrictReconfig  bool

	epc *etcdProcessCluster

	envMap map[string]struct{}

	dialTimeout time.Duration

	quorum      bool // if true, set up 3-node cluster and linearizable read
	interactive bool

	user string
	pass string

	// for compaction
	compactPhysical bool
}

type ctlOption func(*ctlCtx)

func (cx *ctlCtx) applyOpts(opts []ctlOption) {
	for _, opt := range opts {
		opt(cx)
	}
}

func withCfg(cfg etcdProcessClusterConfig) ctlOption {
	return func(cx *ctlCtx) { cx.cfg = cfg }
}

func withDialTimeout(timeout time.Duration) ctlOption {
	return func(cx *ctlCtx) { cx.dialTimeout = timeout }
}

func withQuorum() ctlOption {
	return func(cx *ctlCtx) { cx.quorum = true }
}

func withInteractive() ctlOption {
	return func(cx *ctlCtx) { cx.interactive = true }
}

func withQuota(b int64) ctlOption {
	return func(cx *ctlCtx) { cx.quotaBackendBytes = b }
}

func withCompactPhysical() ctlOption {
	return func(cx *ctlCtx) { cx.compactPhysical = true }
}

func withNoStrictReconfig() ctlOption {
	return func(cx *ctlCtx) { cx.noStrictReconfig = true }
}

func withFlagByEnv() ctlOption {
	return func(cx *ctlCtx) { cx.envMap = make(map[string]struct{}) }
}

func testCtl(t *testing.T, testFunc func(ctlCtx), opts ...ctlOption) {
	defer testutil.AfterTest(t)

	ret := ctlCtx{
		t:           t,
		cfg:         configAutoTLS,
		dialTimeout: 7 * time.Second,
	}
	ret.applyOpts(opts)

	os.Setenv("ETCDCTL_API", "3")
	mustEtcdctl(t)
	if !ret.quorum {
		ret.cfg = *configStandalone(ret.cfg)
	}
	if ret.quotaBackendBytes > 0 {
		ret.cfg.quotaBackendBytes = ret.quotaBackendBytes
	}
	ret.cfg.noStrictReconfig = ret.noStrictReconfig

	epc, err := newEtcdProcessCluster(&ret.cfg)
	if err != nil {
		t.Fatalf("could not start etcd process cluster (%v)", err)
	}
	ret.epc = epc

	defer func() {
		os.Unsetenv("ETCDCTL_API")
		if ret.envMap != nil {
			for k := range ret.envMap {
				os.Unsetenv(k)
			}
		}
		if errC := ret.epc.Close(); errC != nil {
			t.Fatalf("error closing etcd processes (%v)", errC)
		}
	}()

	donec := make(chan struct{})
	go func() {
		defer close(donec)
		testFunc(ret)
	}()

	timeout := 2*ret.dialTimeout + time.Second
	if ret.dialTimeout == 0 {
		timeout = 30 * time.Second
	}
	select {
	case <-time.After(timeout):
		t.Fatalf("test timed out after %v", timeout)
	case <-donec:
	}
}

func (cx *ctlCtx) prefixArgs(eps []string) []string {
	if len(cx.epc.proxies()) > 0 { // TODO: add proxy check as in v2
		panic("v3 proxy not implemented")
	}

	fmap := make(map[string]string)
	fmap["endpoints"] = strings.Join(eps, ",")
	fmap["dial-timeout"] = cx.dialTimeout.String()
	if cx.epc.cfg.clientTLS == clientTLS {
		if cx.epc.cfg.isClientAutoTLS {
			fmap["insecure-transport"] = "false"
			fmap["insecure-skip-tls-verify"] = "true"
		} else {
			fmap["cacert"] = caPath
			fmap["cert"] = certPath
			fmap["key"] = privateKeyPath
		}
	}
	if cx.user != "" {
		fmap["user"] = cx.user + ":" + cx.pass
	}

	useEnv := cx.envMap != nil

	cmdArgs := []string{ctlBinPath}
	for k, v := range fmap {
		if useEnv {
			ek := flags.FlagToEnv("ETCDCTL", k)
			os.Setenv(ek, v)
			cx.envMap[ek] = struct{}{}
		} else {
			cmdArgs = append(cmdArgs, fmt.Sprintf("--%s=%s", k, v))
		}
	}
	return cmdArgs
}

// PrefixArgs prefixes etcdctl command.
// Make sure to unset environment variables after tests.
func (cx *ctlCtx) PrefixArgs() []string {
	return cx.prefixArgs(cx.epc.grpcEndpoints())
}

func isGRPCTimedout(err error) bool {
	return strings.Contains(err.Error(), "grpc: timed out trying to connect")
}
<commit_msg>e2e: dump stack on ctlTest timeout<commit_after>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package e2e

import (
	"fmt"
	"os"
	"strings"
	"testing"
	"time"

	"github.com/coreos/etcd/pkg/flags"
	"github.com/coreos/etcd/pkg/testutil"
	"github.com/coreos/etcd/version"
)

func TestCtlV3Version(t *testing.T) { testCtl(t, versionTest) }

func versionTest(cx ctlCtx) {
	if err := ctlV3Version(cx); err != nil {
		cx.t.Fatalf("versionTest ctlV3Version error (%v)", err)
	}
}

func ctlV3Version(cx ctlCtx) error {
	cmdArgs := append(cx.PrefixArgs(), "version")
	return spawnWithExpect(cmdArgs, version.Version)
}

// TestCtlV3DialWithHTTPScheme ensures that client handles endpoints with HTTPS scheme.
func TestCtlV3DialWithHTTPScheme(t *testing.T) {
	testCtl(t, dialWithSchemeTest, withCfg(configClientTLS))
}

func dialWithSchemeTest(cx ctlCtx) {
	cmdArgs := append(cx.prefixArgs(cx.epc.endpoints()), "put", "foo", "bar")
	if err := spawnWithExpect(cmdArgs, "OK"); err != nil {
		cx.t.Fatal(err)
	}
}

type ctlCtx struct {
	t                 *testing.T
	cfg               etcdProcessClusterConfig
	quotaBackendBytes int64
	noStrictReconfig  bool

	epc *etcdProcessCluster

	envMap map[string]struct{}

	dialTimeout time.Duration

	quorum      bool // if true, set up 3-node cluster and linearizable read
	interactive bool

	user string
	pass string

	// for compaction
	compactPhysical bool
}

type ctlOption func(*ctlCtx)

func (cx *ctlCtx) applyOpts(opts []ctlOption) {
	for _, opt := range opts {
		opt(cx)
	}
}

func withCfg(cfg etcdProcessClusterConfig) ctlOption {
	return func(cx *ctlCtx) { cx.cfg = cfg }
}

func withDialTimeout(timeout time.Duration) ctlOption {
	return func(cx *ctlCtx) { cx.dialTimeout = timeout }
}

func withQuorum() ctlOption {
	return func(cx *ctlCtx) { cx.quorum = true }
}

func withInteractive() ctlOption {
	return func(cx *ctlCtx) { cx.interactive = true }
}

func withQuota(b int64) ctlOption {
	return func(cx *ctlCtx) { cx.quotaBackendBytes = b }
}

func withCompactPhysical() ctlOption {
	return func(cx *ctlCtx) { cx.compactPhysical = true }
}

func withNoStrictReconfig() ctlOption {
	return func(cx *ctlCtx) { cx.noStrictReconfig = true }
}

func withFlagByEnv() ctlOption {
	return func(cx *ctlCtx) { cx.envMap = make(map[string]struct{}) }
}

func testCtl(t *testing.T, testFunc func(ctlCtx), opts ...ctlOption) {
	defer testutil.AfterTest(t)

	ret := ctlCtx{
		t:           t,
		cfg:         configAutoTLS,
		dialTimeout: 7 * time.Second,
	}
	ret.applyOpts(opts)

	os.Setenv("ETCDCTL_API", "3")
	mustEtcdctl(t)
	if !ret.quorum {
		ret.cfg = *configStandalone(ret.cfg)
	}
	if ret.quotaBackendBytes > 0 {
		ret.cfg.quotaBackendBytes = ret.quotaBackendBytes
	}
	ret.cfg.noStrictReconfig = ret.noStrictReconfig

	epc, err := newEtcdProcessCluster(&ret.cfg)
	if err != nil {
		t.Fatalf("could not start etcd process cluster (%v)", err)
	}
	ret.epc = epc

	defer func() {
		os.Unsetenv("ETCDCTL_API")
		if ret.envMap != nil {
			for k := range ret.envMap {
				os.Unsetenv(k)
			}
		}
		if errC := ret.epc.Close(); errC != nil {
			t.Fatalf("error closing etcd processes (%v)", errC)
		}
	}()

	donec := make(chan struct{})
	go func() {
		defer close(donec)
		testFunc(ret)
	}()

	timeout := 2*ret.dialTimeout + time.Second
	if ret.dialTimeout == 0 {
		timeout = 30 * time.Second
	}
	select {
	case <-time.After(timeout):
		testutil.FatalStack(t, fmt.Sprintf("test timed out after %v", timeout))
	case <-donec:
	}
}

func (cx *ctlCtx) prefixArgs(eps []string) []string {
	if len(cx.epc.proxies()) > 0 { // TODO: add proxy check as in v2
		panic("v3 proxy not implemented")
	}

	fmap := make(map[string]string)
	fmap["endpoints"] = strings.Join(eps, ",")
	fmap["dial-timeout"] = cx.dialTimeout.String()
	if cx.epc.cfg.clientTLS == clientTLS {
		if cx.epc.cfg.isClientAutoTLS {
			fmap["insecure-transport"] = "false"
			fmap["insecure-skip-tls-verify"] = "true"
		} else {
			fmap["cacert"] = caPath
			fmap["cert"] = certPath
			fmap["key"] = privateKeyPath
		}
	}
	if cx.user != "" {
		fmap["user"] = cx.user + ":" + cx.pass
	}

	useEnv := cx.envMap != nil

	cmdArgs := []string{ctlBinPath}
	for k, v := range fmap {
		if useEnv {
			ek := flags.FlagToEnv("ETCDCTL", k)
			os.Setenv(ek, v)
			cx.envMap[ek] = struct{}{}
		} else {
			cmdArgs = append(cmdArgs, fmt.Sprintf("--%s=%s", k, v))
		}
	}
	return cmdArgs
}

// PrefixArgs prefixes etcdctl command.
// Make sure to unset environment variables after tests.
func (cx *ctlCtx) PrefixArgs() []string {
	return cx.prefixArgs(cx.epc.grpcEndpoints())
}

func isGRPCTimedout(err error) bool {
	return strings.Contains(err.Error(), "grpc: timed out trying to connect")
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package model

import (
	"fmt"
	"io/ioutil"
	"regexp"

	"k8s.io/klog"
	"k8s.io/kops/nodeup/pkg/distros"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
)

// NTPBuilder installs and starts NTP, to ensure accurate clock times.
// As well as general log confusion, clock-skew of more than 5 minutes
// causes AWS API calls to fail
type NTPBuilder struct {
	*NodeupModelContext
}

var _ fi.ModelBuilder = &NTPBuilder{}

type ntpDaemon string

var (
	chronyd ntpDaemon = "chronyd"
	ntpd    ntpDaemon = "ntpd"
)

// Build is responsible for configuring NTP
func (b *NTPBuilder) Build(c *fi.ModelBuilderContext) error {
	switch b.Distribution {
	case distros.DistributionContainerOS:
		klog.Infof("Detected ContainerOS; won't install ntp")
		return nil
	case distros.DistributionCoreOS:
		klog.Infof("Detected CoreOS; won't install ntp")
		return nil
	case distros.DistributionFlatcar:
		klog.Infof("Detected Flatcar; won't install ntp")
		return nil
	}

	var ntpIP string
	switch b.Cluster.Spec.CloudProvider {
	case "aws":
		ntpIP = "169.254.169.123"
	case "gce":
		ntpIP = "time.google.com"
	default:
		ntpIP = ""
	}

	if b.Distribution.IsDebianFamily() {
		c.AddTask(&nodetasks.Package{Name: "ntp"})

		if ntpIP != "" {
			bytes, err := updateNtpIP(ntpIP, ntpd)
			if err != nil {
				return err
			}
			c.AddTask(&nodetasks.File{
				Path:     "/etc/ntp.conf",
				Contents: fi.NewBytesResource(bytes),
				Type:     nodetasks.FileType_File,
				Mode:     s("0644"),
			})
		}

		c.AddTask((&nodetasks.Service{Name: "ntp"}).InitDefaults())
	} else if b.Distribution.IsRHELFamily() {
		switch b.Distribution {
		case distros.DistributionCentos8, distros.DistributionRhel8:
			c.AddTask(&nodetasks.Package{Name: "chrony"})

			if ntpIP != "" {
				bytes, err := updateNtpIP(ntpIP, chronyd)
				if err != nil {
					return err
				}
				c.AddTask(&nodetasks.File{
					Path:     "/etc/chrony.conf",
					Contents: fi.NewBytesResource(bytes),
					Type:     nodetasks.FileType_File,
					Mode:     s("0644"),
				})
			}
			c.AddTask((&nodetasks.Service{Name: "chronyd"}).InitDefaults())

		default:
			c.AddTask(&nodetasks.Package{Name: "ntp"})

			if ntpIP != "" {
				bytes, err := updateNtpIP(ntpIP, ntpd)
				if err != nil {
					return err
				}
				c.AddTask(&nodetasks.File{
					Path:     "/etc/ntp.conf",
					Contents: fi.NewBytesResource(bytes),
					Type:     nodetasks.FileType_File,
					Mode:     s("0644"),
				})
			}

			c.AddTask((&nodetasks.Service{Name: "ntpd"}).InitDefaults())
		}
	} else {
		klog.Warningf("unknown distribution, skipping ntp install: %v", b.Distribution)
		return nil
	}
	return nil
}

// updateNtpIP takes a ip and a ntpDaemon and will comment out
// the default server or pool values and append the correct cloud
// ip to the ntp config file.
func updateNtpIP(ip string, daemon ntpDaemon) ([]byte, error) {
	var address string
	var r *regexp.Regexp
	var path string
	switch daemon {
	case ntpd:
		address = fmt.Sprintf("server %s prefer iburst", ip)
		// the regex strings might need a bit more work
		r = regexp.MustCompile(`pool\s\d.*[a-z].[a-z].[a-z]\siburst`)
		path = "/etc/ntp.conf"
	case chronyd:
		address = fmt.Sprintf("server %s prefer iburst minpoll 4 maxpoll 4", ip)
		// the regex strings might need a bit more work
		r = regexp.MustCompile(`server\s.*iburst.*`)
		path = "/etc/chrony.conf"
	default:
		return nil, fmt.Errorf("%s is not a supported ntp application", ntpd)
	}

	f, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}

	new := r.ReplaceAllFunc(f, func(b []byte) []byte {
		return []byte(fmt.Sprintf("#commented out by kops %s", string(b)))
	})
	new = append(new, []byte(address)...)
	return new, nil
}
<commit_msg>adjusted regex<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package model

import (
	"fmt"
	"io/ioutil"
	"regexp"

	"k8s.io/klog"
	"k8s.io/kops/nodeup/pkg/distros"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
)

// NTPBuilder installs and starts NTP, to ensure accurate clock times.
// As well as general log confusion, clock-skew of more than 5 minutes
// causes AWS API calls to fail
type NTPBuilder struct {
	*NodeupModelContext
}

var _ fi.ModelBuilder = &NTPBuilder{}

type ntpDaemon string

var (
	chronyd ntpDaemon = "chronyd"
	ntpd    ntpDaemon = "ntpd"
)

// Build is responsible for configuring NTP
func (b *NTPBuilder) Build(c *fi.ModelBuilderContext) error {
	switch b.Distribution {
	case distros.DistributionContainerOS:
		klog.Infof("Detected ContainerOS; won't install ntp")
		return nil
	case distros.DistributionCoreOS:
		klog.Infof("Detected CoreOS; won't install ntp")
		return nil
	case distros.DistributionFlatcar:
		klog.Infof("Detected Flatcar; won't install ntp")
		return nil
	}

	var ntpIP string
	switch b.Cluster.Spec.CloudProvider {
	case "aws":
		ntpIP = "169.254.169.123"
	case "gce":
		ntpIP = "time.google.com"
	default:
		ntpIP = ""
	}

	if b.Distribution.IsDebianFamily() {
		c.AddTask(&nodetasks.Package{Name: "ntp"})

		if ntpIP != "" {
			bytes, err := updateNtpIP(ntpIP, ntpd)
			if err != nil {
				return err
			}
			c.AddTask(&nodetasks.File{
				Path:     "/etc/ntp.conf",
				Contents: fi.NewBytesResource(bytes),
				Type:     nodetasks.FileType_File,
				Mode:     s("0644"),
			})
		}

		c.AddTask((&nodetasks.Service{Name: "ntp"}).InitDefaults())
	} else if b.Distribution.IsRHELFamily() {
		switch b.Distribution {
		case distros.DistributionCentos8, distros.DistributionRhel8:
			c.AddTask(&nodetasks.Package{Name: "chrony"})

			if ntpIP != "" {
				bytes, err := updateNtpIP(ntpIP, chronyd)
				if err != nil {
					return err
				}
				c.AddTask(&nodetasks.File{
					Path:     "/etc/chrony.conf",
					Contents: fi.NewBytesResource(bytes),
					Type:     nodetasks.FileType_File,
					Mode:     s("0644"),
				})
			}
			c.AddTask((&nodetasks.Service{Name: "chronyd"}).InitDefaults())

		default:
			c.AddTask(&nodetasks.Package{Name: "ntp"})

			if ntpIP != "" {
				bytes, err := updateNtpIP(ntpIP, ntpd)
				if err != nil {
					return err
				}
				c.AddTask(&nodetasks.File{
					Path:     "/etc/ntp.conf",
					Contents: fi.NewBytesResource(bytes),
					Type:     nodetasks.FileType_File,
					Mode:     s("0644"),
				})
			}

			c.AddTask((&nodetasks.Service{Name: "ntpd"}).InitDefaults())
		}
	} else {
		klog.Warningf("unknown distribution, skipping ntp install: %v", b.Distribution)
		return nil
	}
	return nil
}

// updateNtpIP takes a ip and a ntpDaemon and will comment out
// the default server or pool values and append the correct cloud
// ip to the ntp config file.
func updateNtpIP(ip string, daemon ntpDaemon) ([]byte, error) {
	var address string
	var path string
	r := regexp.MustCompile(`(?m)(^pool|^server)\s.*`)
	switch daemon {
	case ntpd:
		address = fmt.Sprintf("server %s prefer iburst\n", ip)
		path = "/etc/ntp.conf"
	case chronyd:
		address = fmt.Sprintf("server %s prefer iburst minpoll 4 maxpoll 4\n", ip)
		path = "/etc/chrony.conf"
	default:
		return nil, fmt.Errorf("%s is not a supported ntp application", daemon)
	}

	f, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}

	new := r.ReplaceAllFunc(f, func(b []byte) []byte {
		return []byte(fmt.Sprintf("#commented out by kops %s", string(b)))
	})
	new = append(new, []byte(address)...)
	return new, nil
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"log"
	"os"
	"testing"

	"github.com/xogeny/impact/crawl"
	"github.com/xogeny/impact/index"

	. "github.com/smartystreets/goconvey/convey"
	. "github.com/xogeny/xconvey"
)

func TestGitHub(t *testing.T) {
	Convey("Testing GitHub crawler", t, func(c C) {
		logger := log.New(os.Stdout, "impact: ", 0)
		ind := index.NewIndex()

		cr, err := crawl.MakeGitHubCrawler("modelica-3rdparty", "", "")
		NoError(c, err)
		err = cr.Crawl(ind, true, logger)
		NoError(c, err)

		cr, err = crawl.MakeGitHubCrawler("modelica", "", "")
		NoError(c, err)
		err = cr.Crawl(ind, false, logger)
		NoError(c, err)

		str, err := ind.JSON()
		NoError(c, err)
		ioutil.WriteFile("gh_crawl.json", []byte(str), os.ModePerm)
	})
}
<commit_msg>Turned off debugging output<commit_after>package main

import (
	"io/ioutil"
	"log"
	"os"
	"testing"

	"github.com/xogeny/impact/crawl"
	"github.com/xogeny/impact/index"

	. "github.com/smartystreets/goconvey/convey"
	. "github.com/xogeny/xconvey"
)

func TestGitHub(t *testing.T) {
	Convey("Testing GitHub crawler", t, func(c C) {
		logger := log.New(os.Stdout, "impact: ", 0)
		ind := index.NewIndex()

		cr, err := crawl.MakeGitHubCrawler("modelica-3rdparty", "", "")
		NoError(c, err)
		err = cr.Crawl(ind, false, logger)
		NoError(c, err)

		cr, err = crawl.MakeGitHubCrawler("modelica", "", "")
		NoError(c, err)
		err = cr.Crawl(ind, false, logger)
		NoError(c, err)

		str, err := ind.JSON()
		NoError(c, err)
		ioutil.WriteFile("gh_crawl.json", []byte(str), os.ModePerm)
	})
}
<|endoftext|>


<commit_before>package fs

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"strings"

	"github.com/pivotal-golang/lager"
)

type BtrfsFilesystem struct {
	imagePath string
	mountPath string

	logger lager.Logger
}

func New(logger lager.Logger, imagePath string, mountPath string) *BtrfsFilesystem {
	return &BtrfsFilesystem{
		imagePath: imagePath,
		mountPath: mountPath,
		logger:    logger,
	}
}

// lower your expectations
func (fs *BtrfsFilesystem) Create(bytes uint64) error {
	kiloBytes := bytes / 1024
	bytes = kiloBytes * 1024

	// significantly
	idempotent := exec.Command("bash", "-e", "-x", "-c", `
		if [ ! -e $IMAGE_PATH ] || [ "$(stat --printf="%s" $IMAGE_PATH)" != "$SIZE_IN_BYTES" ]; then
			dd if=/dev/zero of=$IMAGE_PATH bs=1024 count=$SIZE_IN_KILOBYTES
		fi

		lo=""
		if [ -z "$(losetup -j $IMAGE_PATH)" ]; then
			lo="$(losetup -f --show $IMAGE_PATH)"
		fi

		if file $IMAGE_PATH | grep -v BTRFS; then
			mkfs.btrfs $lo
		fi

		mkdir -p $MOUNT_PATH

		if ! mountpoint -q $MOUNT_PATH; then
			mount $lo $MOUNT_PATH
		fi
	`)

	idempotent.Env = []string{
		"PATH=" + os.Getenv("PATH"),
		"MOUNT_PATH=" + fs.mountPath,
		"IMAGE_PATH=" + fs.imagePath,
		fmt.Sprintf("SIZE_IN_BYTES=%d", bytes),
		fmt.Sprintf("SIZE_IN_KILOBYTES=%d", kiloBytes),
	}

	_, err := fs.run(idempotent)
	return err
}

func (fs *BtrfsFilesystem) Delete() error {
	_, err := fs.run(exec.Command(
		"umount",
		fs.mountPath,
	))
	if err != nil {
		return err
	}

	if err := os.RemoveAll(fs.mountPath); err != nil {
		return err
	}

	loopbackOutput, err := fs.run(exec.Command(
		"losetup",
		"-j",
		fs.imagePath,
	))
	if err != nil {
		return err
	}

	loopbackDevice := strings.Split(loopbackOutput, ":")[0]

	_, err = fs.run(exec.Command(
		"losetup",
		"-d",
		loopbackDevice,
	))
	if err != nil {
		return err
	}

	return os.Remove(fs.imagePath)
}

func (fs *BtrfsFilesystem) run(cmd *exec.Cmd) (string, error) {
	logger := fs.logger.Session("run-command", lager.Data{
		"command": cmd.Path,
		"args":    cmd.Args,
		"env":     cmd.Env,
	})

	stdout := &bytes.Buffer{}
	stderr := &bytes.Buffer{}

	cmd.Stdout = stdout
	cmd.Stderr = stderr

	err := cmd.Run()

	loggerData := lager.Data{
		"stdout": stdout.String(),
		"stderr": stderr.String(),
	}

	if err != nil {
		logger.Error("failed", err, loggerData)
		return "", err
	}

	logger.Debug("ran", loggerData)

	return stdout.String(), nil
}
<commit_msg>do not perform TRIM on mkfs<commit_after>package fs

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"strings"

	"github.com/pivotal-golang/lager"
)

type BtrfsFilesystem struct {
	imagePath string
	mountPath string

	logger lager.Logger
}

func New(logger lager.Logger, imagePath string, mountPath string) *BtrfsFilesystem {
	return &BtrfsFilesystem{
		imagePath: imagePath,
		mountPath: mountPath,
		logger:    logger,
	}
}

// lower your expectations
func (fs *BtrfsFilesystem) Create(bytes uint64) error {
	kiloBytes := bytes / 1024
	bytes = kiloBytes * 1024

	// significantly
	idempotent := exec.Command("bash", "-e", "-x", "-c", `
		if [ ! -e $IMAGE_PATH ] || [ "$(stat --printf="%s" $IMAGE_PATH)" != "$SIZE_IN_BYTES" ]; then
			dd if=/dev/zero of=$IMAGE_PATH bs=1024 count=$SIZE_IN_KILOBYTES
		fi

		lo=""
		if [ -z "$(losetup -j $IMAGE_PATH)" ]; then
			lo="$(losetup -f --show $IMAGE_PATH)"
		fi

		if file $IMAGE_PATH | grep -v BTRFS; then
			mkfs.btrfs --nodiscard $lo
		fi

		mkdir -p $MOUNT_PATH

		if ! mountpoint -q $MOUNT_PATH; then
			mount $lo $MOUNT_PATH
		fi
	`)

	idempotent.Env = []string{
		"PATH=" + os.Getenv("PATH"),
		"MOUNT_PATH=" + fs.mountPath,
		"IMAGE_PATH=" + fs.imagePath,
		fmt.Sprintf("SIZE_IN_BYTES=%d", bytes),
		fmt.Sprintf("SIZE_IN_KILOBYTES=%d", kiloBytes),
	}

	_, err := fs.run(idempotent)
	return err
}

func (fs *BtrfsFilesystem) Delete() error {
	_, err := fs.run(exec.Command(
		"umount",
		fs.mountPath,
	))
	if err != nil {
		return err
	}

	if err := os.RemoveAll(fs.mountPath); err != nil {
		return err
	}

	loopbackOutput, err := fs.run(exec.Command(
		"losetup",
		"-j",
		fs.imagePath,
	))
	if err != nil {
		return err
	}

	loopbackDevice := strings.Split(loopbackOutput, ":")[0]

	_, err = fs.run(exec.Command(
		"losetup",
		"-d",
		loopbackDevice,
	))
	if err != nil {
		return err
	}

	return os.Remove(fs.imagePath)
}

func (fs *BtrfsFilesystem) run(cmd *exec.Cmd) (string, error) {
	logger := fs.logger.Session("run-command", lager.Data{
		"command": cmd.Path,
		"args":    cmd.Args,
		"env":     cmd.Env,
	})

	stdout := &bytes.Buffer{}
	stderr := &bytes.Buffer{}

	cmd.Stdout = stdout
	cmd.Stderr = stderr

	err := cmd.Run()

	loggerData := lager.Data{
		"stdout": stdout.String(),
		"stderr": stderr.String(),
	}

	if err != nil {
		logger.Error("failed", err, loggerData)
		return "", err
	}

	logger.Debug("ran", loggerData)

	return stdout.String(), nil
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/sha256"
	"database/sql"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	_ "github.com/mattn/go-sqlite3"
)

// TODO: change these to be cmd args instead!
const DB_FIL = "/home/lmas/projects/ss13_se/src/db.sqlite3"

// Dir to save new graphs in
const SAVE_DIR = "/home/lmas/projects/ss13_se/src/static/graphs"

// How far back in time the graphs will go
var LAST_WEEK = time.Now().AddDate(0, 0, -7)
var LAST_MONTH = time.Now().AddDate(0, -1, 0)

var WEEK_DAYS = [7]string{
	"Sunday",
	"Monday",
	"Tuesday",
	"Wednesday",
	"Thursday",
	"Friday",
	"Saturday",
}

func main() {
	// open a db connection
	db, err := sql.Open("sqlite3", DB_FIL)
	checkerror(err)
	defer db.Close()

	// loop over each server in db
	rows, err := db.Query("select id, title from gameservers_server")
	checkerror(err)
	defer rows.Close()

	var (
		id    int
		title string
	)
	for rows.Next() {
		err := rows.Scan(&id, &title)
		checkerror(err)
		weeklyhistorygraph(db, id, title)
		monthlyhistorygraph(db, id, title)
		monthlyaveragedaygraph(db, id, title)
	}
	err = rows.Err()
	checkerror(err)
}

func checkerror(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func setuptemppaths(prefix string, title string) (f *os.File, name string, path string) {
	// create a tmp file
	file, err := ioutil.TempFile("", prefix)
	checkerror(err)

	// Make sure we have somewhere to save the stored graphs in
	err = os.MkdirAll(SAVE_DIR, 0777)
	checkerror(err)
	hash := fmt.Sprintf("%x", sha256.Sum256([]byte(title)))
	path = filepath.Join(SAVE_DIR, fmt.Sprintf("%s%s", prefix, hash))

	return file, file.Name(), path
}

func weeklyhistorygraph(db *sql.DB, id int, title string) {
	prefix := "week-time-"
	// Create a new temp file, get it's filepath and get the final storage path
	ifile, ifilename, ofilename := setuptemppaths(prefix, title)
	defer ifile.Close()

	// Rows for server, newer then LAST_WEEK and only on the hour
	rows, err := db.Query("select created,players "+
		"from gameservers_serverhistory "+
		"where server_id = ? and created >= ? and strftime('%M', created) = '00' "+
		"order by created asc",
		id, LAST_WEEK)
	checkerror(err)
	defer rows.Close()

	var (
		created time.Time
		players int
	)
	// Scan in each row and write it to the tmp file
	for rows.Next() {
		err := rows.Scan(&created, &players)
		checkerror(err)
		_, err = ifile.WriteString(fmt.Sprintf("%d, %d\n", created.Unix(), players))
		checkerror(err)
	}
	err = rows.Err()
	checkerror(err)

	// run the plotter against the tmp file
	err = exec.Command("./plot_time.sh", ifilename, ofilename).Run()
	checkerror(err)

	// close and remove the file
	ifile.Close()
	os.Remove(ifilename)
}

func monthlyhistorygraph(db *sql.DB, id int, title string) {
	prefix := "month-time-"
	ifile, ifilename, ofilename := setuptemppaths(prefix, title)
	defer ifile.Close()

	// Rows for server, newer then LAST_WEEK and only every 6th hour
	rows, err := db.Query("select created,players "+
		"from gameservers_serverhistory "+
		"where server_id = ? and created >= ? and strftime('%M', created) = '00' "+
		"and strftime('%H', created) in ('00', '06', '12', '18') "+
		"order by created asc",
		id, LAST_MONTH)
	checkerror(err)
	defer rows.Close()

	var (
		created time.Time
		players int
	)
	for rows.Next() {
		err := rows.Scan(&created, &players)
		checkerror(err)
		_, err = ifile.WriteString(fmt.Sprintf("%d, %d\n", created.Unix(), players))
		checkerror(err)
	}
	err = rows.Err()
	checkerror(err)

	err = exec.Command("./plot_time.sh", ifilename, ofilename).Run()
	checkerror(err)

	ifile.Close()
	os.Remove(ifilename)
}

func monthlyaveragedaygraph(db *sql.DB, id int, title string) {
	prefix := "month-avg_day-"
	ifile, ifilename, ofilename := setuptemppaths(prefix, title)
	defer ifile.Close()

	rows, err := db.Query("select "+
		"strftime('%w', created) as weekday, avg(players) "+
		"from gameservers_serverhistory "+
		"where server_id = ? and created >= ? "+
		"group by weekday;", id, LAST_MONTH)
	checkerror(err)
	defer rows.Close()

	var (
		day         int
		players     float64
		avg_players [7]float64
	)

	for rows.Next() {
		err := rows.Scan(&day, &players)
		checkerror(err)
		avg_players[day] = players
	}
	err = rows.Err()
	checkerror(err)

	// Write each day's average to the file
	for i := 1; i <= 6; i++ {
		_, err = ifile.WriteString(fmt.Sprintf("%s, %f\n", WEEK_DAYS[i], avg_players[i]))
		checkerror(err)
	}
	// Oh hey! Look at what I found! It's sunday!!
	_, err = ifile.WriteString(fmt.Sprintf("%s, %f\n", WEEK_DAYS[0], avg_players[0]))
	checkerror(err)
	// Fucking wankers and their stupid usage of sunday as the first day of week...

	err = exec.Command("./plot_bar.sh", ifilename, ofilename).Run()
	checkerror(err)

	ifile.Close()
	os.Remove(ifilename)
}
<commit_msg>Handle errors from external scripts run by update_graphs.go.<commit_after>package main

import (
	"crypto/sha256"
	"database/sql"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	_ "github.com/mattn/go-sqlite3"
)

// TODO: change these to be cmd args instead!
const DB_FIL = "/home/lmas/projects/ss13_se/src/db.sqlite3"

// Dir to save new graphs in
const SAVE_DIR = "/home/lmas/projects/ss13_se/src/static/graphs"

// How far back in time the graphs will go
var LAST_WEEK = time.Now().AddDate(0, 0, -7)
var LAST_MONTH = time.Now().AddDate(0, -1, 0)

var WEEK_DAYS = [7]string{
	"Sunday",
	"Monday",
	"Tuesday",
	"Wednesday",
	"Thursday",
	"Friday",
	"Saturday",
}

func main() {
	// open a db connection
	db, err := sql.Open("sqlite3", DB_FIL)
	checkerror(err)
	defer db.Close()

	// loop over each server in db
	rows, err := db.Query("select id, title from gameservers_server")
	checkerror(err)
	defer rows.Close()

	var (
		id    int
		title string
	)
	for rows.Next() {
		err := rows.Scan(&id, &title)
		checkerror(err)
		weeklyhistorygraph(db, id, title)
		monthlyhistorygraph(db, id, title)
		monthlyaveragedaygraph(db, id, title)
	}
	err = rows.Err()
	checkerror(err)
}

func checkerror(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func setuptemppaths(prefix string, title string) (f *os.File, name string, path string) {
	// create a tmp file
	file, err := ioutil.TempFile("", prefix)
	checkerror(err)

	// Make sure we have somewhere to save the stored graphs in
	err = os.MkdirAll(SAVE_DIR, 0777)
	checkerror(err)
	hash := fmt.Sprintf("%x", sha256.Sum256([]byte(title)))
	path = filepath.Join(SAVE_DIR, fmt.Sprintf("%s%s", prefix, hash))

	return file, file.Name(), path
}

func runcommand(cmd string, title string, args ...string) {
	out, err := exec.Command(cmd, args...).CombinedOutput()
	if err != nil && len(out) > 0 {
		log.Printf("ERROR running '%s' for '%s': \n%s\n", cmd, title, out)
	}
	checkerror(err)
}
func weeklyhistorygraph(db *sql.DB, id int, title string) {
	prefix := "week-time-"
	// Create a new temp file, get it's filepath and get the final storage path
	ifile, ifilename, ofilename := setuptemppaths(prefix, title)
	defer ifile.Close()

	// Rows for server, newer then LAST_WEEK and only on the hour
	rows, err := db.Query("select created,players "+
		"from gameservers_serverhistory "+
		"where server_id = ? and created >= ? and strftime('%M', created) = '00' "+
		"order by created asc",
		id, LAST_WEEK)
	checkerror(err)
	defer rows.Close()

	var (
		created time.Time
		players int
	)
	// Scan in each row and write it to the tmp file
	for rows.Next() {
		err := rows.Scan(&created, &players)
		checkerror(err)
		_, err = ifile.WriteString(fmt.Sprintf("%d, %d\n", created.Unix(), players))
		checkerror(err)
	}
	err = rows.Err()
	checkerror(err)

	// run the plotter against the tmp file
	runcommand("./plot_time.sh", title, ifilename, ofilename)

	// close and remove the file
	ifile.Close()
	os.Remove(ifilename)
}

func monthlyhistorygraph(db *sql.DB, id int, title string) {
	prefix := "month-time-"
	ifile, ifilename, ofilename := setuptemppaths(prefix, title)
	defer ifile.Close()

	// Rows for server, newer then LAST_WEEK and only every 6th hour
	rows, err := db.Query("select created,players "+
		"from gameservers_serverhistory "+
		"where server_id = ? and created >= ? and strftime('%M', created) = '00' "+
		"and strftime('%H', created) in ('00', '06', '12', '18') "+
		"order by created asc",
		id, LAST_MONTH)
	checkerror(err)
	defer rows.Close()

	var (
		created time.Time
		players int
	)
	for rows.Next() {
		err := rows.Scan(&created, &players)
		checkerror(err)
		_, err = ifile.WriteString(fmt.Sprintf("%d, %d\n", created.Unix(), players))
		checkerror(err)
	}
	err = rows.Err()
	checkerror(err)

	runcommand("./plot_time.sh", title, ifilename, ofilename)

	ifile.Close()
	os.Remove(ifilename)
}

func monthlyaveragedaygraph(db *sql.DB, id int, title string) {
	prefix := "month-avg_day-"
	ifile, ifilename, ofilename := setuptemppaths(prefix, title)
	defer ifile.Close()

	rows, err := db.Query("select "+
		"strftime('%w', created) as weekday, avg(players) "+
		"from gameservers_serverhistory "+
		"where server_id = ? and created >= ? "+
		"group by weekday;", id, LAST_MONTH)
	checkerror(err)
	defer rows.Close()

	var (
		day         int
		players     float64
		avg_players [7]float64
	)

	for rows.Next() {
		err := rows.Scan(&day, &players)
		checkerror(err)
		avg_players[day] = players
	}
	err = rows.Err()
	checkerror(err)

	// Write each day's average to the file
	for i := 1; i <= 6; i++ {
		_, err = ifile.WriteString(fmt.Sprintf("%s, %f\n", WEEK_DAYS[i], avg_players[i]))
		checkerror(err)
	}
	// Oh hey! Look at what I found! It's sunday!!
	_, err = ifile.WriteString(fmt.Sprintf("%s, %f\n", WEEK_DAYS[0], avg_players[0]))
	checkerror(err)
	// Fucking wankers and their stupid usage of sunday as the first day of week...

	runcommand("./plot_bar.sh", title, ifilename, ofilename)

	ifile.Close()
	os.Remove(ifilename)
}
<|endoftext|>