<commit_before>package libnetwork

import (
	"fmt"
	"strings"

	"github.com/docker/libnetwork/netlabel"
	"github.com/docker/libnetwork/types"
	"github.com/sirupsen/logrus"
)

const (
	gwEPlen = 12
)

var procGwNetwork = make(chan (bool), 1)

/*
   libnetwork creates a bridge network "docker_gw_bridge" for providing
   default gateway for the containers if none of the container's endpoints
   have GW set by the driver. ICC is set to false for the GW_bridge network.

   If a driver can't provide external connectivity it can choose to not set
   the GW IP for the endpoint.

   endpoint on the GW_bridge network is managed dynamically by libnetwork.
   ie:
   - its created when an endpoint without GW joins the container
   - its deleted when an endpoint with GW joins the container
*/

func (sb *sandbox) setupDefaultGW() error {

	// check if the container already has a GW endpoint
	if ep := sb.getEndpointInGWNetwork(); ep != nil {
		return nil
	}

	c := sb.controller

	// Look for default gw network. In case of error (includes not found),
	// retry and create it if needed in a serialized execution.
	n, err := c.NetworkByName(libnGWNetwork)
	if err != nil {
		if n, err = c.defaultGwNetwork(); err != nil {
			return err
		}
	}

	createOptions := []EndpointOption{CreateOptionAnonymous()}

	eplen := gwEPlen
	if len(sb.containerID) < gwEPlen {
		eplen = len(sb.containerID)
	}

	sbLabels := sb.Labels()

	if sbLabels[netlabel.PortMap] != nil {
		createOptions = append(createOptions, CreateOptionPortMapping(sbLabels[netlabel.PortMap].([]types.PortBinding)))
	}

	if sbLabels[netlabel.ExposedPorts] != nil {
		createOptions = append(createOptions, CreateOptionExposedPorts(sbLabels[netlabel.ExposedPorts].([]types.TransportPort)))
	}

	epOption := getPlatformOption()
	if epOption != nil {
		createOptions = append(createOptions, epOption)
	}

	newEp, err := n.CreateEndpoint("gateway_"+sb.containerID[0:eplen], createOptions...)
	if err != nil {
		return fmt.Errorf("container %s: endpoint create on GW Network failed: %v", sb.containerID, err)
	}

	defer func() {
		if err != nil {
			if err2 := newEp.Delete(true); err2 != nil {
				logrus.Warnf("Failed to remove gw endpoint for container %s after failing to join the gateway network: %v",
					sb.containerID, err2)
			}
		}
	}()

	epLocal := newEp.(*endpoint)

	if err = epLocal.sbJoin(sb); err != nil {
		return fmt.Errorf("container %s: endpoint join on GW Network failed: %v", sb.containerID, err)
	}

	return nil
}

// If present, detach and remove the endpoint connecting the sandbox to the default gw network.
func (sb *sandbox) clearDefaultGW() error {
	var ep *endpoint

	if ep = sb.getEndpointInGWNetwork(); ep == nil {
		return nil
	}
	if err := ep.sbLeave(sb, false); err != nil {
		return fmt.Errorf("container %s: endpoint leaving GW Network failed: %v", sb.containerID, err)
	}
	if err := ep.Delete(false); err != nil {
		return fmt.Errorf("container %s: deleting endpoint on GW Network failed: %v", sb.containerID, err)
	}
	return nil
}

// Evaluate whether the sandbox requires a default gateway based
// on the endpoints to which it is connected. It does not account
// for the default gateway network endpoint.

func (sb *sandbox) needDefaultGW() bool {
	var needGW bool

	for _, ep := range sb.getConnectedEndpoints() {
		if ep.endpointInGWNetwork() {
			continue
		}
		if ep.getNetwork().Type() == "null" || ep.getNetwork().Type() == "host" {
			continue
		}
		if ep.getNetwork().Internal() {
			continue
		}
		// During stale sandbox cleanup, joinInfo may be nil
		if ep.joinInfo != nil && ep.joinInfo.disableGatewayService {
			continue
		}
		// TODO v6 needs to be handled.
		if len(ep.Gateway()) > 0 {
			return false
		}
		for _, r := range ep.StaticRoutes() {
			if r.Destination != nil && r.Destination.String() == "0.0.0.0/0" {
				return false
			}
		}
		needGW = true
	}

	return needGW
}

func (sb *sandbox) getEndpointInGWNetwork() *endpoint {
	for _, ep := range sb.getConnectedEndpoints() {
		if ep.getNetwork().name == libnGWNetwork && strings.HasPrefix(ep.Name(), "gateway_") {
			return ep
		}
	}
	return nil
}

func (ep *endpoint) endpointInGWNetwork() bool {
	if ep.getNetwork().name == libnGWNetwork && strings.HasPrefix(ep.Name(), "gateway_") {
		return true
	}
	return false
}

func (sb *sandbox) getEPwithoutGateway() *endpoint {
	for _, ep := range sb.getConnectedEndpoints() {
		if ep.getNetwork().Type() == "null" || ep.getNetwork().Type() == "host" {
			continue
		}
		if len(ep.Gateway()) == 0 {
			return ep
		}
	}
	return nil
}

// Looks for the default gw network and creates it if not there.
// Parallel executions are serialized.
func (c *controller) defaultGwNetwork() (Network, error) {
	procGwNetwork <- true
	defer func() { <-procGwNetwork }()

	n, err := c.NetworkByName(libnGWNetwork)
	if err != nil {
		if _, ok := err.(types.NotFoundError); ok {
			n, err = c.createGWNetwork()
		}
	}
	return n, err
}

// Returns the endpoint which is providing external connectivity to the sandbox
func (sb *sandbox) getGatewayEndpoint() *endpoint {
	for _, ep := range sb.getConnectedEndpoints() {
		if ep.getNetwork().Type() == "null" || ep.getNetwork().Type() == "host" {
			continue
		}
		if len(ep.Gateway()) != 0 {
			return ep
		}
	}
	return nil
}
<commit_msg>Avoid default gateway collisions<commit_after>package libnetwork

import (
	"fmt"
	"strings"

	"github.com/docker/libnetwork/netlabel"
	"github.com/docker/libnetwork/types"
	"github.com/sirupsen/logrus"
)

const (
	gwEPlen = 12
)

var procGwNetwork = make(chan (bool), 1)

/*
   libnetwork creates a bridge network "docker_gw_bridge" for providing
   default gateway for the containers if none of the container's endpoints
   have GW set by the driver. ICC is set to false for the GW_bridge network.

   If a driver can't provide external connectivity it can choose to not set
   the GW IP for the endpoint.

   endpoint on the GW_bridge network is managed dynamically by libnetwork.
   ie:
   - its created when an endpoint without GW joins the container
   - its deleted when an endpoint with GW joins the container
*/

func (sb *sandbox) setupDefaultGW() error {

	// check if the container already has a GW endpoint
	if ep := sb.getEndpointInGWNetwork(); ep != nil {
		return nil
	}

	c := sb.controller

	// Look for default gw network. In case of error (includes not found),
	// retry and create it if needed in a serialized execution.
	n, err := c.NetworkByName(libnGWNetwork)
	if err != nil {
		if n, err = c.defaultGwNetwork(); err != nil {
			return err
		}
	}

	createOptions := []EndpointOption{CreateOptionAnonymous()}

	var gwName string
	if len(sb.containerID) <= gwEPlen {
		gwName = "gateway_" + sb.containerID
	} else {
		gwName = "gateway_" + sb.id[:gwEPlen]
	}

	sbLabels := sb.Labels()

	if sbLabels[netlabel.PortMap] != nil {
		createOptions = append(createOptions, CreateOptionPortMapping(sbLabels[netlabel.PortMap].([]types.PortBinding)))
	}

	if sbLabels[netlabel.ExposedPorts] != nil {
		createOptions = append(createOptions, CreateOptionExposedPorts(sbLabels[netlabel.ExposedPorts].([]types.TransportPort)))
	}

	epOption := getPlatformOption()
	if epOption != nil {
		createOptions = append(createOptions, epOption)
	}

	newEp, err := n.CreateEndpoint(gwName, createOptions...)
	if err != nil {
		return fmt.Errorf("container %s: endpoint create on GW Network failed: %v", sb.containerID, err)
	}

	defer func() {
		if err != nil {
			if err2 := newEp.Delete(true); err2 != nil {
				logrus.Warnf("Failed to remove gw endpoint for container %s after failing to join the gateway network: %v",
					sb.containerID, err2)
			}
		}
	}()

	epLocal := newEp.(*endpoint)

	if err = epLocal.sbJoin(sb); err != nil {
		return fmt.Errorf("container %s: endpoint join on GW Network failed: %v", sb.containerID, err)
	}

	return nil
}

// If present, detach and remove the endpoint connecting the sandbox to the default gw network.
func (sb *sandbox) clearDefaultGW() error {
	var ep *endpoint

	if ep = sb.getEndpointInGWNetwork(); ep == nil {
		return nil
	}
	if err := ep.sbLeave(sb, false); err != nil {
		return fmt.Errorf("container %s: endpoint leaving GW Network failed: %v", sb.containerID, err)
	}
	if err := ep.Delete(false); err != nil {
		return fmt.Errorf("container %s: deleting endpoint on GW Network failed: %v", sb.containerID, err)
	}
	return nil
}

// Evaluate whether the sandbox requires a default gateway based
// on the endpoints to which it is connected. It does not account
// for the default gateway network endpoint.

func (sb *sandbox) needDefaultGW() bool {
	var needGW bool

	for _, ep := range sb.getConnectedEndpoints() {
		if ep.endpointInGWNetwork() {
			continue
		}
		if ep.getNetwork().Type() == "null" || ep.getNetwork().Type() == "host" {
			continue
		}
		if ep.getNetwork().Internal() {
			continue
		}
		// During stale sandbox cleanup, joinInfo may be nil
		if ep.joinInfo != nil && ep.joinInfo.disableGatewayService {
			continue
		}
		// TODO v6 needs to be handled.
		if len(ep.Gateway()) > 0 {
			return false
		}
		for _, r := range ep.StaticRoutes() {
			if r.Destination != nil && r.Destination.String() == "0.0.0.0/0" {
				return false
			}
		}
		needGW = true
	}

	return needGW
}

func (sb *sandbox) getEndpointInGWNetwork() *endpoint {
	for _, ep := range sb.getConnectedEndpoints() {
		if ep.getNetwork().name == libnGWNetwork && strings.HasPrefix(ep.Name(), "gateway_") {
			return ep
		}
	}
	return nil
}

func (ep *endpoint) endpointInGWNetwork() bool {
	if ep.getNetwork().name == libnGWNetwork && strings.HasPrefix(ep.Name(), "gateway_") {
		return true
	}
	return false
}

func (sb *sandbox) getEPwithoutGateway() *endpoint {
	for _, ep := range sb.getConnectedEndpoints() {
		if ep.getNetwork().Type() == "null" || ep.getNetwork().Type() == "host" {
			continue
		}
		if len(ep.Gateway()) == 0 {
			return ep
		}
	}
	return nil
}

// Looks for the default gw network and creates it if not there.
// Parallel executions are serialized.
func (c *controller) defaultGwNetwork() (Network, error) {
	procGwNetwork <- true
	defer func() { <-procGwNetwork }()

	n, err := c.NetworkByName(libnGWNetwork)
	if err != nil {
		if _, ok := err.(types.NotFoundError); ok {
			n, err = c.createGWNetwork()
		}
	}
	return n, err
}

// Returns the endpoint which is providing external connectivity to the sandbox
func (sb *sandbox) getGatewayEndpoint() *endpoint {
	for _, ep := range sb.getConnectedEndpoints() {
		if ep.getNetwork().Type() == "null" || ep.getNetwork().Type() == "host" {
			continue
		}
		if len(ep.Gateway()) != 0 {
			return ep
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package utils

import (
	"bufio"
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"errors"
	"fmt"
	"image"
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"
	"io"
	"math/rand"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/eirka/eirka-libs/amazon"
	"github.com/eirka/eirka-libs/config"

	local "github.com/eirka/eirka-post/config"
)

// valid file extensions
var validExt = map[string]bool{
	".jpg":  true,
	".jpeg": true,
	".png":  true,
	".gif":  true,
	".webm": true,
}

type ImageType struct {
	File        multipart.File
	Header      *multipart.FileHeader
	Filename    string
	Thumbnail   string
	Ext         string
	MD5         string
	OrigWidth   int
	OrigHeight  int
	ThumbWidth  int
	ThumbHeight int
	image       bytes.Buffer
	mime        string
	duration    int
}

// ProcessFile will check file integrity, get an md5, and make filenames
func (i *ImageType) ProcessFile() (err error) {

	// check given file ext
	err = i.checkReqExt()
	if err != nil {
		return
	}

	// get file md5
	err = i.getMD5()
	if err != nil {
		return
	}

	// check file magic sig
	err = i.checkMagic()
	if err != nil {
		return
	}

	return

}

// save an image file
func (i *ImageType) SaveImage() (err error) {

	// check image stats
	err = i.getStats()
	if err != nil {
		return
	}

	// save the file to disk
	err = i.saveFile()
	if err != nil {
		return
	}

	// create a thumbnail
	err = i.createThumbnail()
	if err != nil {
		return
	}

	return

}

// Get file extension from request header
func (i *ImageType) checkReqExt() (err error) {
	// Get ext from request header
	name := i.Header.Filename
	ext := filepath.Ext(name)

	if ext == "" {
		return errors.New("no file extension")
	}

	// Check to see if extension is allowed
	ext_check := isAllowedExt(ext)
	if !ext_check {
		return errors.New("format not supported")
	}

	return

}

// Check if file ext allowed
func isAllowedExt(ext string) bool {

	if validExt[strings.ToLower(ext)] {
		return true
	}

	return false

}

// Get image MD5 and write file into buffer
func (i *ImageType) getMD5() (err error) {

	defer i.File.Close()

	hasher := md5.New()

	// Save file and also read into hasher for md5
	_, err = io.Copy(i.image, io.TeeReader(i.File, hasher))
	if err != nil {
		return errors.New("problem copying file")
	}

	// Set md5sum from hasher
	i.MD5 = hex.EncodeToString(hasher.Sum(nil))

	return

}

func (i *ImageType) checkMagic() (err error) {

	// detect the mime type
	i.mime = http.DetectContentType(i.image.Bytes())

	switch i.mime {
	case "image/png":
		i.Ext = ".png"
	case "image/jpeg":
		i.Ext = ".jpg"
	case "image/gif":
		i.Ext = ".gif"
	case "video/webm":
		i.Ext = ".webm"
	default:
		return errors.New("unknown file type")
	}

	// Check to see if extension is allowed
	ext_check := isAllowedExt(i.Ext)
	if !ext_check {
		return errors.New("format not supported")
	}

	return

}

func (i *ImageType) getStats() (err error) {

	// decode image config
	img, _, err := image.DecodeConfig(i.image)
	if err != nil {
		return errors.New("problem decoding image")
	}

	// set original width
	i.OrigWidth = img.Width
	// set original height
	i.OrigHeight = img.Height

	// Check against maximum sizes
	switch {
	case i.OrigWidth > config.Settings.Limits.ImageMaxWidth:
		return errors.New("image width too large")
	case img.Width < config.Settings.Limits.ImageMinWidth:
		return errors.New("image width too small")
	case i.OrigHeight > config.Settings.Limits.ImageMaxHeight:
		return errors.New("image height too large")
	case img.Height < config.Settings.Limits.ImageMinHeight:
		return errors.New("image height too small")
	case i.image.Len() > config.Settings.Limits.ImageMaxSize:
		return errors.New("image size too large")
	}

	return

}

func (i *ImageType) saveFile() (err error) {

	// generate filenames
	i.makeFilenames()

	imagefile := filepath.Join(local.Settings.Directories.ImageDir, i.Filename)

	image, err := os.Create(imagefile)
	if err != nil {
		return errors.New("problem saving file")
	}
	defer image.Close()

	_, err = io.Copy(image, i.image)
	if err != nil {
		return errors.New("problem saving file")
	}

	s3 := amazon.New()

	err = s3.Save(imagefile, fmt.Sprintf("src/%s", i.Filename), i.mime)
	if err != nil {
		return
	}

	return

}

// Make a random unix time filename
func (i *ImageType) makeFilenames() {

	// Create seed for random
	rand.Seed(time.Now().UnixNano())

	// Get random 3 digit int to append to unix time
	rand_t := rand.Intn(899) + 100

	// Get current unix time
	time_t := time.Now().Unix()

	// Append random int to unix time
	file_t := fmt.Sprintf("%d%d", time_t, rand_t)

	// Append ext to filename
	i.Filename = fmt.Sprintf("%s%s", file_t, i.Ext)

	// Append jpg to thumbnail name because it is always a jpg
	i.Thumbnail = fmt.Sprintf("%ss.jpg", file_t)

	return

}

func (i *ImageType) createThumbnail() (err error) {

	object := amazon.LambdaThumbnail{
		Bucket:    config.Settings.Amazon.Bucket,
		Filename:  i.Filename,
		Thumbnail: i.Thumbnail,
		MaxWidth:  config.Settings.Limits.ThumbnailMaxWidth,
		MaxHeight: config.Settings.Limits.ThumbnailMaxHeight,
	}

	lambda := amazon.New()

	// run our lambda job and get the dimensions
	i.ThumbWidth, i.ThumbHeight, err = lambda.Execute(object)
	if err != nil {
		return
	}

	return

}
<commit_msg>add tests and change image functions a bit<commit_after>package utils

import (
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"errors"
	"fmt"
	"image"
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"
	"io"
	"math/rand"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/eirka/eirka-libs/amazon"
	"github.com/eirka/eirka-libs/config"

	local "github.com/eirka/eirka-post/config"
)

// valid file extensions
var validExt = map[string]bool{
	".jpg":  true,
	".jpeg": true,
	".png":  true,
	".gif":  true,
	".webm": true,
}

type ImageType struct {
	File        multipart.File
	Header      *multipart.FileHeader
	Filename    string
	Thumbnail   string
	Ext         string
	MD5         string
	OrigWidth   int
	OrigHeight  int
	ThumbWidth  int
	ThumbHeight int
	image       *bytes.Buffer
	mime        string
	duration    int
}

// ProcessFile will check file integrity, get an md5, and make filenames
func (i *ImageType) ProcessFile() (err error) {

	// check given file ext
	err = i.checkReqExt()
	if err != nil {
		return
	}

	// get file md5
	err = i.getMD5()
	if err != nil {
		return
	}

	// check file magic sig
	err = i.checkMagic()
	if err != nil {
		return
	}

	return

}

// save an image file
func (i *ImageType) SaveImage() (err error) {

	// check image stats
	err = i.getStats()
	if err != nil {
		return
	}

	// save the file to disk
	err = i.saveFile()
	if err != nil {
		return
	}

	// create a thumbnail
	err = i.createThumbnail()
	if err != nil {
		return
	}

	return

}

// Get file extension from request header
func (i *ImageType) checkReqExt() (err error) {
	// Get ext from request header
	name := i.Header.Filename
	ext := filepath.Ext(name)

	if ext == "" {
		return errors.New("no file extension")
	}

	// Check to see if extension is allowed
	ext_check := isAllowedExt(ext)
	if !ext_check {
		return errors.New("format not supported")
	}

	return

}

// Check if file ext allowed
func isAllowedExt(ext string) bool {

	if validExt[strings.ToLower(ext)] {
		return true
	}

	return false

}

// Get image MD5 and write file into buffer
func (i *ImageType) getMD5() (err error) {

	defer i.File.Close()

	hasher := md5.New()

	// Save file and also read into hasher for md5
	_, err = io.Copy(i.image, io.TeeReader(i.File, hasher))
	if err != nil {
		return errors.New("problem copying file")
	}

	// Set md5sum from hasher
	i.MD5 = hex.EncodeToString(hasher.Sum(nil))

	return

}

func (i *ImageType) checkMagic() (err error) {

	// detect the mime type
	i.mime = http.DetectContentType(i.image.Bytes())

	switch i.mime {
	case "image/png":
		i.Ext = ".png"
	case "image/jpeg":
		i.Ext = ".jpg"
	case "image/gif":
		i.Ext = ".gif"
	case "video/webm":
		i.Ext = ".webm"
	default:
		return errors.New("unknown file type")
	}

	// Check to see if extension is allowed
	ext_check := isAllowedExt(i.Ext)
	if !ext_check {
		return errors.New("format not supported")
	}

	return

}

func (i *ImageType) getStats() (err error) {

	// decode image config
	img, _, err := image.DecodeConfig(i.image)
	if err != nil {
		return errors.New("problem decoding image")
	}

	// set original width
	i.OrigWidth = img.Width
	// set original height
	i.OrigHeight = img.Height

	// Check against maximum sizes
	switch {
	case i.OrigWidth > config.Settings.Limits.ImageMaxWidth:
		return errors.New("image width too large")
	case img.Width < config.Settings.Limits.ImageMinWidth:
		return errors.New("image width too small")
	case i.OrigHeight > config.Settings.Limits.ImageMaxHeight:
		return errors.New("image height too large")
	case img.Height < config.Settings.Limits.ImageMinHeight:
		return errors.New("image height too small")
	case i.image.Len() > config.Settings.Limits.ImageMaxSize:
		return errors.New("image size too large")
	}

	return

}

func (i *ImageType) saveFile() (err error) {

	// generate filenames
	i.makeFilenames()

	imagefile := filepath.Join(local.Settings.Directories.ImageDir, i.Filename)

	image, err := os.Create(imagefile)
	if err != nil {
		return errors.New("problem saving file")
	}
	defer image.Close()

	_, err = io.Copy(image, i.image)
	if err != nil {
		return errors.New("problem saving file")
	}

	s3 := amazon.New()

	err = s3.Save(imagefile, fmt.Sprintf("src/%s", i.Filename), i.mime)
	if err != nil {
		return
	}

	return

}

// Make a random unix time filename
func (i *ImageType) makeFilenames() {

	// Create seed for random
	rand.Seed(time.Now().UnixNano())

	// Get random 3 digit int to append to unix time
	rand_t := rand.Intn(899) + 100

	// Get current unix time
	time_t := time.Now().Unix()

	// Append random int to unix time
	file_t := fmt.Sprintf("%d%d", time_t, rand_t)

	// Append ext to filename
	i.Filename = fmt.Sprintf("%s%s", file_t, i.Ext)

	// Append jpg to thumbnail name because it is always a jpg
	i.Thumbnail = fmt.Sprintf("%ss.jpg", file_t)

	return

}

func (i *ImageType) createThumbnail() (err error) {

	object := amazon.LambdaThumbnail{
		Bucket:    config.Settings.Amazon.Bucket,
		Filename:  i.Filename,
		Thumbnail: i.Thumbnail,
		MaxWidth:  config.Settings.Limits.ThumbnailMaxWidth,
		MaxHeight: config.Settings.Limits.ThumbnailMaxHeight,
	}

	lambda := amazon.New()

	// run our lambda job and get the dimensions
	i.ThumbWidth, i.ThumbHeight, err = lambda.Execute(object)
	if err != nil {
		return
	}

	return

}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"github.com/yokujin/gousb/usb"
	"log"
	"os"
)

const (
	dynastreamUsbVendid = 0x0fcf
	antstick            = 0x1008
	// USB endpoint info
	uconf  = 01
	uiface = 00
	usetup = 00
	uep    = 0x01
)

func main() {
	fmt.Println("- Life Begins -\n")

	// Get context
	ctx := usb.NewContext()
	defer ctx.Close()

	ctx.Debug(3)

	// Find and open the device
	devs, err := ctx.ListDevices(func(desc *usb.Descriptor) bool {
		if desc.Vendor == dynastreamUsbVendid && desc.Product == antstick {
			fmt.Println("Found antstick")
			return true
		}
		return false
	})

	defer func() {
		for _, d := range devs {
			d.Close()
		}
	}()

	if err != nil {
		log.Fatalln("ERROR! ", err)
		return
	}

	// Exit if no devices opened
	if len(devs) == 0 {
		log.Fatalln("No devices found")
		return
	}

	// Pick off the first device

	antdev := devs[0]

	log.Println("Opening Endpoints...")
	epRead, err := antdev.OpenEndpoint(
		uconf,
		uiface,
		usetup,
		uint8(uep)|uint8(usb.ENDPOINT_DIR_IN),
	)
	epWrite, err := antdev.OpenEndpoint(
		uconf,
		uiface,
		usetup,
		uint8(uep)|uint8(usb.ENDPOINT_DIR_OUT),
	)

	if err != nil {
		log.Println("Error opening endpoint, ", err)
		return
	}

	// Get the ant plus network key
	key, err := getNetworkKey()
	if err != nil {
		log.Fatalln("Error getting key, ", err)
	}

	// Create antbuffer
	antbuf, err := NewAntbuffer(epRead, epWrite, key)
	if err != nil {
		log.Fatalln("Error in creating antbuffer, ", err)
	}

	// ListenForCheststrap
	// TODO: this should be using the returned channel to listen
	// All errors at a higher than channel level are to be handled
	// by the Antbuffer
	_, err = antbuf.SetupChannel(0x01, heartrate)
	if err != nil {
		log.Fatalln("Error listening to Heart Rate sensor, ", err)
	}

	// TODO: Move this somewhere else
	// Catch close signal
	killchan := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill)

	// TODO: This should be implicit in the Antbuffer
	defer func() {
		_, err = antbuf.GenSendAndWait(CloseChannel, 0x01)
	}()

	// Listen for everything forever
	for {
		pkt, err := antbuf.Wait()
		if err == ErrAntTimedout {
			// Depending on stuff... might need to relisten for device
			// Device relisten should really be based off of error
			// events from the stick.
			continue
		} else if err != nil {
			log.Fatalln("Error in waiting, ", err)
		}
		log.Println(pkt)
	}

	// Exiting
	fmt.Println("Exiting...")
}

func getNetworkKey() (key []byte, err error) {
	// Network Key
	// TODO: Publish additional go binary to write the key from command line
	// TODO: Make flag for where this is
	// TODO: Note this in documentation
	file, err := os.Open("/etc/ant/antPlusNetworkKey")
	if err != nil {
		return nil, errors.New(fmt.Sprint("Error opening ant key file in /etc/ant/antPlusNetworkKey, ", err))
	}
	defer file.Close()

	// The network key is 8 bytes long
	key = make([]byte, 8)
	n, err := file.Read(key)
	if err != nil {
		return nil, err
	} else if n != 8 {
		return nil, ErrNetworkKeyLength
	}

	return key, nil
}
<commit_msg>Made application listen for kill so channels are closed properly<commit_after>package main

import (
	"errors"
	"fmt"
	"github.com/yokujin/gousb/usb"
	"log"
	"os"
	"os/signal"
	"time"
)

const (
	dynastreamUsbVendid = 0x0fcf
	antstick            = 0x1008
	// USB endpoint info
	uconf  = 01
	uiface = 00
	usetup = 00
	uep    = 0x01
)

func main() {
	fmt.Println("- Life Begins -\n")

	// Get context
	ctx := usb.NewContext()
	defer ctx.Close()

	ctx.Debug(3)

	// Find and open the device
	devs, err := ctx.ListDevices(func(desc *usb.Descriptor) bool {
		if desc.Vendor == dynastreamUsbVendid && desc.Product == antstick {
			fmt.Println("Found antstick")
			return true
		}
		return false
	})

	defer func() {
		for _, d := range devs {
			d.Close()
		}
	}()

	if err != nil {
		log.Fatalln("ERROR! ", err)
		return
	}

	// Exit if no devices opened
	if len(devs) == 0 {
		log.Fatalln("No devices found")
		return
	}

	// Pick off the first device

	antdev := devs[0]

	log.Println("Opening Endpoints...")
	epRead, err := antdev.OpenEndpoint(
		uconf,
		uiface,
		usetup,
		uint8(uep)|uint8(usb.ENDPOINT_DIR_IN),
	)
	epWrite, err := antdev.OpenEndpoint(
		uconf,
		uiface,
		usetup,
		uint8(uep)|uint8(usb.ENDPOINT_DIR_OUT),
	)

	if err != nil {
		log.Println("Error opening endpoint, ", err)
		return
	}

	// Get the ant plus network key
	key, err := getNetworkKey()
	if err != nil {
		log.Fatalln("Error getting key, ", err)
	}

	// Create antbuffer
	antbuf, err := NewAntbuffer(epRead, epWrite, key)
	if err != nil {
		log.Fatalln("Error in creating antbuffer, ", err)
	}

	// ListenForCheststrap
	// TODO: this should be using the returned channel to listen
	// All errors at a higher than channel level are to be handled
	// by the Antbuffer
	_, err = antbuf.SetupChannel(0x01, heartrate)
	if err != nil {
		log.Fatalln("Error listening to Heart Rate sensor, ", err)
	}
	// TODO: This should be implicit in the Antbuffer
	defer func() {
		log.Println("Closing channels...")
		antbuf.GenSendAndWait(CloseChannel, 0x01)
		// Wait for complete close
		log.Println("Waiting for confirmation...")
		for {
			pkt, err := antbuf.Wait()
			if err != nil {
				log.Fatalln("Error while closing, ", err)
			}
			if pkt.id == ChannelResponseOrEvent {
				if pkt.data[2] == 0x07 {
					// Channel was successfully closed
					log.Println("Successfully closed channel ", pkt.data[0], " proceeding to exit...")
					break
				}
			}
		}
	}()

	// TODO: Move this somewhere else
	// Catch close signal
	killchan := make(chan os.Signal, 1)
	signal.Notify(killchan, os.Interrupt, os.Kill)

	// Listen for everything forever
readloop:
	for {
		// Die if killed
		select {
		case <-killchan:
			log.Println("Recieved KILL!")
			break readloop
		case <-time.After(10 * time.Millisecond):
		}
		pkt, err := antbuf.Wait()
		if err == ErrAntTimedout {
			// Depending on stuff... might need to relisten for device
			// Device relisten should really be based off of error
			// events from the stick.
			continue
		} else if err != nil {
			log.Fatalln("Error in waiting, ", err)
		}
		log.Println(pkt)
	}

	// Exiting
	fmt.Println("Exiting...")
}

func getNetworkKey() (key []byte, err error) {
	// Network Key
	// TODO: Publish additional go binary to write the key from command line
	// TODO: Make flag for where this is
	// TODO: Note this in documentation
	file, err := os.Open("/etc/ant/antPlusNetworkKey")
	if err != nil {
		return nil, errors.New(fmt.Sprint("Error opening ant key file in /etc/ant/antPlusNetworkKey, ", err))
	}
	defer file.Close()

	// The network key is 8 bytes long
	key = make([]byte, 8)
	n, err := file.Read(key)
	if err != nil {
		return nil, err
	} else if n != 8 {
		return nil, ErrNetworkKeyLength
	}

	return key, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package wavefront

import (
	"fmt"
	"github.com/golang/glog"
	"k8s.io/heapster/metrics/core"
	"net"
	"net/url"
	"sort"
	"strconv"
	"strings"
	"time"
)

const (
	sysSubContainerName = "system.slice/"
)

var excludeTagList = [...]string{"namespace_id", "host_id", "pod_id", "hostname"}

type wavefrontSink struct {
	Conn              net.Conn
	ProxyAddress      string
	ClusterName       string
	Prefix            string
	IncludeLabels     bool
	IncludeContainers bool
	testMode          bool
	testReceivedLines []string
}

func (wfSink *wavefrontSink) Name() string {
	return "Wavefront Sink"
}

func (wfSink *wavefrontSink) Stop() {
	// Do nothing.
	wfSink.Conn.Close()
}

func (wfSink *wavefrontSink) sendLine(line string) {
	if wfSink.testMode {
		wfSink.testReceivedLines = append(wfSink.testReceivedLines, line)
		glog.Infoln(line)
		return
	}
	//if the connection was closed or interrupted - don't cause a panic (we'll retry at next interval)
	defer func() {
		if r := recover(); r != nil {
			//we couldn't write the line so something is wrong with the connection
			wfSink.Conn = nil
		}
	}()
	if wfSink.Conn != nil {
		wfSink.Conn.Write([]byte(line))
	}
}

func (wfSink *wavefrontSink) sendPoint(metricName string, metricValStr string, ts string, source string, tagStr string) {
	metricLine := fmt.Sprintf("%s %s %s source=\"%s\" %s\n", metricName, metricValStr, ts, source, tagStr)
	wfSink.sendLine(metricLine)
}

func tagsToString(tags map[string]string) string {
	tagStr := ""
	for k, v := range tags {
		//if k != "hostname" {
		if excludeTag(k) == false {
			tagStr += k + "=\"" + v + "\" "
		}
	}
	return tagStr
}

func excludeTag(a string) bool {
	for _, b := range excludeTagList {
		if b == a {
			return true
		}
	}
	return false
}

func (wfSink *wavefrontSink) cleanMetricName(metricType string, metricName string) string {
	return wfSink.Prefix + metricType + "." + strings.Replace(metricName, "/", ".", -1)
}

func (wfSink *wavefrontSink) addLabelTags(ms *core.MetricSet, tags map[string]string) {
	for _, labelName := range sortedLabelKeys(ms.Labels) {
		labelValue := ms.Labels[labelName]
		if labelName == "labels" {
			//only parse labels if IncludeLabels == true
			if wfSink.IncludeLabels {
				for _, label := range strings.Split(labelValue, ",") {
					//labels = app:webproxy,version:latest
					tagParts := strings.SplitN(label, ":", 2)
					if len(tagParts) == 2 {
						tags["label."+tagParts[0]] = tagParts[1]
					}
				}
			}
		} else {
			tags[labelName] = labelValue
		}
	}

}

func (wfSink *wavefrontSink) send(batch *core.DataBatch) {

	metricCounter := 0

	for _, key := range sortedMetricSetKeys(batch.MetricSets) {
		ms := batch.MetricSets[key]

		// Populate tag map
		tags := make(map[string]string)
		// Make sure all metrics are tagged with the cluster name
		tags["cluster"] = wfSink.ClusterName
		// Add pod labels as tags
		wfSink.addLabelTags(ms, tags)
		metricType := tags["type"]
		if strings.Contains(tags["container_name"], sysSubContainerName) {
			//don't send system subcontainers
			continue
		}
		if wfSink.IncludeContainers == false && strings.Contains(metricType, "pod_container") {
			// the user doesn't want to include container metrics (only pod and above)
			continue
		}
		for _, metricName := range sortedMetricValueKeys(ms.MetricValues) {
			var metricValStr string
			metricValue := ms.MetricValues[metricName]
			if core.ValueInt64 == metricValue.ValueType {
				metricValStr = fmt.Sprintf("%d", metricValue.IntValue)
			} else if core.ValueFloat == metricValue.ValueType { // W
				metricValStr = fmt.Sprintf("%f", metricValue.FloatValue)
			} else {
				//do nothing for now
				metricValStr = ""
			}
			if metricValStr != "" {
				ts := strconv.FormatInt(batch.Timestamp.Unix(), 10)
				source := ""
				if metricType == "cluster" {
					source = wfSink.ClusterName
				} else if metricType == "ns" {
					source = tags["namespace_name"] + "-ns"
				} else {
					source = tags["hostname"]
				}
				tagStr := tagsToString(tags)
				wfSink.sendPoint(wfSink.cleanMetricName(metricType, metricName), metricValStr, ts, source, tagStr)
				metricCounter = metricCounter + 1
			}
		}
		for _, metric := range ms.LabeledMetrics {
			metricName := wfSink.cleanMetricName(metricType, metric.Name)
			metricValStr := ""
			if core.ValueInt64 == metric.ValueType {
				metricValStr = fmt.Sprintf("%d", metric.IntValue)
			} else if core.ValueFloat == metric.ValueType { // W
				metricValStr = fmt.Sprintf("%f", metric.FloatValue)
			} else {
				//do nothing for now
				metricValStr = ""
			}
			if metricValStr != "" {
				ts := strconv.FormatInt(batch.Timestamp.Unix(), 10)
				source := tags["hostname"]
				tagStr := tagsToString(tags)
				for labelName, labelValue := range metric.Labels {
					tagStr += labelName + "=\"" + labelValue + "\" "
				}
				metricCounter = metricCounter + 1
				wfSink.sendPoint(metricName, metricValStr, ts, source, tagStr)
			}
		}
	}

}

func (wfSink *wavefrontSink) ExportData(batch *core.DataBatch) {

	if wfSink.testMode {
		//clear lines from last batch
		wfSink.testReceivedLines = wfSink.testReceivedLines[:0]
		wfSink.send(batch)
		return
	}

	//make sure we're Connected before sending a real batch
	err := wfSink.connect()
	if err != nil {
		glog.Warning(err)
	}

	if wfSink.Conn != nil && err == nil {
		wfSink.send(batch)
	}
}

func (wfSink *wavefrontSink) connect() error {
	var err error
	wfSink.Conn, err = net.DialTimeout("tcp", wfSink.ProxyAddress, time.Second*10)
	if err != nil {
		glog.Warningf("Unable to connect to Wavefront proxy at address: %s", wfSink.ProxyAddress)
		return err
	} else {
		glog.Infof("Connected to Wavefront proxy at address: %s", wfSink.ProxyAddress)
		return nil
	}
}

func NewWavefrontSink(uri *url.URL) (core.DataSink, error) {

	storage := &wavefrontSink{
		ProxyAddress:      uri.Scheme + ":" + uri.Opaque,
		ClusterName:       "k8s-cluster",
		Prefix:            "heapster.",
		IncludeLabels:     false,
		IncludeContainers: true,
		testMode:          false,
	}

	vals := uri.Query()
	if len(vals["clusterName"]) > 0 {
		storage.ClusterName = vals["clusterName"][0]
	}
	if len(vals["prefix"]) > 0 {
		storage.Prefix = vals["prefix"][0]
	}
	if len(vals["includeLabels"]) > 0 {
		incLabels := false
		incLabels, err := strconv.ParseBool(vals["includeLabels"][0])
		if err != nil {
			glog.Warning("Unable to parse the includeLabels argument. This argument is a boolean, please pass \"true\" or \"false\"")
			return nil, err
		}
		storage.IncludeLabels = incLabels
	}
	if len(vals["includeContainers"]) > 0 {
		incContainers := false
		incContainers, err := strconv.ParseBool(vals["includeContainers"][0])
		if err != nil {
			glog.Warning("Unable to parse the includeContainers argument. This argument is a boolean, please pass \"true\" or \"false\"")
			return nil, err
		}
		storage.IncludeContainers = incContainers
	}
	if len(vals["testMode"]) > 0 {
		testMode := false
		testMode, err := strconv.ParseBool(vals["testMode"][0])
		if err != nil {
			glog.Warning("Unable to parse the testMode argument. This argument is a boolean, please pass \"true\" or \"false\"")
			return nil, err
		}
		storage.testMode = testMode
	}
	return storage, nil
}

func sortedMetricSetKeys(m map[string]*core.MetricSet) []string {
	keys := make([]string, len(m))
	i := 0
	for k := range m {
		keys[i] = k
		i++
	}
	sort.Strings(keys)
	return keys
}

func sortedLabelKeys(m map[string]string) []string {
	keys := make([]string, len(m))
	i := 0
	for k := range m {
		keys[i] = k
		i++
	}
	sort.Strings(keys)
	return keys
}

func sortedMetricValueKeys(m map[string]core.MetricValue) []string {
	keys := make([]string, len(m))
	i := 0
	for k := range m {
		keys[i] = k
		i++
	}
	sort.Strings(keys)
	return keys
}
<commit_msg>Wavefront sink: filter out tags with blank values<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package wavefront

import (
	"fmt"
	"github.com/golang/glog"
	"k8s.io/heapster/metrics/core"
	"net"
	"net/url"
	"sort"
	"strconv"
	"strings"
	"time"
)

const (
	sysSubContainerName = "system.slice/"
)

var excludeTagList = [...]string{"namespace_id", "host_id", "pod_id", "hostname"}

type wavefrontSink struct {
	Conn              net.Conn
	ProxyAddress      string
	ClusterName       string
	Prefix            string
	IncludeLabels     bool
	IncludeContainers bool
	testMode          bool
	testReceivedLines []string
}

func (wfSink *wavefrontSink) Name() string {
	return "Wavefront Sink"
}

func (wfSink *wavefrontSink) Stop() {
	// Do nothing.
	wfSink.Conn.Close()
}

func (wfSink *wavefrontSink) sendLine(line string) {
	if wfSink.testMode {
		wfSink.testReceivedLines = append(wfSink.testReceivedLines, line)
		glog.Infoln(line)
		return
	}
	//if the connection was closed or interrupted - don't cause a panic (we'll retry at next interval)
	defer func() {
		if r := recover(); r != nil {
			//we couldn't write the line so something is wrong with the connection
			wfSink.Conn = nil
		}
	}()
	if wfSink.Conn != nil {
		wfSink.Conn.Write([]byte(line))
	}
}

func (wfSink *wavefrontSink) sendPoint(metricName string, metricValStr string, ts string, source string, tagStr string) {
	metricLine := fmt.Sprintf("%s %s %s source=\"%s\" %s\n", metricName, metricValStr, ts, source, tagStr)
	wfSink.sendLine(metricLine)
}

func tagsToString(tags map[string]string) string {
	tagStr := ""
	for k, v := range tags {
		// ignore tags with empty values as well so the data point doesn't fail validation
		if excludeTag(k) == false && len(v) > 0 {
			tagStr += k + "=\"" + v + "\" "
		}
	}
	return tagStr
}

func excludeTag(a string) bool {
	for _, b := range excludeTagList {
		if b == a {
			return true
		}
	}
	return false
}

func (wfSink *wavefrontSink) cleanMetricName(metricType string, metricName string) string {
	return wfSink.Prefix + metricType + "." + strings.Replace(metricName, "/", ".", -1)
}

func (wfSink *wavefrontSink) addLabelTags(ms *core.MetricSet, tags map[string]string) {
	for _, labelName := range sortedLabelKeys(ms.Labels) {
		labelValue := ms.Labels[labelName]
		if labelName == "labels" {
			//only parse labels if IncludeLabels == true
			if wfSink.IncludeLabels {
				for _, label := range strings.Split(labelValue, ",") {
					//labels = app:webproxy,version:latest
					tagParts := strings.SplitN(label, ":", 2)
					if len(tagParts) == 2 {
						tags["label."+tagParts[0]] = tagParts[1]
					}
				}
			}
		} else {
			tags[labelName] = labelValue
		}
	}

}

func (wfSink *wavefrontSink) send(batch *core.DataBatch) {

	metricCounter := 0

	for _, key := range sortedMetricSetKeys(batch.MetricSets) {
		ms := batch.MetricSets[key]

		// Populate tag map
		tags := make(map[string]string)
		// Make sure all metrics are tagged with the cluster name
		tags["cluster"] = wfSink.ClusterName
		// Add pod labels as tags
		wfSink.addLabelTags(ms, tags)
		metricType := tags["type"]
		if strings.Contains(tags["container_name"], sysSubContainerName) {
			//don't send system subcontainers
			continue
		}
		if wfSink.IncludeContainers == false && strings.Contains(metricType, "pod_container") {
			// the user doesn't want to include container metrics (only pod and above)
			continue
		}
		for _, metricName := range sortedMetricValueKeys(ms.MetricValues) {
			var metricValStr string
			metricValue := ms.MetricValues[metricName]
			if core.ValueInt64 == metricValue.ValueType {
				metricValStr = fmt.Sprintf("%d", metricValue.IntValue)
			} else if core.ValueFloat == metricValue.ValueType { // W
				metricValStr = fmt.Sprintf("%f", metricValue.FloatValue)
			} else {
				//do nothing for now
				metricValStr = ""
			}
			if metricValStr != "" {
				ts := strconv.FormatInt(batch.Timestamp.Unix(), 10)
				source := ""
				if metricType == "cluster" {
					source = wfSink.ClusterName
				} else if metricType == "ns" {
					source = tags["namespace_name"] + "-ns"
				} else {
					source = tags["hostname"]
				}
				tagStr := tagsToString(tags)
				wfSink.sendPoint(wfSink.cleanMetricName(metricType, metricName), metricValStr, ts, source, tagStr)
				metricCounter = metricCounter + 1
			}
		}
		for _, metric := range ms.LabeledMetrics {
			metricName := wfSink.cleanMetricName(metricType, metric.Name)
			metricValStr := ""
			if core.ValueInt64 == metric.ValueType {
				metricValStr = fmt.Sprintf("%d", metric.IntValue)
			} else if core.ValueFloat == metric.ValueType { // W
				metricValStr = fmt.Sprintf("%f", metric.FloatValue)
			} else {
				//do nothing for now
				metricValStr = ""
			}
			if metricValStr != "" {
				ts := strconv.FormatInt(batch.Timestamp.Unix(), 10)
				source := tags["hostname"]
				tagStr := tagsToString(tags)
				for labelName, labelValue := range metric.Labels {
					tagStr += labelName + "=\"" + labelValue + "\" "
				}
				metricCounter = metricCounter + 1
				wfSink.sendPoint(metricName, metricValStr, ts, source, tagStr)
			}
		}
	}

}

func (wfSink *wavefrontSink) ExportData(batch *core.DataBatch) {

	if wfSink.testMode {
		//clear lines from last batch
		wfSink.testReceivedLines = wfSink.testReceivedLines[:0]
		wfSink.send(batch)
		return
	}

	//make sure we're Connected before sending a real batch
	err := wfSink.connect()
	if err != nil {
		glog.Warning(err)
	}

	if wfSink.Conn != nil && err == nil {
		wfSink.send(batch)
	}
}

func (wfSink *wavefrontSink) connect() error {
	var err error
	wfSink.Conn, err = net.DialTimeout("tcp", wfSink.ProxyAddress, time.Second*10)
	if err != nil {
		glog.Warningf("Unable to connect to Wavefront proxy at address: %s", wfSink.ProxyAddress)
		return err
	} else {
		glog.Infof("Connected to Wavefront proxy at address: %s", wfSink.ProxyAddress)
		return nil
	}
}

func NewWavefrontSink(uri *url.URL) (core.DataSink, error) {

	storage := &wavefrontSink{
		ProxyAddress:      uri.Scheme + ":" + uri.Opaque,
		ClusterName:       "k8s-cluster",
		Prefix:            "heapster.",
		IncludeLabels:     false,
		IncludeContainers: true,
		testMode:          false,
	}

	vals := uri.Query()
	if len(vals["clusterName"]) > 0 {
		storage.ClusterName = vals["clusterName"][0]
	}
	if len(vals["prefix"]) > 0 {
		storage.Prefix = vals["prefix"][0]
	}
	if len(vals["includeLabels"]) > 0 {
		incLabels := false
		incLabels, err := strconv.ParseBool(vals["includeLabels"][0])
		if err != nil {
			glog.Warning("Unable to parse the includeLabels argument. This argument is a boolean, please pass \"true\" or \"false\"")
			return nil, err
		}
		storage.IncludeLabels = incLabels
	}
	if len(vals["includeContainers"]) > 0 {
		incContainers := false
		incContainers, err := strconv.ParseBool(vals["includeContainers"][0])
		if err != nil {
			glog.Warning("Unable to parse the includeContainers argument. This argument is a boolean, please pass \"true\" or \"false\"")
			return nil, err
		}
		storage.IncludeContainers = incContainers
	}
	if len(vals["testMode"]) > 0 {
		testMode := false
		testMode, err := strconv.ParseBool(vals["testMode"][0])
		if err != nil {
			glog.Warning("Unable to parse the testMode argument. This argument is a boolean, please pass \"true\" or \"false\"")
			return nil, err
		}
		storage.testMode = testMode
	}
	return storage, nil
}

func sortedMetricSetKeys(m map[string]*core.MetricSet) []string {
	keys := make([]string, len(m))
	i := 0
	for k := range m {
		keys[i] = k
		i++
	}
	sort.Strings(keys)
	return keys
}

func sortedLabelKeys(m map[string]string) []string {
	keys := make([]string, len(m))
	i := 0
	for k := range m {
		keys[i] = k
		i++
	}
	sort.Strings(keys)
	return keys
}

func sortedMetricValueKeys(m map[string]core.MetricValue) []string {
	keys := make([]string, len(m))
	i := 0
	for k := range m {
		keys[i] = k
		i++
	}
	sort.Strings(keys)
	return keys
}
<|endoftext|>


<commit_before>package utils

import (
	"crypto/rand"
	"fmt"
	"math/big"
	"os"
	"strings"

	log "github.com/Sirupsen/logrus"
	engineapi "github.com/docker/docker/client"
	"gitlab.ricebook.net/platform/core/g"
	"gitlab.ricebook.net/platform/core/types"
	"golang.org/x/net/context"
)

const (
	letters       = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
	shortenLength = 7
	CpuPeriodBase = 100000
	StreamPrefix  = 8
)

func RandomString(n int) string {
	r := make([]byte, n)
	for i := 0; i < n; i++ {
		n, err := rand.Int(rand.Reader, big.NewInt(int64(len(letters))))
		// 没那么惨吧
		if err != nil {
			continue
		}
		r[i] = letters[n.Int64()]
	}
	return string(r)
}

func TruncateID(id string) string {
	if len(id) > shortenLength {
		return id[:shortenLength]
	}
	return id
}

func Tail(path string) string {
	parts := strings.Split(path, "/")
	return parts[len(parts)-1]
}

func GetGitRepoName(url string) (string, error) {
	if !strings.HasPrefix(url, "git@") || !strings.HasSuffix(url, ".git") {
		return "", fmt.Errorf("Bad git url format %q", url)
	}

	x := strings.SplitN(url, ":", 2)
	if len(x) != 2 {
		return "", fmt.Errorf("Bad git url format %q", url)
	}

	y := strings.SplitN(x[1], "/", 2)
	if len(y) != 2 {
		return "", fmt.Errorf("Bad git url format %q", url)
	}
	return strings.TrimSuffix(y[1], ".git"), nil
}

func GetVersion(image string) string {
	if !strings.Contains(image, ":") {
		return "unknown"
	}

	parts := strings.Split(image, ":")
	if len(parts) != 2 {
		return "unknown"
	}

	return parts[1]
}

// Bind a docker engine client to context
func ToDockerContext(client *engineapi.Client) context.Context {
	return context.WithValue(context.Background(), "engine", client)
}

// Get a docker engine client from a context
func FromDockerContext(ctx context.Context) (*engineapi.Client, bool) {
	client, ok := ctx.Value("engine").(*engineapi.Client)
	return client, ok
}

func SaveFile(content, path string, mode os.FileMode) error {
	file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, mode)
	if err != nil {
		return err
	}
	defer file.Close()

	_, err = file.WriteString(content)
	return err
}

// copied from https://gist.github.com/jmervine/d88c75329f98e09f5c87
func safeSplit(s string) []string {
	split := strings.Split(s, " ")

	var result []string
	var inquote string
	var block string
	for _, i := range split {
		if inquote == "" {
			if strings.HasPrefix(i, "'") || strings.HasPrefix(i, "\"") {
				inquote = string(i[0])
				block = strings.TrimPrefix(i, inquote) + " "
			} else {
				result = append(result, i)
			}
		} else {
			if !strings.HasSuffix(i, inquote) {
				block += i + " "
			} else {
				block += strings.TrimSuffix(i, inquote)
				inquote = ""
				result = append(result, block)
				block = ""
			}
		}
	}

	return result
}

func MakeCommandLineArgs(s string) []string {
	r := []string{}
	for _, part := range safeSplit(s) {
		if len(part) == 0 {
			continue
		}
		r = append(r, part)
	}
	return r
}

func SendMemCap(cpumemmap map[string]types.CPUAndMem, tag string) {
	data := map[string]float64{}
	for node, cpuandmem := range cpumemmap {
		data[node] = float64(cpuandmem.MemCap)
	}
	clean_host := strings.Replace(g.Hostname, ".", "-", -1)

	err := g.Statsd.Send(data, clean_host, tag)
	if err != nil {
		log.Errorf("Error occured while sending data to statsd: %v", err)
	}
	return
}

// MakeContainerName joins appname, entrypoint, ident using '_'
func MakeContainerName(appname, entrypoint, ident string) string {
	return strings.Join([]string{appname, entrypoint, ident}, "_")
}

// ParseContainerName does the opposite thing as MakeContainerName
func ParseContainerName(containerName string) (string, string, string, error) {
	splits := strings.Split(containerName, "_")
	length := len(splits)
	if length >= 3 {
		return strings.Join(splits[0:length-2], "_"), splits[length-2], splits[length-1], nil
	}
	return "", "", "", fmt.Errorf("Bad containerName: %s", containerName)
}
<commit_msg>clean useless code<commit_after>package utils

import (
	"crypto/rand"
	"fmt"
	"math/big"
	"os"
	"strings"

	log "github.com/Sirupsen/logrus"
	engineapi "github.com/docker/docker/client"
	"gitlab.ricebook.net/platform/core/g"
	"gitlab.ricebook.net/platform/core/types"
	"golang.org/x/net/context"
)

const (
	letters       = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
	shortenLength = 7
	CpuPeriodBase = 100000
)

func RandomString(n int) string {
	r := make([]byte, n)
	for i := 0; i < n; i++ {
		n, err := rand.Int(rand.Reader, big.NewInt(int64(len(letters))))
		// 没那么惨吧
		if err != nil {
			continue
		}
		r[i] = letters[n.Int64()]
	}
	return string(r)
}

func TruncateID(id string) string {
	if len(id) > shortenLength {
		return id[:shortenLength]
	}
	return id
}

func Tail(path string) string {
	parts := strings.Split(path, "/")
	return parts[len(parts)-1]
}

func GetGitRepoName(url string) (string, error) {
	if !strings.HasPrefix(url, "git@") || !strings.HasSuffix(url, ".git") {
		return "", fmt.Errorf("Bad git url format %q", url)
	}

	x := strings.SplitN(url, ":", 2)
	if len(x) != 2 {
		return "", fmt.Errorf("Bad git url format %q", url)
	}

	y := strings.SplitN(x[1], "/", 2)
	if len(y) != 2 {
		return "", fmt.Errorf("Bad git url format %q", url)
	}
	return strings.TrimSuffix(y[1], ".git"), nil
}

func GetVersion(image string) string {
	if !strings.Contains(image, ":") {
		return "unknown"
	}

	parts := strings.Split(image, ":")
	if len(parts) != 2 {
		return "unknown"
	}

	return parts[1]
}

// Bind a docker engine client to context
func ToDockerContext(client *engineapi.Client) context.Context {
	return context.WithValue(context.Background(), "engine", client)
}

// Get a docker engine client from a context
func FromDockerContext(ctx context.Context) (*engineapi.Client, bool) {
	client, ok := ctx.Value("engine").(*engineapi.Client)
	return client, ok
}

func SaveFile(content, path string, mode os.FileMode) error {
	file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, mode)
	if err != nil {
		return err
	}
	defer file.Close()

	_, err = file.WriteString(content)
	return err
}

// copied from https://gist.github.com/jmervine/d88c75329f98e09f5c87
func safeSplit(s string) []string {
	split := strings.Split(s, " ")

	var result []string
	var inquote string
	var block string
	for _, i := range split {
		if inquote == "" {
			if strings.HasPrefix(i, "'") || strings.HasPrefix(i, "\"") {
				inquote = string(i[0])
				block = strings.TrimPrefix(i, inquote) + " "
			} else {
				result = append(result, i)
			}
		} else {
			if !strings.HasSuffix(i, inquote) {
				block += i + " "
			} else {
				block += strings.TrimSuffix(i, inquote)
				inquote = ""
				result = append(result, block)
				block = ""
			}
		}
	}

	return result
}

func MakeCommandLineArgs(s string) []string {
	r := []string{}
	for _, part := range safeSplit(s) {
		if len(part) == 0 {
			continue
		}
		r = append(r, part)
	}
	return r
}

func SendMemCap(cpumemmap map[string]types.CPUAndMem, tag string) {
	data := map[string]float64{}
	for node, cpuandmem := range cpumemmap {
		data[node] = float64(cpuandmem.MemCap)
	}
	clean_host := strings.Replace(g.Hostname, ".", "-", -1)

	err := g.Statsd.Send(data, clean_host, tag)
	if err != nil {
		log.Errorf("Error occured while sending data to statsd: %v", err)
	}
	return
}

// MakeContainerName joins appname, entrypoint, ident using '_'
func MakeContainerName(appname, entrypoint, ident string) string {
	return strings.Join([]string{appname, entrypoint, ident}, "_")
}

// ParseContainerName does the opposite thing as MakeContainerName
func ParseContainerName(containerName string) (string, string, string, error) {
	splits := strings.Split(containerName, "_")
	length := len(splits)
	if length >= 3 {
		return strings.Join(splits[0:length-2], "_"), splits[length-2], splits[length-1], nil
	}
	return "", "", "", fmt.Errorf("Bad containerName: %s", containerName)
}
<|endoftext|>


<commit_before>// Package utils - some utils.
package utils

import (
	"regexp"
	"strconv"
	"time"
)

// ExchangeMap is simple to check tse or otc.
var ExchangeMap = map[string]bool{"tse": true, "otc": true}

// TaipeiTimeZone is for time.Data() setting.
var TaipeiTimeZone = time.FixedZone("Asia/Taipei", 8*3600)

// TWSE base url.
const (
	TWSEURL     string = "http://mis.tse.com.tw"
	TWSEHOST    string = "http://www.twse.com.tw"
	OTCHOST     string = "http://www.tpex.org.tw"
	OTCCSV      string = "/ch/stock/aftertrading/daily_trading_info/st43_download.php?d=%d/%02d&stkno=%s&r=%d" // year, mon, stock, rand
	TWSECSV     string = "/ch/trading/exchange/STOCK_DAY/STOCK_DAY_print.php?genpage=genpage/Report%d%02d/%d%02d_F3_1_8_%s.php&type=csv&r=%d"
	TWSELISTCSV string = "/ch/trading/exchange/MI_INDEX/MI_INDEX2_print.php?genpage=genpage/Report%d%02d/A112%d%02d%02d%s.php&type=csv"
	TWSEREAL    string = "/stock/api/getStockInfo.jsp?ex_ch=%s_%s.tw_%s&json=1&delay=0&_=%d"
)

// RandInt return random int.
func RandInt() int {
	return time.Now().Nanosecond()
}

//func RandInt() int64 {
//	result, _ := rand.Int(rand.Reader, big.NewInt(102400))
//	return result.Int64()
//}

//func RandInt() int64 {
//	r := rand.New(rand.NewSource(time.Now().UnixNano()))
//	return r.Int63n(100000)
//}

var dateReg = regexp.MustCompile(`([\d]{2,})/([\d]{1,2})/([\d]{1,2})`)

// ParseDate is to parse "104/01/13" format.
func ParseDate(strDate string) time.Time {
	p := dateReg.FindStringSubmatch(strDate)
	year, _ := strconv.Atoi(p[1])
	mon, _ := strconv.Atoi(p[2])
	day, _ := strconv.Atoi(p[3])
	return time.Date(year+1911, time.Month(mon), day, 0, 0, 0, 0, TaipeiTimeZone)
}

// SumFloat64 計算總和（float64）
func SumFloat64(data []float64) float64 {
	var result float64
	for _, v := range data {
		result += v
	}
	return result
}

// AvgFlast64 計算平均（float64）
func AvgFlast64(data []float64) float64 {
	return float64(int(SumFloat64(data)*100)/len(data)) / 100
}

// SumUint64 計算總和（uint64）
func SumUint64(data []uint64) uint64 {
	var result uint64
	for _, v := range data {
		result += v
	}
	return result
}

// AvgUint64 計算平均（uint64）
func AvgUint64(data []uint64) uint64 {
	return SumUint64(data) / uint64(len(data))
}

// ThanPastUint64 計算最後一個數值是否為過去幾天最大或最小（uint64）
func ThanPastUint64(data []uint64, days int, max bool) bool {
	var dataFloat64 = make([]float64, days+1)
	for i, v := range data[len(data)-1-days:] {
		dataFloat64[i] = float64(v)
	}
	return thanPast(dataFloat64, max)
}

// ThanPastFloat64 計算最後一個數值是否為過去幾天最大或最小（float64）
func ThanPastFloat64(data []float64, days int, max bool) bool {
	return thanPast(data[len(data)-1-days:], max)
}

func thanPast(data []float64, max bool) bool {
	//var dataFloat64 []float64
	//dataFloat64 = make([]float64, days+1)
	//for i, v := range data[len(data)-1-days : len(data)-1] {
	//	switch v.(type) {
	//	case int64:
	//		dataFloat64[i] = float64(v.(int64))
	//	case float64:
	//		dataFloat64[i] = v.(float64)
	//	}
	//}

	var base = data[len(data)-1]
	var condition func(b float64) bool

	if max {
		condition = func(b float64) bool { return base > b }
	} else {
		condition = func(b float64) bool { return base < b }
	}

	for _, v := range data[:len(data)-1] {
		if condition(v) {
			continue
		} else {
			return false
		}
	}
	return true
}
<commit_msg>Fixed `TWSELISTCSV` line.<commit_after>// Package utils - some utils.
package utils

import (
	"regexp"
	"strconv"
	"time"
)

// ExchangeMap is simple to check tse or otc.
var ExchangeMap = map[string]bool{"tse": true, "otc": true}

// TaipeiTimeZone is for time.Data() setting.
var TaipeiTimeZone = time.FixedZone("Asia/Taipei", 8*3600)

// TWSE base url.
const (
	TWSEURL     string = "http://mis.tse.com.tw"
	TWSEHOST    string = "http://www.twse.com.tw"
	OTCHOST     string = "http://www.tpex.org.tw"
	OTCCSV      string = "/ch/stock/aftertrading/daily_trading_info/st43_download.php?d=%d/%02d&stkno=%s&r=%d" // year, mon, stock, rand
	TWSECSV     string = "/ch/trading/exchange/STOCK_DAY/STOCK_DAY_print.php?genpage=genpage/Report%d%02d/%d%02d_F3_1_8_%s.php&type=csv&r=%d"
	TWSELISTCSV string = "/ch/trading/exchange/MI_INDEX/MI_INDEX.php" // year, mon, day, type
	TWSEREAL    string = "/stock/api/getStockInfo.jsp?ex_ch=%s_%s.tw_%s&json=1&delay=0&_=%d"
)

// RandInt return random int.
func RandInt() int {
	return time.Now().Nanosecond()
}

//func RandInt() int64 {
//	result, _ := rand.Int(rand.Reader, big.NewInt(102400))
//	return result.Int64()
//}

//func RandInt() int64 {
//	r := rand.New(rand.NewSource(time.Now().UnixNano()))
//	return r.Int63n(100000)
//}

var dateReg = regexp.MustCompile(`([\d]{2,})/([\d]{1,2})/([\d]{1,2})`)

// ParseDate is to parse "104/01/13" format.
func ParseDate(strDate string) time.Time {
	p := dateReg.FindStringSubmatch(strDate)
	year, _ := strconv.Atoi(p[1])
	mon, _ := strconv.Atoi(p[2])
	day, _ := strconv.Atoi(p[3])
	return time.Date(year+1911, time.Month(mon), day, 0, 0, 0, 0, TaipeiTimeZone)
}

// SumFloat64 計算總和（float64）
func SumFloat64(data []float64) float64 {
	var result float64
	for _, v := range data {
		result += v
	}
	return result
}

// AvgFlast64 計算平均（float64）
func AvgFlast64(data []float64) float64 {
	return float64(int(SumFloat64(data)*100)/len(data)) / 100
}

// SumUint64 計算總和（uint64）
func SumUint64(data []uint64) uint64 {
	var result uint64
	for _, v := range data {
		result += v
	}
	return result
}

// AvgUint64 計算平均（uint64）
func AvgUint64(data []uint64) uint64 {
	return SumUint64(data) / uint64(len(data))
}

// ThanPastUint64 計算最後一個數值是否為過去幾天最大或最小（uint64）
func ThanPastUint64(data []uint64, days int, max bool) bool {
	var dataFloat64 = make([]float64, days+1)
	for i, v := range data[len(data)-1-days:] {
		dataFloat64[i] = float64(v)
	}
	return thanPast(dataFloat64, max)
}

// ThanPastFloat64 計算最後一個數值是否為過去幾天最大或最小（float64）
func ThanPastFloat64(data []float64, days int, max bool) bool {
	return thanPast(data[len(data)-1-days:], max)
}

func thanPast(data []float64, max bool) bool {
	//var dataFloat64 []float64
	//dataFloat64 = make([]float64, days+1)
	//for i, v := range data[len(data)-1-days : len(data)-1] {
	//	switch v.(type) {
	//	case int64:
	//		dataFloat64[i] = float64(v.(int64))
	//	case float64:
	//		dataFloat64[i] = v.(float64)
	//	}
	//}

	var base = data[len(data)-1]
	var condition func(b float64) bool

	if max {
		condition = func(b float64) bool { return base > b }
	} else {
		condition = func(b float64) bool { return base < b }
	}

	for _, v := range data[:len(data)-1] {
		if condition(v) {
			continue
		} else {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package neutrino

import (
	"container/heap"
	"github.com/roasbeef/btcd/wire"
	"sync"
)

type OutPointHint struct {
	outPoint   *wire.OutPoint
	heightHint uint32
}

func NewOutPointHint(outPoint *wire.OutPoint, heightHint uint32) OutPointHint {
	return OutPointHint{
		outPoint:   outPoint,
		heightHint: heightHint,
	}
}

type GetUtxoRequest struct {
	OutPoint OutPointHint
	Result   func(SpendReport)
}

type Item struct {
	value    GetUtxoRequest
	priority int
	// The index is needed by update and is maintained by the heap.Interface methods.
	index int // The index of the item in the heap.
}

// A PriorityQueue implements heap.Interface and holds Items.
type PriorityQueue []*Item

func (pq PriorityQueue) Len() int { return len(pq) }

func (pq PriorityQueue) Less(i, j int) bool {
	return pq[i].priority < pq[j].priority
}

func (pq PriorityQueue) Swap(i, j int) {
	pq[i], pq[j] = pq[j], pq[i]
	pq[i].index = i
	pq[j].index = j
}

func (pq *PriorityQueue) Push(x interface{}) {
	n := len(*pq)
	item := x.(*Item)
	item.index = n
	*pq = append(*pq, item)
}

func (pq *PriorityQueue) Pop() interface{} {
	old := *pq
	n := len(old)
	item := old[n-1]
	item.index = -1
	*pq = old[0 : n-1]
	return item
}

type UtxoScanner struct {
	pq PriorityQueue

	cv       *sync.Cond
	queueMut sync.Mutex
}

func NewUtxoScanner() UtxoScanner {
	var queueMut sync.Mutex
	return UtxoScanner{
		pq:       make(PriorityQueue, 0),
		cv:       sync.NewCond(&queueMut),
		queueMut: queueMut,
	}
}

func (s *UtxoScanner) Start() {
	go func() {
		// TODO(simon): Block until an item is added to the queue
	}()
}

// Schedules a new batch if one isn't already running.
func (s *UtxoScanner) schedule() {
}

func (s *UtxoScanner) Stop() {
	// TODO(simon): Stop any in-progress scan.
}

func (s *UtxoScanner) Enqueue(req GetUtxoRequest) {
	s.queueMut.Lock()
	defer s.queueMut.Unlock()
	// TODO(simon): Insert the request into the queue.
	 heap.Push(&s.pq, req)
	// TODO(simon): Signal CV
}
<commit_msg>utxoscanner: Add methods for queuing requests<commit_after>package neutrino

import (
	"container/heap"
	"github.com/roasbeef/btcd/wire"
	"sync"
)

type GetUtxoRequest struct {
	OutPoint    *wire.OutPoint
	StartHeight uint32
	Result      func(SpendReport)
}

type Item struct {
	value  GetUtxoRequest
	height uint32
}

// A PriorityQueue implements heap.Interface and holds Items. The queue
// maintains that Pop() will always return the GetUtxo request with the greatest
// starting height. This allows us to add new GetUtxo requests to an already
// running batch that's still in the process of catching up to the start height
// of the request.
type PriorityQueue []*Item

func (pq PriorityQueue) Len() int { return len(pq) }

func (pq PriorityQueue) Less(i, j int) bool {
	return pq[i].height > pq[j].height
}

func (pq PriorityQueue) Swap(i, j int) {
	pq[i], pq[j] = pq[j], pq[i]
}

func (pq *PriorityQueue) Push(x interface{}) {
	item := x.(*Item)
	item.height = item.value.StartHeight
	*pq = append(*pq, item)
}

func (pq *PriorityQueue) Peek() *Item {
	return (*pq)[len(*pq)-1]
}

// Pop returns the greatest height GetUtxo request from the queue, removing it
// from the queue in the process.
func (pq *PriorityQueue) Pop() interface{} {
	old := *pq
	n := len(old)
	item := old[n-1]
	*pq = old[0 : n-1]
	return item
}

func (pq *PriorityQueue) IsEmpty() bool {
	return pq.Len() == 0
}

type UtxoScanner struct {
	pq      PriorityQueue
	stopped bool

	cv *sync.Cond
}

func NewUtxoScanner() UtxoScanner {
	return UtxoScanner{
		pq:      make(PriorityQueue, 0),
		cv:      sync.NewCond(&sync.Mutex{}),
		stopped: false,
	}
}

func (s *UtxoScanner) Start() {
	go func() {
		for {
			s.cv.L.Lock()
			if s.stopped {
				s.cv.L.Unlock()
				return
			}
			s.cv.L.Unlock()

			s.runBatch()
		}
	}()
}

// drain removes all items currently in the queue and returns a slice. If there
// are no items in the queue, this method blocks until one is available. The
// requests are returned in descending height order.
func (s *UtxoScanner) drain() []GetUtxoRequest {
	s.cv.L.Lock()
	defer s.cv.L.Unlock()

	// Block until the queue is no longer empty.
	for !s.stopped && s.pq.IsEmpty() {
		s.cv.Wait()
	}

	// We return nil only in the case that we've been interrupted, so callers
	// can use this to determine that the UtxoScanner is shutting down.
	if s.stopped {
		return nil
	}

	// Empty the queue.
	var requests []GetUtxoRequest
	for !s.pq.IsEmpty() {
		item := s.pq.Pop().(*Item)
		requests = append(requests, item.value)
	}

	return requests
}

// Returns all GetUtxo requests that have starting height greater than the given
// height.
func (s *UtxoScanner) getAfterHeight(height uint32) []GetUtxoRequest {
	s.cv.L.Lock()
	defer s.cv.L.Unlock()

	var requests []GetUtxoRequest
	for !s.pq.IsEmpty() && s.pq.Peek().height > height {
		item := s.pq.Pop().(*Item)
		requests = append(requests, item.value)
	}
	return requests
}

// Runs a single batch.
func (s *UtxoScanner) runBatch() {
	// Fetch all queued requests, blocking if the queue is empty.
	requests := s.drain()

	if requests == nil {
		return
	}

	startHeight := requests[len(requests)-1].StartHeight
	bestHeight := uint32(500000)

	for height := startHeight; startHeight < bestHeight; height++ {
		// If there are any new requests that can safely be added to this batch,
		// then try and fetch them.
		reqs := s.getAfterHeight(height)

		if len(reqs) > 0 {
			for _, req := range reqs {
				requests = append(requests, req)
			}
			// TODO(simon): Re-calculate filters.
		}

		// TODO(simon): Actual scanning here.
	}
}

// Stop any in-progress scan.
func (s *UtxoScanner) Stop() {
	s.cv.L.Lock()
	defer s.cv.L.Unlock()

	s.stopped = true
}

func (s *UtxoScanner) Enqueue(req GetUtxoRequest) {
	s.cv.L.Lock()
	defer s.cv.L.Unlock()

	// Insert the request into the queue and signal any threads that might be
	// waiting for new elements.
	heap.Push(&s.pq, req)
	s.cv.Signal()
}
<|endoftext|>


<commit_before>// Package colonycore define the struct that needed by other package in colony
package colonycore

import (
	/*"github.com/eaciit/errorlib"
		// "github.com/eaciit/orm/v1"
		// "log"
		"mime/multipart"
		"strconv"
		"strings"
	    "github.com/eaciit/toolkit"
	*/
	"time"

	"github.com/eaciit/orm/v1"
	"github.com/eaciit/toolkit"
	tk "github.com/eaciit/toolkit"
)

const (
	ACTION_TYPE_HIVE     = "HIVE"
	ACTION_TYPE_HDFS     = "HDFS"
	ACTION_TYPE_SPARK    = "SPARK"
	ACTION_TYPE_DECISION = "DECISION"
	ACTION_TYPE_SSH      = "SSH"
	ACTION_TYPE_KAFKA    = "KAFKA"

	FORK_TYPE_ALL       = "ALL"
	FORK_TYPE_ONE       = "ONE"
	FORK_TYPE_MANDATORY = "MANDATORY"

	SSH_OPERATION_MKDIR = "MKDIR"
	// SSH_OPERATION_* please define
)

// DataFlow to define the flow name and description
// and also have the list of the actions inside the flow
// Actions list of action, the content of the action can be FlowAction or "list of FlowAction" -> for fork action
type DataFlow struct {
	orm.ModelBase
	ID           string       `json:"_id",bson:"_id"`
	Name         string       `json:"name"`
	Description  string       `json:"description"`
	CreatedDate  time.Time    `json:"createddate"`
	LastModified time.Time    `json:"lastmodified"`
	CreatedBy    string       `json:"createdby"`
	Actions      []FlowAction `json:"actions"`
	DataShapes   tk.M         `json:"datashapes"`
	GlobalParam  tk.M
}

/*// ActionBridge, to define the input and output of the previous action and next action
// PrevAction, ID of the previouse action
// NextAction, ID of the next action
// MappingParam, define the output and input paremeter name of the previous and next action
//      e.g.:
//              OUTPUT                INPUT
//              "id"                : "id"
//              "name"              : "name"
//              "address"           : "note"
//              "tag"               : "global.tag"
//              "global.address"    : "address"
type Bridge struct {
	Id           string
	Type         string
	PrevAction   string
	NextAction   string
	MappingParam tk.M
}*/

// FlowAction define the action that exist
// Type refer to cons ACTION_TYPE_*
// Action define the type of action e.g. Spark, HDFS, etc
// Server define which server for the action
// OK next step to go if the result is OK
// KO next step to go if the result is KO or NOT OK or ERROR
// Retry how many time the system will do the process if ERROR happen and then go to KO condition, default will be 3
// Wait interval in minute, default will be 1
// InputParam, will be the list of input that needed by the action
// OutputParam, will be the list of output from the action
// OutputType, will be the type of the output (text, json, *sv, xml)
type FlowAction struct {
	Id          string      `json:"_id"`
	Name        string      `json:"name"`
	Description string      `json:"description"`
	Type        string      `json:"type"`
	Server      Server      `json:"server"`
	Action      interface{} `json:"action"`
	OK          []string    `json:"ok"`
	KO          []string    `json:"ko"`
	Retry       int         `json:"retry"`
	Interval    int         `json:"interval"`
	FirstAction bool
	InputParam  tk.M
	OutputParam tk.M
	OutputType  string
}

// ActionHive action for HIVE
// ScriptPath path of the hive script to run
type ActionHive struct {
	ScriptPath string
	Params     []string
}

// ActionHDFS action for HDFS type for server
// Operation type operation
// The HDFS connection will be provided by the server from FlowAction
// Path can be used for the current path, or old path, also can included the filename
// NewPath can be used for move, copy, etc
// Permission e.g. rwx-r--r-
type ActionHDFS struct {
	/*Operation  string `json:"operation"`
	Path       string `json:"path"`
	NewPath    string `json:"newpath"`
	Permission string
	User       string
	Group      string*/
	Command string
}

// ActionSSH action for SSH type of server
// Operation type operation
// The ssh connection will be provided by the server from FlowAction
// Path can be used for the current path, or old path, also can included the filename
// NewPath can be used for move, copy, etc
// Permission e.g. rwx-r--r-
type ActionSSH struct {
	Command string
}

// ActionSpark action for SPARK
// Type of the code upload e.g. Java, Scala
// Mode e.g. client, cluster
// Args for other arguments, e.g. --executor-memory 2G
// Application the spark code will be mantain as application
type ActionSpark struct {
	Type   string `json: "type"`
	Master string
	Mode   string
	File   string
	// AppName   string
	MainClass string
	// Args      []string
	Args string
}

// ActionSpark action for Application
// Type e.g. Java, Scala, GO
// Application refer to application page
// when creating the action with type = application then should follow the application screen standard
type ActionApplication struct {
	Type        string `json: "type"`
	Application Application
}

// ActionHadoopStreaming for hadoop streaming
// Mapper name
// Reducer name
// Input can be folder or file
// Output folder
// Files list of file inside the hdfs
// the server type should be hdfs
type ActionHadoopStreaming struct {
	Jar     string
	Mapper  string
	Reducer string
	Input   string
	Output  string
	Files   []string
	// Params  []string
	Params string
}

// ActionKafka action for KAFKA
type ActionKafka struct {
	// other field will be different for each action type
}

/*
Doesn't need the Queestion, can be achived by using OK and KO or using decision
type ActionQuestion struct {
	Question string `json:"question"`
	Yes      string `json:"yes"`
	No       string `json:"no"`
	// other field will be different for each action type
}*/

// ActionDecision action to define the list of decision that action have
type ActionDecision struct {
	Conditions []Condition `json:"conditions"`
	// other field will be different for each action type
}

// Condition condition for the action decision
type Condition struct {
	Result     string `json:"result"`
	FlowAction string `json:"flowaction"`
}

type ActionStop struct {
	Message string `json:"message"`
}

type ActionShellScript struct {
	Script string `json:"script"`
}

type ActionJavaApp struct {
	Jar string `json:"jar"`
}

type ActionEmail struct {
	To      string `json:"to"`
	Subject string `json:"subject"`
	Body    string `json:"body"`
}

// ActionFork to define the forking condition
// Type refer to cons FORK_TYPE_*
type ActionFork struct {
	Actions []FlowAction `json:"actions"`
	Type    string       `json: "type"`
}

type DataFlowProcess struct {
	orm.ModelBase
	Id          string `json:"_id"`
	Flow        DataFlow
	Steps       []FlowAction
	StartDate   time.Time
	EndDate     time.Time
	StartedBy   string
	GlobalParam tk.M
}

func (c *DataFlow) TableName() string {
	return "dataflow"
}

func (c *DataFlow) RecordID() interface{} {
	return c.ID
}

type ActionContext struct {
	Keys  interface{}
	Infos []toolkit.M
}
<commit_msg>bug fixing<commit_after>// Package colonycore define the struct that needed by other package in colony
package colonycore

import (
	/*"github.com/eaciit/errorlib"
		// "github.com/eaciit/orm/v1"
		// "log"
		"mime/multipart"
		"strconv"
		"strings"
	    "github.com/eaciit/toolkit"
	*/
	"time"

	"github.com/eaciit/orm/v1"
	tk "github.com/eaciit/toolkit"
)

const (
/*ACTION_TYPE_HIVE     = "HIVE"
ACTION_TYPE_HDFS     = "HDFS"
ACTION_TYPE_SPARK    = "SPARK"
ACTION_TYPE_DECISION = "DECISION"
ACTION_TYPE_SSH      = "SSH"
ACTION_TYPE_KAFKA    = "KAFKA"

FORK_TYPE_ALL       = "ALL"
FORK_TYPE_ONE       = "ONE"
FORK_TYPE_MANDATORY = "MANDATORY"

SSH_OPERATION_MKDIR = "MKDIR"*/
// SSH_OPERATION_* please define
)

// DataFlow to define the flow name and description
// and also have the list of the actions inside the flow
// Actions list of action, the content of the action can be FlowAction or "list of FlowAction" -> for fork action
type DataFlow struct {
	orm.ModelBase
	ID           string       `json:"_id",bson:"_id"`
	Name         string       `json:"name"`
	Description  string       `json:"description"`
	CreatedDate  time.Time    `json:"createddate"`
	LastModified time.Time    `json:"lastmodified"`
	CreatedBy    string       `json:"createdby"`
	Actions      []FlowAction `json:"actions"`
	DataShapes   tk.M         `json:"datashapes"`
	GlobalParam  tk.M
}

func (c *DataFlow) TableName() string {
	return "dataflow"
}

func (c *DataFlow) RecordID() interface{} {
	return c.ID
}

/*// ActionBridge, to define the input and output of the previous action and next action
// PrevAction, ID of the previouse action
// NextAction, ID of the next action
// MappingParam, define the output and input paremeter name of the previous and next action
//      e.g.:
//              OUTPUT                INPUT
//              "id"                : "id"
//              "name"              : "name"
//              "address"           : "note"
//              "tag"               : "global.tag"
//              "global.address"    : "address"
type Bridge struct {
	Id           string
	Type         string
	PrevAction   string
	NextAction   string
	MappingParam tk.M
}*/

// FlowAction define the action that exist
// Type refer to cons ACTION_TYPE_*
// Action define the type of action e.g. Spark, HDFS, etc
// Server define which server for the action
// OK next step to go if the result is OK
// KO next step to go if the result is KO or NOT OK or ERROR
// Retry how many time the system will do the process if ERROR happen and then go to KO condition, default will be 3
// Wait interval in minute, default will be 1
// InputParam, will be the list of input that needed by the action
// OutputParam, will be the list of output from the action
// OutputType, will be the type of the output (text, json, *sv, xml)
// OutputPath, path of the output in the hdfs if empty then consider as stdout
type FlowAction struct {
	Id          string      `json:"_id"`
	Name        string      `json:"name"`
	Description string      `json:"description"`
	Type        string      `json:"type"`
	Server      Server      `json:"server"`
	Action      interface{} `json:"action"`
	OK          []string    `json:"ok"`
	KO          []string    `json:"ko"`
	Retry       int         `json:"retry"`
	Interval    int         `json:"interval"`
	FirstAction bool
	InputParam  tk.M
	OutputParam tk.M
	OutputType  string
	OutputPath  string
}

// ActionHive action for HIVE
// ScriptPath path of the hive script to run
type ActionHive struct {
	ScriptPath string
	Params     []string
}

// ActionHDFS action for HDFS type for server
// Operation type operation
// The HDFS connection will be provided by the server from FlowAction
// Path can be used for the current path, or old path, also can included the filename
// NewPath can be used for move, copy, etc
// Permission e.g. rwx-r--r-
type ActionHDFS struct {
	/*Operation  string `json:"operation"`
	Path       string `json:"path"`
	NewPath    string `json:"newpath"`
	Permission string
	User       string
	Group      string*/
	Command string
}

// ActionSSH action for SSH type of server
// Operation type operation
// The ssh connection will be provided by the server from FlowAction
// Path can be used for the current path, or old path, also can included the filename
// NewPath can be used for move, copy, etc
// Permission e.g. rwx-r--r-
type ActionSSH struct {
	Command string
}

// ActionSpark action for SPARK
// Type of the code upload e.g. Java, Scala
// Mode e.g. client, cluster
// Args for other arguments, e.g. --executor-memory 2G
// Application the spark code will be mantain as application
type ActionSpark struct {
	Type   string `json: "type"`
	Master string
	Mode   string
	File   string
	// AppName   string
	MainClass string
	// Args      []string
	Args string
}

// ActionSpark action for Application
// Type e.g. Java, Scala, GO
// Application refer to application page
// when creating the action with type = application then should follow the application screen standard
type ActionApplication struct {
	Type        string `json: "type"`
	Application Application
}

// ActionHadoopStreaming for hadoop streaming
// Mapper name
// Reducer name
// Input can be folder or file
// Output folder
// Files list of file inside the hdfs
// the server type should be hdfs
type ActionHadoopStreaming struct {
	Jar     string
	Mapper  string
	Reducer string
	Input   string
	Output  string
	Files   []string
	// Params  []string
	Params string
}

// ActionKafka action for KAFKA
type ActionKafka struct {
	// other field will be different for each action type
}

/*
Doesn't need the Queestion, can be achived by using OK and KO or using decision
type ActionQuestion struct {
	Question string `json:"question"`
	Yes      string `json:"yes"`
	No       string `json:"no"`
	// other field will be different for each action type
}*/

// ActionDecision action to define the list of decision that action have
type ActionDecision struct {
	Conditions []Condition `json:"conditions"`
	// other field will be different for each action type
}

// Condition condition for the action decision
type Condition struct {
	Result     string `json:"result"`
	FlowAction string `json:"flowaction"`
}

type ActionStop struct {
	Message string `json:"message"`
}

type ActionShellScript struct {
	Script string `json:"script"`
}

type ActionJavaApp struct {
	Jar string `json:"jar"`
}

type ActionEmail struct {
	To      string `json:"to"`
	Subject string `json:"subject"`
	Body    string `json:"body"`
}

// ActionFork to define the forking condition
// Type refer to cons FORK_TYPE_*
type ActionFork struct {
	Actions []FlowAction `json:"actions"`
	Type    string       `json: "type"`
}

type DataFlowProcess struct {
	orm.ModelBase
	Id          string `json:"_id"`
	Flow        DataFlow
	Steps       []FlowAction
	StartDate   time.Time
	EndDate     time.Time
	StartedBy   string
	GlobalParam tk.M
}

func (c *DataFlowProcess) TableName() string {
	return "dataflowprocess"
}

func (c *DataFlowProcess) RecordID() interface{} {
	return c.Id
}
<|endoftext|>


<commit_before>package kv

import (
	"errors"
	"fmt"
	"strings"
	"text/template"
	"time"

	"github.com/BurntSushi/ty/fun"
	"github.com/cenk/backoff"
	"github.com/containous/traefik/job"
	"github.com/containous/traefik/log"
	"github.com/containous/traefik/provider"
	"github.com/containous/traefik/safe"
	"github.com/containous/traefik/types"
	"github.com/docker/libkv"
	"github.com/docker/libkv/store"
)

// Provider holds common configurations of key-value providers.
type Provider struct {
	provider.BaseProvider `mapstructure:",squash"`
	Endpoint              string              `description:"Comma sepparated server endpoints"`
	Prefix                string              `description:"Prefix used for KV store"`
	TLS                   *provider.ClientTLS `description:"Enable TLS support"`
	StoreType             store.Backend
	Kvclient              store.Store
}

// CreateStore create the K/V store
func (p *Provider) CreateStore() (store.Store, error) {
	storeConfig := &store.Config{
		ConnectionTimeout: 30 * time.Second,
		Bucket:            "traefik",
	}

	if p.TLS != nil {
		var err error
		storeConfig.TLS, err = p.TLS.CreateTLSConfig()
		if err != nil {
			return nil, err
		}
	}
	return libkv.NewStore(
		p.StoreType,
		strings.Split(p.Endpoint, ","),
		storeConfig,
	)
}

func (p *Provider) watchKv(configurationChan chan<- types.ConfigMessage, prefix string, stop chan bool) error {
	operation := func() error {
		events, err := p.Kvclient.WatchTree(p.Prefix, make(chan struct{}))
		if err != nil {
			return fmt.Errorf("Failed to KV WatchTree: %v", err)
		}
		for {
			select {
			case <-stop:
				return nil
			case _, ok := <-events:
				if !ok {
					return errors.New("watchtree channel closed")
				}
				configuration := p.loadConfig()
				if configuration != nil {
					configurationChan <- types.ConfigMessage{
						ProviderName:  string(p.StoreType),
						Configuration: configuration,
					}
				}
			}
		}
	}

	notify := func(err error, time time.Duration) {
		log.Errorf("KV connection error: %+v, retrying in %s", err, time)
	}
	err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify)
	if err != nil {
		return fmt.Errorf("Cannot connect to KV server: %v", err)
	}
	return nil
}

// Provide provides the configuration to traefik via the configuration channel
func (p *Provider) Provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool, constraints types.Constraints) error {
	p.Constraints = append(p.Constraints, constraints...)
	operation := func() error {
		if _, err := p.Kvclient.Exists("qmslkjdfmqlskdjfmqlksjazçueznbvbwzlkajzebvkwjdcqmlsfj"); err != nil {
			return fmt.Errorf("Failed to test KV store connection: %v", err)
		}
		if p.Watch {
			pool.Go(func(stop chan bool) {
				err := p.watchKv(configurationChan, p.Prefix, stop)
				if err != nil {
					log.Errorf("Cannot watch KV store: %v", err)
				}
			})
		}
		configuration := p.loadConfig()
		configurationChan <- types.ConfigMessage{
			ProviderName:  string(p.StoreType),
			Configuration: configuration,
		}
		return nil
	}
	notify := func(err error, time time.Duration) {
		log.Errorf("KV connection error: %+v, retrying in %s", err, time)
	}
	err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify)
	if err != nil {
		return fmt.Errorf("Cannot connect to KV server: %v", err)
	}
	return nil
}

func (p *Provider) loadConfig() *types.Configuration {
	templateObjects := struct {
		Prefix string
	}{
		// Allow `/traefik/alias` to superesede `p.Prefix`
		strings.TrimSuffix(p.get(p.Prefix, p.Prefix+"/alias"), "/"),
	}

	var KvFuncMap = template.FuncMap{
		"List":        p.list,
		"ListServers": p.listServers,
		"Get":         p.get,
		"SplitGet":    p.splitGet,
		"Last":        p.last,
	}

	configuration, err := p.GetConfiguration("templates/kv.tmpl", KvFuncMap, templateObjects)
	if err != nil {
		log.Error(err)
	}

	for key, frontend := range configuration.Frontends {
		if _, ok := configuration.Backends[frontend.Backend]; ok == false {
			delete(configuration.Frontends, key)
		}
	}

	return configuration
}

func (p *Provider) list(keys ...string) []string {
	joinedKeys := strings.Join(keys, "")
	keysPairs, err := p.Kvclient.List(joinedKeys)
	if err != nil {
		log.Debugf("Cannot get keys %s %s ", joinedKeys, err)
		return nil
	}
	directoryKeys := make(map[string]string)
	for _, key := range keysPairs {
		directory := strings.Split(strings.TrimPrefix(key.Key, joinedKeys), "/")[0]
		directoryKeys[directory] = joinedKeys + directory
	}
	return fun.Values(directoryKeys).([]string)
}

func (p *Provider) listServers(backend string) []string {
	serverNames := p.list(backend, "/servers/")
	return fun.Filter(func(serverName string) bool {
		key := fmt.Sprint(serverName, "/url")
		if _, err := p.Kvclient.Get(key); err != nil {
			if err != store.ErrKeyNotFound {
				log.Errorf("Failed to retrieve value for key %s: %s", key, err)
			}
			return false
		}
		return p.checkConstraints(serverName, "/tags")
	}, serverNames).([]string)
}

func (p *Provider) get(defaultValue string, keys ...string) string {
	joinedKeys := strings.Join(keys, "")
	keyPair, err := p.Kvclient.Get(strings.TrimPrefix(joinedKeys, "/"))
	if err != nil {
		log.Debugf("Cannot get key %s %s, setting default %s", joinedKeys, err, defaultValue)
		return defaultValue
	} else if keyPair == nil {
		log.Debugf("Cannot get key %s, setting default %s", joinedKeys, defaultValue)
		return defaultValue
	}
	return string(keyPair.Value)
}

func (p *Provider) splitGet(keys ...string) []string {
	joinedKeys := strings.Join(keys, "")
	keyPair, err := p.Kvclient.Get(joinedKeys)
	if err != nil {
		log.Debugf("Cannot get key %s %s, setting default empty", joinedKeys, err)
		return []string{}
	} else if keyPair == nil {
		log.Debugf("Cannot get key %s, setting default %empty", joinedKeys)
		return []string{}
	}
	return strings.Split(string(keyPair.Value), ",")
}

func (p *Provider) last(key string) string {
	splittedKey := strings.Split(key, "/")
	return splittedKey[len(splittedKey)-1]
}

func (p *Provider) checkConstraints(keys ...string) bool {
	joinedKeys := strings.Join(keys, "")
	keyPair, err := p.Kvclient.Get(joinedKeys)

	value := ""
	if err == nil && keyPair != nil && keyPair.Value != nil {
		value = string(keyPair.Value)
	}

	constraintTags := strings.Split(value, ",")
	ok, failingConstraint := p.MatchConstraints(constraintTags)
	if ok == false {
		if failingConstraint != nil {
			log.Debugf("Constraint %v not matching with following tags: %v", failingConstraint.String(), value)
		}
		return false
	}
	return true
}
<commit_msg>Fix typo in command line help.<commit_after>package kv

import (
	"errors"
	"fmt"
	"strings"
	"text/template"
	"time"

	"github.com/BurntSushi/ty/fun"
	"github.com/cenk/backoff"
	"github.com/containous/traefik/job"
	"github.com/containous/traefik/log"
	"github.com/containous/traefik/provider"
	"github.com/containous/traefik/safe"
	"github.com/containous/traefik/types"
	"github.com/docker/libkv"
	"github.com/docker/libkv/store"
)

// Provider holds common configurations of key-value providers.
type Provider struct {
	provider.BaseProvider `mapstructure:",squash"`
	Endpoint              string              `description:"Comma separated server endpoints"`
	Prefix                string              `description:"Prefix used for KV store"`
	TLS                   *provider.ClientTLS `description:"Enable TLS support"`
	StoreType             store.Backend
	Kvclient              store.Store
}

// CreateStore create the K/V store
func (p *Provider) CreateStore() (store.Store, error) {
	storeConfig := &store.Config{
		ConnectionTimeout: 30 * time.Second,
		Bucket:            "traefik",
	}

	if p.TLS != nil {
		var err error
		storeConfig.TLS, err = p.TLS.CreateTLSConfig()
		if err != nil {
			return nil, err
		}
	}
	return libkv.NewStore(
		p.StoreType,
		strings.Split(p.Endpoint, ","),
		storeConfig,
	)
}

func (p *Provider) watchKv(configurationChan chan<- types.ConfigMessage, prefix string, stop chan bool) error {
	operation := func() error {
		events, err := p.Kvclient.WatchTree(p.Prefix, make(chan struct{}))
		if err != nil {
			return fmt.Errorf("Failed to KV WatchTree: %v", err)
		}
		for {
			select {
			case <-stop:
				return nil
			case _, ok := <-events:
				if !ok {
					return errors.New("watchtree channel closed")
				}
				configuration := p.loadConfig()
				if configuration != nil {
					configurationChan <- types.ConfigMessage{
						ProviderName:  string(p.StoreType),
						Configuration: configuration,
					}
				}
			}
		}
	}

	notify := func(err error, time time.Duration) {
		log.Errorf("KV connection error: %+v, retrying in %s", err, time)
	}
	err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify)
	if err != nil {
		return fmt.Errorf("Cannot connect to KV server: %v", err)
	}
	return nil
}

// Provide provides the configuration to traefik via the configuration channel
func (p *Provider) Provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool, constraints types.Constraints) error {
	p.Constraints = append(p.Constraints, constraints...)
	operation := func() error {
		if _, err := p.Kvclient.Exists("qmslkjdfmqlskdjfmqlksjazçueznbvbwzlkajzebvkwjdcqmlsfj"); err != nil {
			return fmt.Errorf("Failed to test KV store connection: %v", err)
		}
		if p.Watch {
			pool.Go(func(stop chan bool) {
				err := p.watchKv(configurationChan, p.Prefix, stop)
				if err != nil {
					log.Errorf("Cannot watch KV store: %v", err)
				}
			})
		}
		configuration := p.loadConfig()
		configurationChan <- types.ConfigMessage{
			ProviderName:  string(p.StoreType),
			Configuration: configuration,
		}
		return nil
	}
	notify := func(err error, time time.Duration) {
		log.Errorf("KV connection error: %+v, retrying in %s", err, time)
	}
	err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify)
	if err != nil {
		return fmt.Errorf("Cannot connect to KV server: %v", err)
	}
	return nil
}

func (p *Provider) loadConfig() *types.Configuration {
	templateObjects := struct {
		Prefix string
	}{
		// Allow `/traefik/alias` to superesede `p.Prefix`
		strings.TrimSuffix(p.get(p.Prefix, p.Prefix+"/alias"), "/"),
	}

	var KvFuncMap = template.FuncMap{
		"List":        p.list,
		"ListServers": p.listServers,
		"Get":         p.get,
		"SplitGet":    p.splitGet,
		"Last":        p.last,
	}

	configuration, err := p.GetConfiguration("templates/kv.tmpl", KvFuncMap, templateObjects)
	if err != nil {
		log.Error(err)
	}

	for key, frontend := range configuration.Frontends {
		if _, ok := configuration.Backends[frontend.Backend]; ok == false {
			delete(configuration.Frontends, key)
		}
	}

	return configuration
}

func (p *Provider) list(keys ...string) []string {
	joinedKeys := strings.Join(keys, "")
	keysPairs, err := p.Kvclient.List(joinedKeys)
	if err != nil {
		log.Debugf("Cannot get keys %s %s ", joinedKeys, err)
		return nil
	}
	directoryKeys := make(map[string]string)
	for _, key := range keysPairs {
		directory := strings.Split(strings.TrimPrefix(key.Key, joinedKeys), "/")[0]
		directoryKeys[directory] = joinedKeys + directory
	}
	return fun.Values(directoryKeys).([]string)
}

func (p *Provider) listServers(backend string) []string {
	serverNames := p.list(backend, "/servers/")
	return fun.Filter(func(serverName string) bool {
		key := fmt.Sprint(serverName, "/url")
		if _, err := p.Kvclient.Get(key); err != nil {
			if err != store.ErrKeyNotFound {
				log.Errorf("Failed to retrieve value for key %s: %s", key, err)
			}
			return false
		}
		return p.checkConstraints(serverName, "/tags")
	}, serverNames).([]string)
}

func (p *Provider) get(defaultValue string, keys ...string) string {
	joinedKeys := strings.Join(keys, "")
	keyPair, err := p.Kvclient.Get(strings.TrimPrefix(joinedKeys, "/"))
	if err != nil {
		log.Debugf("Cannot get key %s %s, setting default %s", joinedKeys, err, defaultValue)
		return defaultValue
	} else if keyPair == nil {
		log.Debugf("Cannot get key %s, setting default %s", joinedKeys, defaultValue)
		return defaultValue
	}
	return string(keyPair.Value)
}

func (p *Provider) splitGet(keys ...string) []string {
	joinedKeys := strings.Join(keys, "")
	keyPair, err := p.Kvclient.Get(joinedKeys)
	if err != nil {
		log.Debugf("Cannot get key %s %s, setting default empty", joinedKeys, err)
		return []string{}
	} else if keyPair == nil {
		log.Debugf("Cannot get key %s, setting default %empty", joinedKeys)
		return []string{}
	}
	return strings.Split(string(keyPair.Value), ",")
}

func (p *Provider) last(key string) string {
	splittedKey := strings.Split(key, "/")
	return splittedKey[len(splittedKey)-1]
}

func (p *Provider) checkConstraints(keys ...string) bool {
	joinedKeys := strings.Join(keys, "")
	keyPair, err := p.Kvclient.Get(joinedKeys)

	value := ""
	if err == nil && keyPair != nil && keyPair.Value != nil {
		value = string(keyPair.Value)
	}

	constraintTags := strings.Split(value, ",")
	ok, failingConstraint := p.MatchConstraints(constraintTags)
	if ok == false {
		if failingConstraint != nil {
			log.Debugf("Constraint %v not matching with following tags: %v", failingConstraint.String(), value)
		}
		return false
	}
	return true
}
<|endoftext|>


<commit_before>package pb

import (
	"math/rand"
	"sync"
	"text/template"

	"github.com/fatih/color"
)

// ProgressBarTemplate that template string
type ProgressBarTemplate string

// New creates new bar from template
func (pbt ProgressBarTemplate) New(total int) *ProgressBar {
	return New(total).SetTemplate(pbt)
}

// Start64 create and start new bar with given int64 total value
func (pbt ProgressBarTemplate) Start64(total int64) *ProgressBar {
	return New64(total).SetTemplate(pbt).Start()
}

// Start create and start new bar with given int total value
func (pbt ProgressBarTemplate) Start(total int) *ProgressBar {
	return pbt.Start64(int64(total))
}

var templateCacheMu sync.Mutex
var templateCache = make(map[string]*template.Template)

var defaultTemplateFuncs = template.FuncMap{
	// colors
	"black":    color.New(color.FgBlack).SprintFunc(),
	"red":      color.New(color.FgRed).SprintFunc(),
	"green":    color.New(color.FgGreen).SprintFunc(),
	"yellow":   color.New(color.FgYellow).SprintFunc(),
	"blue":     color.New(color.FgBlue).SprintFunc(),
	"magenta":  color.New(color.FgMagenta).SprintFunc(),
	"cyan":     color.New(color.FgCyan).SprintFunc(),
	"white":    color.New(color.FgWhite).SprintFunc(),
	"rndcolor": rndcolor,
	"rnd":      rnd,
}

func getTemplate(tmpl string) (t *template.Template, err error) {
	templateCacheMu.Lock()
	defer templateCacheMu.Unlock()
	t = templateCache[tmpl]
	if t != nil {
		// found in cache
		return
	}
	t = template.New("")
	fillTemplateFuncs(t)
	_, err = t.Parse(tmpl)
	if err != nil {
		t = nil
		return
	}
	templateCache[tmpl] = t
	return
}

func fillTemplateFuncs(t *template.Template) {
	t.Funcs(defaultTemplateFuncs)
	emf := make(template.FuncMap)
	elementsM.Lock()
	for k, v := range elements {
		emf[k] = v
	}
	elementsM.Unlock()
	t.Funcs(emf)
	return
}

func rndcolor(s string) string {
	c := rand.Intn(int(color.FgWhite-color.FgBlack)) + int(color.FgBlack)
	return color.New(color.Attribute(c)).Sprint(s)
}

func rnd(args ...string) string {
	if len(args) == 0 {
		return ""
	}
	return args[rand.Intn(len(args))]
}
<commit_msg>Add 'reset' for more flexibility<commit_after>package pb

import (
	"math/rand"
	"sync"
	"text/template"

	"github.com/fatih/color"
)

// ProgressBarTemplate that template string
type ProgressBarTemplate string

// New creates new bar from template
func (pbt ProgressBarTemplate) New(total int) *ProgressBar {
	return New(total).SetTemplate(pbt)
}

// Start64 create and start new bar with given int64 total value
func (pbt ProgressBarTemplate) Start64(total int64) *ProgressBar {
	return New64(total).SetTemplate(pbt).Start()
}

// Start create and start new bar with given int total value
func (pbt ProgressBarTemplate) Start(total int) *ProgressBar {
	return pbt.Start64(int64(total))
}

var templateCacheMu sync.Mutex
var templateCache = make(map[string]*template.Template)

var defaultTemplateFuncs = template.FuncMap{
	// colors
	"black":    color.New(color.FgBlack).SprintFunc(),
	"red":      color.New(color.FgRed).SprintFunc(),
	"green":    color.New(color.FgGreen).SprintFunc(),
	"yellow":   color.New(color.FgYellow).SprintFunc(),
	"blue":     color.New(color.FgBlue).SprintFunc(),
	"magenta":  color.New(color.FgMagenta).SprintFunc(),
	"cyan":     color.New(color.FgCyan).SprintFunc(),
	"white":    color.New(color.FgWhite).SprintFunc(),
	"reset":    color.New(color.Reset).SprintFunc(),
	"rndcolor": rndcolor,
	"rnd":      rnd,
}

func getTemplate(tmpl string) (t *template.Template, err error) {
	templateCacheMu.Lock()
	defer templateCacheMu.Unlock()
	t = templateCache[tmpl]
	if t != nil {
		// found in cache
		return
	}
	t = template.New("")
	fillTemplateFuncs(t)
	_, err = t.Parse(tmpl)
	if err != nil {
		t = nil
		return
	}
	templateCache[tmpl] = t
	return
}

func fillTemplateFuncs(t *template.Template) {
	t.Funcs(defaultTemplateFuncs)
	emf := make(template.FuncMap)
	elementsM.Lock()
	for k, v := range elements {
		emf[k] = v
	}
	elementsM.Unlock()
	t.Funcs(emf)
	return
}

func rndcolor(s string) string {
	c := rand.Intn(int(color.FgWhite-color.FgBlack)) + int(color.FgBlack)
	return color.New(color.Attribute(c)).Sprint(s)
}

func rnd(args ...string) string {
	if len(args) == 0 {
		return ""
	}
	return args[rand.Intn(len(args))]
}
<|endoftext|>