<commit_before>package server

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/Sirupsen/logrus"
	"github.com/gin-gonic/gin"
	"github.com/jingweno/jqplay/jq"
)

const (
	JSONPayloadLimit   = JSONPayloadLimitMB * OneMB
	JSONPayloadLimitMB = 10
	OneMB              = 1024000
)

type JQHandlerContext struct {
	*Config
	JQ string
}

func (c *JQHandlerContext) Asset(path string) string {
	return fmt.Sprintf("%s/%s", c.Config.AssetHost, path)
}

func (c *JQHandlerContext) ShouldInitJQ() bool {
	return c.JQ != ""
}

type JQHandler struct {
	c *Config
}

func (h *JQHandler) handleIndex(c *gin.Context) {
	c.HTML(200, "index.tmpl", &JQHandlerContext{Config: h.c})
}

func (h *JQHandler) handleJqPost(c *gin.Context) {
	l, _ := c.Get("logger")
	logger := l.(logrus.Entry)

	if c.Request.ContentLength > JSONPayloadLimit {
		size := float64(c.Request.ContentLength) / OneMB
		err := fmt.Errorf("JSON payload size is %.1fMB, larger than limit %dMB.", size, JSONPayloadLimitMB)
		logger.WithError(err).WithField("size", size).Infof(err.Error())
		c.String(http.StatusExpectationFailed, err.Error())
		return
	}

	c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, JSONPayloadLimit)

	var jq *jq.JQ
	err := c.BindJSON(&jq)
	if err != nil {
		err = fmt.Errorf("error parsing JSON: %s", err)
		logger.WithError(err).Infof("error parsing JSON: %s", err)
		c.String(422, err.Error())
		return
	}

	// Evaling into ResponseWriter sets the status code to 200
	// appending error message in the end if there's any
	err = jq.Eval(c.Writer)
	if err != nil {
		logger.WithError(err).Infof("error evaluating jq query: %s", err)
		fmt.Fprint(c.Writer, err.Error())
	}
}

func (h *JQHandler) handleJqGet(c *gin.Context) {
	jq := &jq.JQ{
		J: c.Query("j"),
		Q: c.Query("q"),
	}

	var jqData string
	if err := jq.Validate(); err == nil {
		d, err := json.Marshal(jq)
		if err == nil {
			jqData = string(d)
		}
	}

	c.HTML(200, "index.tmpl", &JQHandlerContext{Config: h.c, JQ: jqData})
}
<commit_msg>Fix typo<commit_after>package server

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/Sirupsen/logrus"
	"github.com/gin-gonic/gin"
	"github.com/jingweno/jqplay/jq"
)

const (
	JSONPayloadLimit   = JSONPayloadLimitMB * OneMB
	JSONPayloadLimitMB = 10
	OneMB              = 1024000
)

type JQHandlerContext struct {
	*Config
	JQ string
}

func (c *JQHandlerContext) Asset(path string) string {
	return fmt.Sprintf("%s/%s", c.Config.AssetHost, path)
}

func (c *JQHandlerContext) ShouldInitJQ() bool {
	return c.JQ != ""
}

type JQHandler struct {
	c *Config
}

func (h *JQHandler) handleIndex(c *gin.Context) {
	c.HTML(200, "index.tmpl", &JQHandlerContext{Config: h.c})
}

func (h *JQHandler) handleJqPost(c *gin.Context) {
	l, _ := c.Get("logger")
	logger := l.(*logrus.Entry)

	if c.Request.ContentLength > JSONPayloadLimit {
		size := float64(c.Request.ContentLength) / OneMB
		err := fmt.Errorf("JSON payload size is %.1fMB, larger than limit %dMB.", size, JSONPayloadLimitMB)
		logger.WithError(err).WithField("size", size).Infof(err.Error())
		c.String(http.StatusExpectationFailed, err.Error())
		return
	}

	c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, JSONPayloadLimit)

	var jq *jq.JQ
	err := c.BindJSON(&jq)
	if err != nil {
		err = fmt.Errorf("error parsing JSON: %s", err)
		logger.WithError(err).Infof("error parsing JSON: %s", err)
		c.String(422, err.Error())
		return
	}

	// Evaling into ResponseWriter sets the status code to 200
	// appending error message in the end if there's any
	err = jq.Eval(c.Writer)
	if err != nil {
		logger.WithError(err).Infof("error evaluating jq query: %s", err)
		fmt.Fprint(c.Writer, err.Error())
	}
}

func (h *JQHandler) handleJqGet(c *gin.Context) {
	jq := &jq.JQ{
		J: c.Query("j"),
		Q: c.Query("q"),
	}

	var jqData string
	if err := jq.Validate(); err == nil {
		d, err := json.Marshal(jq)
		if err == nil {
			jqData = string(d)
		}
	}

	c.HTML(200, "index.tmpl", &JQHandlerContext{Config: h.c, JQ: jqData})
}
<|endoftext|>


<commit_before>package server

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"time"

	"github.com/Sirupsen/logrus"
	log "github.com/Sirupsen/logrus"
	"github.com/jingweno/jqplay/config"
	"github.com/jingweno/jqplay/jq"
	"golang.org/x/net/context"
	"gopkg.in/gin-gonic/gin.v1"
)

const (
	JSONPayloadLimit   = JSONPayloadLimitMB * OneMB
	JSONPayloadLimitMB = 10
	OneMB              = 1024000
	JQExecTimeout      = 15 * time.Second
)

type JQHandlerContext struct {
	*config.Config
	JQ string
}

func (c *JQHandlerContext) Asset(path string) string {
	return fmt.Sprintf("%s/%s", c.AssetHost, path)
}

func (c *JQHandlerContext) ShouldInitJQ() bool {
	return c.JQ != ""
}

type JQHandler struct {
	DB     *DB
	Config *config.Config
}

func (h *JQHandler) handleIndex(c *gin.Context) {
	c.HTML(200, "index.tmpl", &JQHandlerContext{Config: h.Config})
}

func (h *JQHandler) checkReqSize(c *gin.Context) (float64, error) {
	if c.Request.ContentLength > JSONPayloadLimit {
		size := float64(c.Request.ContentLength) / OneMB
		return size, fmt.Errorf("JSON payload size is %.1fMB, larger than limit %dMB.", size, JSONPayloadLimitMB)
	}

	return 0, nil
}

func (h *JQHandler) handleJqPost(c *gin.Context) {
	if size, err := h.checkReqSize(c); err != nil {
		h.logger(c).WithField("size", size).WithError(err)
		c.String(http.StatusExpectationFailed, err.Error())
		return
	}

	c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, JSONPayloadLimit)

	var j *jq.JQ
	if err := c.BindJSON(&j); err != nil {
		err = fmt.Errorf("error parsing JSON: %s", err)
		h.logger(c).WithError(err).Infof("error parsing JSON")
		c.String(http.StatusUnprocessableEntity, err.Error())
		return
	}

	ctx, cancel := context.WithTimeout(c.Request.Context(), JQExecTimeout)
	defer cancel()

	// Evaling into ResponseWriter sets the status code to 200
	// appending error message in the end if there's any
	var debug bytes.Buffer
	w := io.MultiWriter(c.Writer, &debug)
	err = j.Eval(ctx, w)
	if err != nil {
		if _, ok := err.(*jq.JQValidationError); !ok {
			h.logger(c).WithError(err).WithFields(log.Fields{
				"j": j.J,
				"q": j.Q,
				"o": j.Opts(),
				"r": debug.String(),
			}).Info("jq error")
		}

		fmt.Fprint(c.Writer, err.Error())
	}
}

func (h *JQHandler) handleJqGet(c *gin.Context) {
	jq := &jq.JQ{
		J: c.Query("j"),
		Q: c.Query("q"),
	}

	var jqData string
	if err := jq.Validate(); err == nil {
		d, err := json.Marshal(jq)
		if err == nil {
			jqData = string(d)
		}
	}

	c.HTML(http.StatusOK, "index.tmpl", &JQHandlerContext{Config: h.Config, JQ: jqData})
}

func (h *JQHandler) handleJqSharePost(c *gin.Context) {
	if size, err := h.checkReqSize(c); err != nil {
		h.logger(c).WithField("size", size).WithError(err)
		c.String(http.StatusExpectationFailed, err.Error())
		return
	}

	c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, JSONPayloadLimit)

	var jq *jq.JQ
	if err := c.BindJSON(&jq); err != nil {
		err = fmt.Errorf("error parsing JSON: %s", err)
		h.logger(c).WithError(err)
		c.String(http.StatusUnprocessableEntity, err.Error())
		return
	}

	if err := jq.Validate(); err != nil {
		c.String(http.StatusUnprocessableEntity, err.Error())
		return
	}

	id, err := h.DB.UpsertSnippet(FromJQ(jq))
	if err != nil {
		h.logger(c).WithError(err)
		c.String(http.StatusUnprocessableEntity, "error sharing snippet")
		return
	}

	c.String(http.StatusCreated, id)
}

func (h *JQHandler) handleJqShareGet(c *gin.Context) {
	id := c.Param("id")

	s, err := h.DB.GetSnippet(id)
	if err != nil {
		h.logger(c).WithError(err).Info("error getting snippet")
		c.Redirect(http.StatusFound, "/")
		return
	}

	var jqData string
	d, err := json.Marshal(ToJQ(s))
	if err == nil {
		jqData = string(d)
	}

	c.HTML(200, "index.tmpl", &JQHandlerContext{
		Config: h.Config,
		JQ:     jqData,
	})
}

func (h *JQHandler) logger(c *gin.Context) *logrus.Entry {
	l, _ := c.Get("logger")
	return l.(*logrus.Entry)
}
<commit_msg>Fix broken code<commit_after>package server

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"time"

	"github.com/Sirupsen/logrus"
	log "github.com/Sirupsen/logrus"
	"github.com/jingweno/jqplay/config"
	"github.com/jingweno/jqplay/jq"
	"golang.org/x/net/context"
	"gopkg.in/gin-gonic/gin.v1"
)

const (
	JSONPayloadLimit   = JSONPayloadLimitMB * OneMB
	JSONPayloadLimitMB = 10
	OneMB              = 1024000
	JQExecTimeout      = 15 * time.Second
)

type JQHandlerContext struct {
	*config.Config
	JQ string
}

func (c *JQHandlerContext) Asset(path string) string {
	return fmt.Sprintf("%s/%s", c.AssetHost, path)
}

func (c *JQHandlerContext) ShouldInitJQ() bool {
	return c.JQ != ""
}

type JQHandler struct {
	DB     *DB
	Config *config.Config
}

func (h *JQHandler) handleIndex(c *gin.Context) {
	c.HTML(200, "index.tmpl", &JQHandlerContext{Config: h.Config})
}

func (h *JQHandler) checkReqSize(c *gin.Context) (float64, error) {
	if c.Request.ContentLength > JSONPayloadLimit {
		size := float64(c.Request.ContentLength) / OneMB
		return size, fmt.Errorf("JSON payload size is %.1fMB, larger than limit %dMB.", size, JSONPayloadLimitMB)
	}

	return 0, nil
}

func (h *JQHandler) handleJqPost(c *gin.Context) {
	if size, err := h.checkReqSize(c); err != nil {
		h.logger(c).WithField("size", size).WithError(err)
		c.String(http.StatusExpectationFailed, err.Error())
		return
	}

	c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, JSONPayloadLimit)

	var j *jq.JQ
	if err := c.BindJSON(&j); err != nil {
		err = fmt.Errorf("error parsing JSON: %s", err)
		h.logger(c).WithError(err).Infof("error parsing JSON")
		c.String(http.StatusUnprocessableEntity, err.Error())
		return
	}

	ctx, cancel := context.WithTimeout(c.Request.Context(), JQExecTimeout)
	defer cancel()

	// Evaling into ResponseWriter sets the status code to 200
	// appending error message in the end if there's any
	var debug bytes.Buffer
	w := io.MultiWriter(c.Writer, &debug)
	if err := j.Eval(ctx, w); err != nil {
		if _, ok := err.(*jq.JQValidationError); !ok {
			h.logger(c).WithError(err).WithFields(log.Fields{
				"j": j.J,
				"q": j.Q,
				"o": j.Opts(),
				"r": debug.String(),
			}).Info("jq error")
		}

		fmt.Fprint(c.Writer, err.Error())
	}
}

func (h *JQHandler) handleJqGet(c *gin.Context) {
	jq := &jq.JQ{
		J: c.Query("j"),
		Q: c.Query("q"),
	}

	var jqData string
	if err := jq.Validate(); err == nil {
		d, err := json.Marshal(jq)
		if err == nil {
			jqData = string(d)
		}
	}

	c.HTML(http.StatusOK, "index.tmpl", &JQHandlerContext{Config: h.Config, JQ: jqData})
}

func (h *JQHandler) handleJqSharePost(c *gin.Context) {
	if size, err := h.checkReqSize(c); err != nil {
		h.logger(c).WithField("size", size).WithError(err)
		c.String(http.StatusExpectationFailed, err.Error())
		return
	}

	c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, JSONPayloadLimit)

	var jq *jq.JQ
	if err := c.BindJSON(&jq); err != nil {
		err = fmt.Errorf("error parsing JSON: %s", err)
		h.logger(c).WithError(err)
		c.String(http.StatusUnprocessableEntity, err.Error())
		return
	}

	if err := jq.Validate(); err != nil {
		c.String(http.StatusUnprocessableEntity, err.Error())
		return
	}

	id, err := h.DB.UpsertSnippet(FromJQ(jq))
	if err != nil {
		h.logger(c).WithError(err)
		c.String(http.StatusUnprocessableEntity, "error sharing snippet")
		return
	}

	c.String(http.StatusCreated, id)
}

func (h *JQHandler) handleJqShareGet(c *gin.Context) {
	id := c.Param("id")

	s, err := h.DB.GetSnippet(id)
	if err != nil {
		h.logger(c).WithError(err).Info("error getting snippet")
		c.Redirect(http.StatusFound, "/")
		return
	}

	var jqData string
	d, err := json.Marshal(ToJQ(s))
	if err == nil {
		jqData = string(d)
	}

	c.HTML(200, "index.tmpl", &JQHandlerContext{
		Config: h.Config,
		JQ:     jqData,
	})
}

func (h *JQHandler) logger(c *gin.Context) *logrus.Entry {
	l, _ := c.Get("logger")
	return l.(*logrus.Entry)
}
<|endoftext|>


<commit_before>package main

import (
	ws "code.google.com/p/go.net/websocket"
	"encoding/json"
	"fmt"
	gcm "github.com/googollee/go-gcm"
	"log"
	"net"
	"sync"
)

type GCMPusher struct {
	sync.RWMutex
	events    chan Event
	deviceIds map[string]struct{}
	client    *gcm.Client
}

func NewGCMPusher(key string) *GCMPusher {
	pusher := &GCMPusher{
		events:    make(chan Event),
		deviceIds: make(map[string]struct{}),
	}
	if len(key) != 0 {
		pusher.client = gcm.New(key)
	}
	go pusher.loop()

	return pusher
}

func (p *GCMPusher) Notify(event Event) {
	p.events <- event
}

func (p *GCMPusher) Subscribe(deviceId string) {
	p.Lock()
	p.deviceIds[deviceId] = struct{}{}
	log.Printf("Subscribed `%s` to GCMPush", deviceId)
	p.Unlock()
}

func (p *GCMPusher) loop() {
	for event := range p.events {
		deviceIds := p.DeviceIds()
		if len(deviceIds) == 0 || p.client == nil {
			log.Printf("Bypassing GCM push: %v", event)
			continue
		}
		message := gcm.NewMessage(deviceIds...)
		payload, err := json.Marshal(event.Payload)
		if err != nil {
			log.Printf("Failed GCM push: %s", err)
			continue
		}
		message.SetPayload(event.Name, string(payload))
		_, err = p.client.Send(message)
		if err != nil {
			log.Printf("Failed GCM push: %s", err)
			continue
		}
		log.Printf("GCMPushed: %v to %v", event, deviceIds)
	}
}

func (p *GCMPusher) DeviceIds() []string {
	p.RLock()
	defer p.RUnlock()
	deviceIds := make([]string, 0, len(p.deviceIds))
	for deviceId, _ := range p.deviceIds {
		deviceIds = append(deviceIds, deviceId)
	}
	return deviceIds
}

type WSPusher struct {
	sync.RWMutex
	events chan Event
	conns  map[net.Addr]*ws.Conn
}

func NewWSPusher() *WSPusher {
	pusher := &WSPusher{
		events: make(chan Event),
		conns:  make(map[net.Addr]*ws.Conn),
	}
	go pusher.loop()
	return pusher
}

func (p *WSPusher) Notify(event Event) {
	p.events <- event
}

func (p *WSPusher) Subscribe(conn *ws.Conn) chan (error) {
	exit := make(chan error)

	p.Lock()
	defer p.Unlock()

	addr := conn.RemoteAddr()
	if _, ok := p.conns[addr]; ok {
		go func() { exit <- fmt.Errorf("%s already connected", addr) }()
	} else {
		p.conns[addr] = conn
		log.Printf("Subscribed `%s` to WSPusher.", addr)
	}
	// TODO: Find a way to signal correct termination
	return exit
}

func (p *WSPusher) loop() {
	for event := range p.events {
		for addr, conn := range p.conns {
			go func(event Event, addr net.Addr, conn *ws.Conn) {
				if err := ws.JSON.Send(conn, event); err != nil {
					log.Printf("Failed WS push to %s: %s", addr, err)
					return
				}
				log.Printf("WSPushed `%v` event to %s.", event, addr)
			}(event, addr, conn)
		}
	}
}
<commit_msg>Thread-safe WSPusher<commit_after>package main

import (
	ws "code.google.com/p/go.net/websocket"
	"encoding/json"
	"fmt"
	gcm "github.com/googollee/go-gcm"
	"log"
	"net"
	"sync"
)

type GCMPusher struct {
	sync.RWMutex
	events    chan Event
	deviceIds map[string]struct{}
	client    *gcm.Client
}

func NewGCMPusher(key string) *GCMPusher {
	pusher := &GCMPusher{
		events:    make(chan Event),
		deviceIds: make(map[string]struct{}),
	}
	if len(key) != 0 {
		pusher.client = gcm.New(key)
	}
	go pusher.loop()

	return pusher
}

func (p *GCMPusher) Notify(event Event) {
	p.events <- event
}

func (p *GCMPusher) Subscribe(deviceId string) {
	p.Lock()
	p.deviceIds[deviceId] = struct{}{}
	log.Printf("Subscribed `%s` to GCMPush", deviceId)
	p.Unlock()
}

func (p *GCMPusher) loop() {
	for event := range p.events {
		deviceIds := p.DeviceIds()
		if len(deviceIds) == 0 || p.client == nil {
			log.Printf("Bypassing GCM push: %v", event)
			continue
		}
		message := gcm.NewMessage(deviceIds...)
		payload, err := json.Marshal(event.Payload)
		if err != nil {
			log.Printf("Failed GCM push: %s", err)
			continue
		}
		message.SetPayload(event.Name, string(payload))
		_, err = p.client.Send(message)
		if err != nil {
			log.Printf("Failed GCM push: %s", err)
			continue
		}
		log.Printf("GCMPushed: %v to %v", event, deviceIds)
	}
}

func (p *GCMPusher) DeviceIds() []string {
	p.RLock()
	defer p.RUnlock()
	deviceIds := make([]string, 0, len(p.deviceIds))
	for deviceId, _ := range p.deviceIds {
		deviceIds = append(deviceIds, deviceId)
	}
	return deviceIds
}

type WSPusher struct {
	sync.RWMutex
	events chan Event
	conns  map[net.Addr]*ws.Conn
}

func NewWSPusher() *WSPusher {
	pusher := &WSPusher{
		events: make(chan Event),
		conns:  make(map[net.Addr]*ws.Conn),
	}
	go pusher.loop()
	return pusher
}

func (p *WSPusher) Notify(event Event) {
	p.events <- event
}

func (p *WSPusher) Subscribe(conn *ws.Conn) chan (error) {
	exit := make(chan error)

	p.Lock()
	defer p.Unlock()

	addr := conn.RemoteAddr()
	if _, ok := p.conns[addr]; ok {
		go func() { exit <- fmt.Errorf("%s already connected", addr) }()
	} else {
		p.conns[addr] = conn
		log.Printf("Subscribed `%s` to WSPusher.", addr)
	}
	// TODO: Find a way to signal correct termination
	return exit
}

func (p *WSPusher) loop() {
	for event := range p.events {
		p.RLock()
		conns := p.conns
		p.RUnlock()
		for addr, conn := range conns {
			go func(event Event, addr net.Addr, conn *ws.Conn) {
				if err := ws.JSON.Send(conn, event); err != nil {
					log.Printf("Failed WS push to %s: %s", addr, err)
					conn.Close()
					p.Lock()
					delete(p.conns, addr)
					p.Unlock()
					return
				}
				log.Printf("WSPushed `%v` event to %s.", event, addr)
			}(event, addr, conn)
		}
	}
}
<|endoftext|>


<commit_before>package server

import "github.com/humpback/discovery"
import "github.com/humpback/gounits/json"
import "github.com/humpback/gounits/rand"
import "github.com/humpback/gounits/network"
import "github.com/humpback/humpback-center/cluster"
import "github.com/bobliu0909/humpback-node/etc"

import (
	"flag"
	"log"
	"net"
	"os"
	"time"
)

type NodeService struct {
	Key           string
	Configuration *etc.Configuration
	discovery     *discovery.Discovery
	stopCh        chan struct{}
}

// NewNodeService exported
func NewNodeService() (*NodeService, error) {

	var conf string
	flag.StringVar(&conf, "f", "etc/config.yaml", "humpback node configuration file.")
	flag.Parse()
	configuration, err := etc.NewConfiguration(conf)
	if err != nil {
		return nil, err
	}

	key, err := rand.UUIDFile("./humpback-node.key")
	if err != nil {
		return nil, err
	}

	heartbeat, err := time.ParseDuration(configuration.Discovery.Heartbeat)
	if err != nil {
		return nil, err
	}

	ttl, err := time.ParseDuration(configuration.Discovery.TTL)
	if err != nil {
		return nil, err
	}

	configopts := map[string]string{"kv.path": configuration.Discovery.Cluster}
	d, err := discovery.New(configuration.Discovery.URIs, heartbeat, ttl, configopts)
	if err != nil {
		return nil, err
	}

	return &NodeService{
		Key:           key,
		Configuration: configuration,
		discovery:     d,
		stopCh:        make(chan struct{}),
	}, nil
}

func (service *NodeService) Startup() error {

	log.Printf("[#service#] service start...\n")
	addr, err := getServiceAddr(service.Configuration.API.Host)
	if err != nil {
		return err
	}

	name, err := os.Hostname()
	if err != nil {
		return err
	}

	regOpts := &cluster.RegistClusterOptions{
		Name:    name,
		Addr:    addr,
		Labels:  []string{"node=wh7", "os=centos6.8"},
		Version: "1.0.0",
	}
	buf, err := json.EnCodeObjectToBuffer(regOpts)
	if err != nil {
		return err
	}

	log.Printf("[#service#] regist key:%s, addr %s\n", service.Key, addr)
	service.discovery.Register(service.Key, buf, service.stopCh, func(key string, err error) {
		log.Printf("[#service#] discovery regist %s error:%s\n", key, err.Error())
	})
	return nil
}

func (service *NodeService) Stop() error {

	log.Printf("[#service#] service closed.\n")
	close(service.stopCh)
	return nil
}

func getServiceAddr(host string) (string, error) {

	ip, port, err := net.SplitHostPort(host)
	if err != nil {
		return "", err
	}

	if len(ip) == 0 {
		ip = network.GetDefaultIP()
	}

	if _, err := net.ResolveIPAddr("ip4", ip); err != nil {
		return "", err
	}
	return ip + ":" + port, nil
}
<commit_msg>add cluster types, modify cluster.Opts to types.Opts.<commit_after>package server

import "github.com/humpback/discovery"
import "github.com/humpback/gounits/rand"
import "github.com/humpback/gounits/network"
import "github.com/humpback/gounits/json"
import "github.com/humpback/humpback-center/cluster/types"
import "github.com/bobliu0909/humpback-node/etc"

import (
	"flag"
	"log"
	"net"
	"time"
)

type NodeService struct {
	Key           string
	Configuration *etc.Configuration
	discovery     *discovery.Discovery
	stopCh        chan struct{}
}

// NewNodeService exported
func NewNodeService() (*NodeService, error) {

	var conf string
	flag.StringVar(&conf, "f", "etc/config.yaml", "humpback node configuration file.")
	flag.Parse()
	configuration, err := etc.NewConfiguration(conf)
	if err != nil {
		return nil, err
	}

	key, err := rand.UUIDFile("./humpback-node.key")
	if err != nil {
		return nil, err
	}

	heartbeat, err := time.ParseDuration(configuration.Discovery.Heartbeat)
	if err != nil {
		return nil, err
	}

	ttl, err := time.ParseDuration(configuration.Discovery.TTL)
	if err != nil {
		return nil, err
	}

	configopts := map[string]string{"kv.path": configuration.Discovery.Cluster}
	d, err := discovery.New(configuration.Discovery.URIs, heartbeat, ttl, configopts)
	if err != nil {
		return nil, err
	}

	return &NodeService{
		Key:           key,
		Configuration: configuration,
		discovery:     d,
		stopCh:        make(chan struct{}),
	}, nil
}

func (service *NodeService) Startup() error {

	log.Printf("[#service#] service start...\n")
	addr, err := getServiceAddr(service.Configuration.API.Host)
	if err != nil {
		return err
	}

	regOpts := types.NewClusterRegistOptions("", "", addr, []string{"node=wh7", "os=centos6.8"}, "1.0.0")
	buf, err := json.EnCodeObjectToBuffer(regOpts)
	if err != nil {
		return err
	}

	log.Printf("[#service#] regist to cluster id:%s, ip:%s, addr %s\n", service.Key, regOpts.IP, regOpts.Addr)
	service.discovery.Register(service.Key, buf, service.stopCh, func(key string, err error) {
		log.Printf("[#service#] discovery regist %s error:%s\n", key, err.Error())
	})
	return nil
}

func (service *NodeService) Stop() error {

	log.Printf("[#service#] service closed.\n")
	close(service.stopCh)
	return nil
}

func getServiceAddr(host string) (string, error) {

	ip, port, err := net.SplitHostPort(host)
	if err != nil {
		return "", err
	}

	if len(ip) == 0 {
		ip = network.GetDefaultIP()
	}

	if _, err := net.ResolveIPAddr("ip4", ip); err != nil {
		return "", err
	}
	return ip + ":" + port, nil
}
<|endoftext|>


<commit_before>package dbx_api

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/tidwall/gjson"
	"go.uber.org/zap"
	"net/http"
	"time"
)

var (
	ReqHeaderSelectUser     = "Dropbox-API-Select-User"
	ReqHeaderSelectAdmin    = "Dropbox-API-Select-Admin"
	ResHeaderRetryAfter     = "Retry-After"
	ResJsonDotTag           = "\\.tag"
	DefaultClientTimeout    = time.Duration(60) * time.Second
	DateTimeFormat          = "2006-01-02T15:04:05Z"
	ErrorBadInputParam      = 400
	ErrorBadOrExpiredToken  = 401
	ErrorAccessError        = 403
	ErrorEndpointSpecific   = 409
	ErrorRateLimit          = 429
	ErrorSuccess            = 0
	ErrorTransport          = 1000
	ErrorUnknown            = 1001
	ErrorUnexpectedDataType = 1002
	ErrorOperationFailed    = 1003
	ErrorServerError        = 1500
)

func ParseApiError(responseBody string) (ae ApiError) {
	ae.ErrorTag = gjson.Get(responseBody, "error."+ResJsonDotTag).String()
	ae.ErrorSummary = gjson.Get(responseBody, "error_summary").String()
	ae.UserMessageLocale = gjson.Get(responseBody, "user_message.locale").String()
	ae.UserMessage = gjson.Get(responseBody, "user_message.text").String()
	ae.ErrorBody = json.RawMessage(gjson.Get(responseBody, "error").Raw)
	ae.UserMessageBody = json.RawMessage(gjson.Get(responseBody, "user_message").Raw)

	return
}

func ParseAccessError(responseBody string) (ae AccessError) {
	ae.PaperAccessDenied = gjson.Get(responseBody, "invalid_account_type.\\.tag").String()
	ae.InvalidAccountType = gjson.Get(responseBody, "paper_access_denied.\\.tag").String()
	ae.ErrorBody = json.RawMessage(responseBody)

	return
}

var (
	Success = ErrorAnnotation{
		ErrorType: ErrorSuccess,
		Error:     nil,
	}
)

type ErrorAnnotation struct {
	ErrorType int
	Error     error
}

func (e ErrorAnnotation) IsSuccess() bool {
	return e.ErrorType == ErrorSuccess
}
func (e ErrorAnnotation) IsFailure() bool {
	return e.ErrorType != ErrorSuccess
}
func (e ErrorAnnotation) ApiError() *ApiError {
	switch ae := e.Error.(type) {
	case ApiError:
		return &ae
	}
	return nil
}
func (e ErrorAnnotation) AccessError() *AccessError {
	switch ae := e.Error.(type) {
	case AccessError:
		return &ae
	}
	return nil
}
func (e ErrorAnnotation) ErrorTypeLabel() string {
	switch e.ErrorType {
	case ErrorBadInputParam:
		return "bad_input_param"
	case ErrorBadOrExpiredToken:
		return "bad_or_expired_token"
	case ErrorAccessError:
		return "access_error"
	case ErrorEndpointSpecific:
		return "endpoint_specific"
	case ErrorRateLimit:
		return "rate_limit"
	case ErrorSuccess:
		return "success"
	case ErrorTransport:
		return "transport_error"
	case ErrorUnknown:
		return "unknown"
	case ErrorOperationFailed:
		return "operation_failed"
	case ErrorUnexpectedDataType:
		return "unexpected_data_type"
	case ErrorServerError:
		return "server_error"
	}
	return "unknown"
}
func (e ErrorAnnotation) UserMessage() string {
	if e.Error == nil {
		return ""
	}
	if ae := e.ApiError(); ae != nil {
		if ae.UserMessage != "" {
			return ae.UserMessage
		} else {
			return ae.ErrorSummary
		}
	}
	if ae := e.AccessError(); ae != nil {
		return ae.Error()
	}
	return e.Error.Error()
}

type ServerError struct {
	StatusCode int
}

func (e ServerError) Error() string {
	return fmt.Sprintf("An error occurred on the Dropbox servers (%d). Check status.dropbox.com for announcements about Dropbox service issues.", e.StatusCode)
}

type ApiError struct {
	ErrorTag          string          `json:"error,omitempty"`
	ErrorSummary      string          `json:"error_summary,omitempty"`
	ErrorBody         json.RawMessage `json:"error,omitempty"`
	UserMessageLocale string          `json:"user_message_lang,omitempty"`
	UserMessage       string          `json:"user_message,omitempty"`
	UserMessageBody   json.RawMessage `json:"user_message,omitempty"`
}

func (e ApiError) Error() string {
	return fmt.Sprintf("Endpoint specific error[%s] %s", e.ErrorTag, e.ErrorSummary)
}

type AccessError struct {
	InvalidAccountType string          `json:"invalid_account_type,omitempty"`
	PaperAccessDenied  string          `json:"paper_access_denied,omitempty"`
	ErrorBody          json.RawMessage `json:"error,omitempty"`
}

func (a AccessError) Error() string {
	if a.InvalidAccountType != "" {
		return a.InvalidAccountType
	}
	if a.PaperAccessDenied != "" {
		return a.PaperAccessDenied
	}
	return "The user or team account doesn't have access to the endpoint or feature"
}

type ApiErrorRateLimit struct {
	RetryAfter int
}

func (e ApiErrorRateLimit) Error() string {
	return fmt.Sprintf("API Rate limit (retry after %d sec)", e.RetryAfter)
}

func RebaseTimeForAPI(t time.Time) time.Time {
	return t.UTC().Round(time.Second)
}

type Context struct {
	Token      string
	Client     *http.Client
	RetryAfter time.Time
	LastErrors []ErrorAnnotation
	Logger     *zap.Logger
}

func (c *Context) Log() *zap.Logger {
	return c.Logger
}

func (c *Context) ParseModel(v interface{}, j gjson.Result) error {
	return parseModel(c.Log(), v, j)
}

func (c *Context) ParseModelJson(v interface{}, raw json.RawMessage) error {
	return parseModelJson(c.Log(), v, raw)
}

func NewContext(token string, logger *zap.Logger) *Context {
	logger.Debug("New context")
	transport := &http.Transport{Proxy: http.ProxyFromEnvironment}

	return &Context{
		Token: token,
		Client: &http.Client{
			Timeout:   DefaultClientTimeout,
			Transport: transport,
		},
		Logger:     logger,
		LastErrors: make([]ErrorAnnotation, 0),
	}
}

func ParserError(msg string, body string, logger *zap.Logger, onError func(annotation ErrorAnnotation) bool) bool {
	logger.Debug(msg, zap.String("body", body))
	err := errors.New(msg)
	ann := ErrorAnnotation{
		ErrorType: ErrorUnexpectedDataType,
		Error:     err,
	}
	if onError != nil {
		return onError(ann)
	}
	return false
}
<commit_msg>remove auto proxy config<commit_after>package dbx_api

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/tidwall/gjson"
	"go.uber.org/zap"
	"net/http"
	"time"
)

var (
	ReqHeaderSelectUser     = "Dropbox-API-Select-User"
	ReqHeaderSelectAdmin    = "Dropbox-API-Select-Admin"
	ResHeaderRetryAfter     = "Retry-After"
	ResJsonDotTag           = "\\.tag"
	DefaultClientTimeout    = time.Duration(60) * time.Second
	DateTimeFormat          = "2006-01-02T15:04:05Z"
	ErrorBadInputParam      = 400
	ErrorBadOrExpiredToken  = 401
	ErrorAccessError        = 403
	ErrorEndpointSpecific   = 409
	ErrorRateLimit          = 429
	ErrorSuccess            = 0
	ErrorTransport          = 1000
	ErrorUnknown            = 1001
	ErrorUnexpectedDataType = 1002
	ErrorOperationFailed    = 1003
	ErrorServerError        = 1500
)

func ParseApiError(responseBody string) (ae ApiError) {
	ae.ErrorTag = gjson.Get(responseBody, "error."+ResJsonDotTag).String()
	ae.ErrorSummary = gjson.Get(responseBody, "error_summary").String()
	ae.UserMessageLocale = gjson.Get(responseBody, "user_message.locale").String()
	ae.UserMessage = gjson.Get(responseBody, "user_message.text").String()
	ae.ErrorBody = json.RawMessage(gjson.Get(responseBody, "error").Raw)
	ae.UserMessageBody = json.RawMessage(gjson.Get(responseBody, "user_message").Raw)

	return
}

func ParseAccessError(responseBody string) (ae AccessError) {
	ae.PaperAccessDenied = gjson.Get(responseBody, "invalid_account_type.\\.tag").String()
	ae.InvalidAccountType = gjson.Get(responseBody, "paper_access_denied.\\.tag").String()
	ae.ErrorBody = json.RawMessage(responseBody)

	return
}

var (
	Success = ErrorAnnotation{
		ErrorType: ErrorSuccess,
		Error:     nil,
	}
)

type ErrorAnnotation struct {
	ErrorType int
	Error     error
}

func (e ErrorAnnotation) IsSuccess() bool {
	return e.ErrorType == ErrorSuccess
}
func (e ErrorAnnotation) IsFailure() bool {
	return e.ErrorType != ErrorSuccess
}
func (e ErrorAnnotation) ApiError() *ApiError {
	switch ae := e.Error.(type) {
	case ApiError:
		return &ae
	}
	return nil
}
func (e ErrorAnnotation) AccessError() *AccessError {
	switch ae := e.Error.(type) {
	case AccessError:
		return &ae
	}
	return nil
}
func (e ErrorAnnotation) ErrorTypeLabel() string {
	switch e.ErrorType {
	case ErrorBadInputParam:
		return "bad_input_param"
	case ErrorBadOrExpiredToken:
		return "bad_or_expired_token"
	case ErrorAccessError:
		return "access_error"
	case ErrorEndpointSpecific:
		return "endpoint_specific"
	case ErrorRateLimit:
		return "rate_limit"
	case ErrorSuccess:
		return "success"
	case ErrorTransport:
		return "transport_error"
	case ErrorUnknown:
		return "unknown"
	case ErrorOperationFailed:
		return "operation_failed"
	case ErrorUnexpectedDataType:
		return "unexpected_data_type"
	case ErrorServerError:
		return "server_error"
	}
	return "unknown"
}
func (e ErrorAnnotation) UserMessage() string {
	if e.Error == nil {
		return ""
	}
	if ae := e.ApiError(); ae != nil {
		if ae.UserMessage != "" {
			return ae.UserMessage
		} else {
			return ae.ErrorSummary
		}
	}
	if ae := e.AccessError(); ae != nil {
		return ae.Error()
	}
	return e.Error.Error()
}

type ServerError struct {
	StatusCode int
}

func (e ServerError) Error() string {
	return fmt.Sprintf("An error occurred on the Dropbox servers (%d). Check status.dropbox.com for announcements about Dropbox service issues.", e.StatusCode)
}

type ApiError struct {
	ErrorTag          string          `json:"error,omitempty"`
	ErrorSummary      string          `json:"error_summary,omitempty"`
	ErrorBody         json.RawMessage `json:"error,omitempty"`
	UserMessageLocale string          `json:"user_message_lang,omitempty"`
	UserMessage       string          `json:"user_message,omitempty"`
	UserMessageBody   json.RawMessage `json:"user_message,omitempty"`
}

func (e ApiError) Error() string {
	return fmt.Sprintf("Endpoint specific error[%s] %s", e.ErrorTag, e.ErrorSummary)
}

type AccessError struct {
	InvalidAccountType string          `json:"invalid_account_type,omitempty"`
	PaperAccessDenied  string          `json:"paper_access_denied,omitempty"`
	ErrorBody          json.RawMessage `json:"error,omitempty"`
}

func (a AccessError) Error() string {
	if a.InvalidAccountType != "" {
		return a.InvalidAccountType
	}
	if a.PaperAccessDenied != "" {
		return a.PaperAccessDenied
	}
	return "The user or team account doesn't have access to the endpoint or feature"
}

type ApiErrorRateLimit struct {
	RetryAfter int
}

func (e ApiErrorRateLimit) Error() string {
	return fmt.Sprintf("API Rate limit (retry after %d sec)", e.RetryAfter)
}

func RebaseTimeForAPI(t time.Time) time.Time {
	return t.UTC().Round(time.Second)
}

type Context struct {
	Token      string
	Client     *http.Client
	RetryAfter time.Time
	LastErrors []ErrorAnnotation
	Logger     *zap.Logger
}

func (c *Context) Log() *zap.Logger {
	return c.Logger
}

func (c *Context) ParseModel(v interface{}, j gjson.Result) error {
	return parseModel(c.Log(), v, j)
}

func (c *Context) ParseModelJson(v interface{}, raw json.RawMessage) error {
	return parseModelJson(c.Log(), v, raw)
}

func NewContext(token string, logger *zap.Logger) *Context {
	logger.Debug("New context")

	return &Context{
		Token: token,
		Client: &http.Client{
			Timeout: DefaultClientTimeout,
		},
		Logger:     logger,
		LastErrors: make([]ErrorAnnotation, 0),
	}
}

func ParserError(msg string, body string, logger *zap.Logger, onError func(annotation ErrorAnnotation) bool) bool {
	logger.Debug(msg, zap.String("body", body))
	err := errors.New(msg)
	ann := ErrorAnnotation{
		ErrorType: ErrorUnexpectedDataType,
		Error:     err,
	}
	if onError != nil {
		return onError(ann)
	}
	return false
}
<|endoftext|>


<commit_before>package render

import (
	"github.com/weaveworks/scope/report"
)

// MapFunc is anything which can take an arbitrary Node and
// return another Node.
//
// If the output ID is blank, the node shall be omitted from the rendered topology.
type MapFunc func(report.Node) report.Node

// Renderer is something that can render a report to a set of Nodes.
type Renderer interface {
	Render(report.Report) Nodes
}

// Nodes is the result of Rendering
type Nodes struct {
	report.Nodes
	Filtered int
}

// Merge merges the results of Rendering
func (r Nodes) Merge(o Nodes) Nodes {
	return Nodes{
		Nodes:    r.Nodes.Merge(o.Nodes),
		Filtered: r.Filtered + o.Filtered,
	}
}

// Transformer is something that transforms one set of Nodes to
// another set of Nodes.
type Transformer interface {
	Transform(nodes Nodes) Nodes
}

// Transformers is a composition of Transformers
type Transformers []Transformer

// Transform implements Transformer
func (ts Transformers) Transform(nodes Nodes) Nodes {
	for _, t := range ts {
		nodes = t.Transform(nodes)
	}
	return nodes
}

// Render renders the report and then transforms it
func Render(rpt report.Report, renderer Renderer, transformer Transformer) Nodes {
	return transformer.Transform(renderer.Render(rpt))
}

// Reduce renderer is a Renderer which merges together the output of several
// other renderers.
type Reduce []Renderer

// MakeReduce is the only sane way to produce a Reduce Renderer.
func MakeReduce(renderers ...Renderer) Renderer {
	return Reduce(renderers)
}

// Render produces a set of Nodes given a Report.
func (r Reduce) Render(rpt report.Report) Nodes {
	l := len(r)
	switch l {
	case 0:
		return Nodes{}
	}
	c := make(chan Nodes, l)
	for _, renderer := range r {
		renderer := renderer // Pike!!
		go func() {
			c <- renderer.Render(rpt)
		}()
	}
	for ; l > 1; l-- {
		left, right := <-c, <-c
		go func() {
			c <- left.Merge(right)
		}()
	}
	return <-c
}

// Map is a Renderer which produces a set of Nodes from the set of
// Nodes produced by another Renderer.
type Map struct {
	MapFunc
	Renderer
}

// MakeMap makes a new Map
func MakeMap(f MapFunc, r Renderer) Renderer {
	return Map{f, r}
}

// Render transforms a set of Nodes produces by another Renderer.
// using a map function
func (m Map) Render(rpt report.Report) Nodes {
	var (
		input  = m.Renderer.Render(rpt)
		output = newJoinResults(nil)
	)

	// Rewrite all the nodes according to the map function
	for _, inRenderable := range input.Nodes {
		outRenderable := m.MapFunc(inRenderable)
		if outRenderable.ID != "" {
			output.add(inRenderable.ID, outRenderable)
		}
	}

	return output.result(input)
}

func propagateLatest(key string, from, to report.Node) report.Node {
	if value, timestamp, ok := from.Latest.LookupEntry(key); ok {
		to.Latest = to.Latest.Set(key, timestamp, value)
	}
	return to
}

// Condition is a predecate over the entire report that can evaluate to true or false.
type Condition func(report.Report) bool

type conditionalRenderer struct {
	Condition
	Renderer
}

// ConditionalRenderer renders nothing if the condition is false, otherwise it defers
// to the wrapped Renderer.
func ConditionalRenderer(c Condition, r Renderer) Renderer {
	return conditionalRenderer{c, r}
}

func (cr conditionalRenderer) Render(rpt report.Report) Nodes {
	if cr.Condition(rpt) {
		return cr.Renderer.Render(rpt)
	}
	return Nodes{}
}

// joinResults is used by Renderers that join sets of nodes
type joinResults struct {
	nodes  report.Nodes
	mapped map[string]string   // input node ID -> output node ID - common case
	multi  map[string][]string // input node ID -> output node IDs - exceptional case
}

func newJoinResults(inputNodes report.Nodes) joinResults {
	nodes := make(report.Nodes, len(inputNodes))
	for id, n := range inputNodes {
		n.Adjacency = nil // result() assumes all nodes start with no adjacencies
		nodes[id] = n
	}
	return joinResults{nodes: nodes, mapped: map[string]string{}, multi: map[string][]string{}}
}

func (ret *joinResults) mapChild(from, to string) {
	if _, ok := ret.mapped[from]; !ok {
		ret.mapped[from] = to
	} else {
		ret.multi[from] = append(ret.multi[from], to)
	}
}

// Add m into the results as a top-level node, mapped from original ID
func (ret *joinResults) add(from string, m report.Node) {
	if existing, ok := ret.nodes[m.ID]; ok {
		m = m.Merge(existing)
	}
	ret.nodes[m.ID] = m
	ret.mapChild(from, m.ID)
}

// Add m as a child of the node at id, creating a new result node in
// the specified topology if not already there.
func (ret *joinResults) addUnmappedChild(m report.Node, id string, topology string) {
	result, exists := ret.nodes[id]
	if !exists {
		result = report.MakeNode(id).WithTopology(topology)
	}
	result.Children = result.Children.Add(m)
	if m.Topology != report.Endpoint { // optimisation: we never look at endpoint counts
		result.Counters = result.Counters.Add(m.Topology, 1)
	}
	ret.nodes[id] = result
}

// Add m as a child of the node at id, creating a new result node in
// the specified topology if not already there, and updating the
// mapping from old ID to new ID.
func (ret *joinResults) addChild(m report.Node, id string, topology string) {
	ret.addUnmappedChild(m, id, topology)
	ret.mapChild(m.ID, id)
}

// Like addChild, but also add m's children.
func (ret *joinResults) addChildAndChildren(m report.Node, id string, topology string) {
	ret.addUnmappedChild(m, id, topology)
	result := ret.nodes[id]
	result.Children = result.Children.Merge(m.Children)
	ret.nodes[id] = result
	ret.mapChild(m.ID, id)
}

// Add a copy of n straight into the results
func (ret *joinResults) passThrough(n report.Node) {
	n.Adjacency = nil // result() assumes all nodes start with no adjacencies
	ret.nodes[n.ID] = n
	ret.mapChild(n.ID, n.ID)
}

// Rewrite Adjacency of nodes in ret mapped from original nodes in
// input, and return the result.
func (ret *joinResults) result(input Nodes) Nodes {
	for _, n := range input.Nodes {
		outID, ok := ret.mapped[n.ID]
		if !ok {
			continue
		}
		ret.rewriteAdjacency(outID, n.Adjacency)
		for _, outID := range ret.multi[n.ID] {
			ret.rewriteAdjacency(outID, n.Adjacency)
		}
	}
	return Nodes{Nodes: ret.nodes}
}

func (ret *joinResults) rewriteAdjacency(outID string, adjacency report.IDList) {
	out := ret.nodes[outID]
	// for each adjacency in the original node, find out what it maps
	// to (if any), and add that to the new node
	for _, a := range adjacency {
		if mappedDest, found := ret.mapped[a]; found {
			out.Adjacency = out.Adjacency.Add(mappedDest)
			out.Adjacency = out.Adjacency.Add(ret.multi[a]...)
		}
	}
	ret.nodes[outID] = out
}

// ResetCache blows away the rendered node cache, and known service
// cache.
func ResetCache() {
	renderCache.Purge()
	purgeKnownServiceCache()
}
<commit_msg>Comment<commit_after>package render

import (
	"github.com/weaveworks/scope/report"
)

// MapFunc is anything which can take an arbitrary Node and
// return another Node.
//
// If the output ID is blank, the node shall be omitted from the rendered topology.
// (we chose not to return an extra bool because it adds clutter)
type MapFunc func(report.Node) report.Node

// Renderer is something that can render a report to a set of Nodes.
type Renderer interface {
	Render(report.Report) Nodes
}

// Nodes is the result of Rendering
type Nodes struct {
	report.Nodes
	Filtered int
}

// Merge merges the results of Rendering
func (r Nodes) Merge(o Nodes) Nodes {
	return Nodes{
		Nodes:    r.Nodes.Merge(o.Nodes),
		Filtered: r.Filtered + o.Filtered,
	}
}

// Transformer is something that transforms one set of Nodes to
// another set of Nodes.
type Transformer interface {
	Transform(nodes Nodes) Nodes
}

// Transformers is a composition of Transformers
type Transformers []Transformer

// Transform implements Transformer
func (ts Transformers) Transform(nodes Nodes) Nodes {
	for _, t := range ts {
		nodes = t.Transform(nodes)
	}
	return nodes
}

// Render renders the report and then transforms it
func Render(rpt report.Report, renderer Renderer, transformer Transformer) Nodes {
	return transformer.Transform(renderer.Render(rpt))
}

// Reduce renderer is a Renderer which merges together the output of several
// other renderers.
type Reduce []Renderer

// MakeReduce is the only sane way to produce a Reduce Renderer.
func MakeReduce(renderers ...Renderer) Renderer {
	return Reduce(renderers)
}

// Render produces a set of Nodes given a Report.
func (r Reduce) Render(rpt report.Report) Nodes {
	l := len(r)
	switch l {
	case 0:
		return Nodes{}
	}
	c := make(chan Nodes, l)
	for _, renderer := range r {
		renderer := renderer // Pike!!
		go func() {
			c <- renderer.Render(rpt)
		}()
	}
	for ; l > 1; l-- {
		left, right := <-c, <-c
		go func() {
			c <- left.Merge(right)
		}()
	}
	return <-c
}

// Map is a Renderer which produces a set of Nodes from the set of
// Nodes produced by another Renderer.
type Map struct {
	MapFunc
	Renderer
}

// MakeMap makes a new Map
func MakeMap(f MapFunc, r Renderer) Renderer {
	return Map{f, r}
}

// Render transforms a set of Nodes produces by another Renderer.
// using a map function
func (m Map) Render(rpt report.Report) Nodes {
	var (
		input  = m.Renderer.Render(rpt)
		output = newJoinResults(nil)
	)

	// Rewrite all the nodes according to the map function
	for _, inRenderable := range input.Nodes {
		outRenderable := m.MapFunc(inRenderable)
		if outRenderable.ID != "" {
			output.add(inRenderable.ID, outRenderable)
		}
	}

	return output.result(input)
}

func propagateLatest(key string, from, to report.Node) report.Node {
	if value, timestamp, ok := from.Latest.LookupEntry(key); ok {
		to.Latest = to.Latest.Set(key, timestamp, value)
	}
	return to
}

// Condition is a predecate over the entire report that can evaluate to true or false.
type Condition func(report.Report) bool

type conditionalRenderer struct {
	Condition
	Renderer
}

// ConditionalRenderer renders nothing if the condition is false, otherwise it defers
// to the wrapped Renderer.
func ConditionalRenderer(c Condition, r Renderer) Renderer {
	return conditionalRenderer{c, r}
}

func (cr conditionalRenderer) Render(rpt report.Report) Nodes {
	if cr.Condition(rpt) {
		return cr.Renderer.Render(rpt)
	}
	return Nodes{}
}

// joinResults is used by Renderers that join sets of nodes
type joinResults struct {
	nodes  report.Nodes
	mapped map[string]string   // input node ID -> output node ID - common case
	multi  map[string][]string // input node ID -> output node IDs - exceptional case
}

func newJoinResults(inputNodes report.Nodes) joinResults {
	nodes := make(report.Nodes, len(inputNodes))
	for id, n := range inputNodes {
		n.Adjacency = nil // result() assumes all nodes start with no adjacencies
		nodes[id] = n
	}
	return joinResults{nodes: nodes, mapped: map[string]string{}, multi: map[string][]string{}}
}

func (ret *joinResults) mapChild(from, to string) {
	if _, ok := ret.mapped[from]; !ok {
		ret.mapped[from] = to
	} else {
		ret.multi[from] = append(ret.multi[from], to)
	}
}

// Add m into the results as a top-level node, mapped from original ID
func (ret *joinResults) add(from string, m report.Node) {
	if existing, ok := ret.nodes[m.ID]; ok {
		m = m.Merge(existing)
	}
	ret.nodes[m.ID] = m
	ret.mapChild(from, m.ID)
}

// Add m as a child of the node at id, creating a new result node in
// the specified topology if not already there.
func (ret *joinResults) addUnmappedChild(m report.Node, id string, topology string) {
	result, exists := ret.nodes[id]
	if !exists {
		result = report.MakeNode(id).WithTopology(topology)
	}
	result.Children = result.Children.Add(m)
	if m.Topology != report.Endpoint { // optimisation: we never look at endpoint counts
		result.Counters = result.Counters.Add(m.Topology, 1)
	}
	ret.nodes[id] = result
}

// Add m as a child of the node at id, creating a new result node in
// the specified topology if not already there, and updating the
// mapping from old ID to new ID.
func (ret *joinResults) addChild(m report.Node, id string, topology string) {
	ret.addUnmappedChild(m, id, topology)
	ret.mapChild(m.ID, id)
}

// Like addChild, but also add m's children.
func (ret *joinResults) addChildAndChildren(m report.Node, id string, topology string) {
	ret.addUnmappedChild(m, id, topology)
	result := ret.nodes[id]
	result.Children = result.Children.Merge(m.Children)
	ret.nodes[id] = result
	ret.mapChild(m.ID, id)
}

// Add a copy of n straight into the results
func (ret *joinResults) passThrough(n report.Node) {
	n.Adjacency = nil // result() assumes all nodes start with no adjacencies
	ret.nodes[n.ID] = n
	ret.mapChild(n.ID, n.ID)
}

// Rewrite Adjacency of nodes in ret mapped from original nodes in
// input, and return the result.
func (ret *joinResults) result(input Nodes) Nodes {
	for _, n := range input.Nodes {
		outID, ok := ret.mapped[n.ID]
		if !ok {
			continue
		}
		ret.rewriteAdjacency(outID, n.Adjacency)
		for _, outID := range ret.multi[n.ID] {
			ret.rewriteAdjacency(outID, n.Adjacency)
		}
	}
	return Nodes{Nodes: ret.nodes}
}

func (ret *joinResults) rewriteAdjacency(outID string, adjacency report.IDList) {
	out := ret.nodes[outID]
	// for each adjacency in the original node, find out what it maps
	// to (if any), and add that to the new node
	for _, a := range adjacency {
		if mappedDest, found := ret.mapped[a]; found {
			out.Adjacency = out.Adjacency.Add(mappedDest)
			out.Adjacency = out.Adjacency.Add(ret.multi[a]...)
		}
	}
	ret.nodes[outID] = out
}

// ResetCache blows away the rendered node cache, and known service
// cache.
func ResetCache() {
	renderCache.Purge()
	purgeKnownServiceCache()
}
<|endoftext|>


<commit_before>package validator_test

import (
	"reflect"
	"testing"

	"github.com/go-jstmpl/go-jsvalidator"
	"github.com/gocraft/dbr"
)

func TestNewRequiredValidator(t *testing.T) {
	type Case struct {
		Message    string
		Definition validator.RequiredValidatorDefinition
		Error      error
	}
	cases := []Case{
		{
			Message:    "single element",
			Definition: validator.RequiredValidatorDefinition{Required: []string{"foo"}},
			Error:      nil,
		},
		{
			Message:    "multi elements",
			Definition: validator.RequiredValidatorDefinition{Required: []string{"foo", "bar"}},
			Error:      nil,
		},
		{
			Message:    "empty slice",
			Definition: validator.RequiredValidatorDefinition{Required: []string{}},
			Error:      validator.RequiredDefinitionEmptyError,
		},
		{
			Message:    "duplicated elements",
			Definition: validator.RequiredValidatorDefinition{Required: []string{"foo", "foo"}},
			Error:      validator.RequiredDefinitionDuplicationError,
		},
		{
			Message:    "duplicated elements at first and second",
			Definition: validator.RequiredValidatorDefinition{Required: []string{"foo", "foo", "bar"}},
			Error:      validator.RequiredDefinitionDuplicationError,
		},
		{
			Message:    "duplicated elements at first and end",
			Definition: validator.RequiredValidatorDefinition{Required: []string{"foo", "bar", "foo"}},
			Error:      validator.RequiredDefinitionDuplicationError,
		},
		{
			Message:    "duplicated elements at second end end",
			Definition: validator.RequiredValidatorDefinition{Required: []string{"bar", "foo", "foo"}},
			Error:      validator.RequiredDefinitionDuplicationError,
		},
		{
			Message:    "duplicated all elements",
			Definition: validator.RequiredValidatorDefinition{Required: []string{"foo", "foo", "foo"}},
			Error:      validator.RequiredDefinitionDuplicationError,
		},
	}
	for _, c := range cases {
		_, err := validator.NewRequiredValidator(c.Definition)
		if err != c.Error {
			t.Errorf("Test with %s: fail to NewPatternValidator with error %v", c.Message, err)
		}
	}
}

func TestValidateOfRequiredValidator(t *testing.T) {
	definition := validator.RequiredValidatorDefinition{
		Required: []string{"ID", "Addr"},
	}
	va, err := validator.NewRequiredValidator(definition)
	if err != nil {
		t.Error(err.Error())
	}

	type Sample struct {
		ID   dbr.NullInt64
		Name dbr.NullString
		Addr dbr.NullString
	}
	sample1 := Sample{
		ID:   dbr.NewNullInt64(1),
		Name: dbr.NewNullString("MyName"),
		Addr: dbr.NewNullString("foo@bar.com"),
	}
	sample2 := Sample{
		ID:   dbr.NewNullInt64(2),
		Name: dbr.NewNullString(nil),
		Addr: dbr.NewNullString("foo@bar.com"),
	}
	sample3 := Sample{
		ID:   dbr.NewNullInt64(nil),
		Name: dbr.NewNullString("hi"),
		Addr: dbr.NewNullString("foo@bar.com"),
	}

	type RequiredValidatorTestCase struct {
		Message  string
		Input    interface{}
		Expected error
	}
	cases := []RequiredValidatorTestCase{
		{
			Message: "non-struct",
			Input:   "foo",
			Expected: &validator.InvalidFieldTypeError{
				Input:      "foo",
				Definition: definition,
			},
		},
		{
			Message:  "non-pointer of sample1",
			Input:    sample1,
			Expected: nil,
		},
		{
			Message:  "non-pointer of sample2",
			Input:    sample2,
			Expected: nil,
		},
		{
			Message: "non-pointer of sample3",
			Input:   sample3,
			Expected: &validator.RequiredValidationError{
				Input:      sample3,
				Definition: definition,
			},
		},
		{
			Message:  "pointer of sample1",
			Input:    &sample1,
			Expected: nil,
		},
		{
			Message:  "pointer of sample2",
			Input:    &sample2,
			Expected: nil,
		},
		{
			Message: "pointer of sample3",
			Input:   &sample3,
			Expected: &validator.RequiredValidationError{
				Input:      &sample3,
				Definition: definition,
			},
		},
	}

	for _, c := range cases {
		err := va.Validate(c.Input)
		if !reflect.DeepEqual(err, c.Expected) {
			t.Errorf("%s: expected %+v, but actual %+v", c.Message, c.Expected, err)
		}
	}
}
<commit_msg>Add test cases for #24<commit_after>package validator_test

import (
	"database/sql"
	"reflect"
	"testing"

	"github.com/go-jstmpl/go-jsvalidator"
	"github.com/gocraft/dbr"
)

func TestNewRequiredValidator(t *testing.T) {
	type Case struct {
		Message    string
		Definition validator.RequiredValidatorDefinition
		Error      error
	}
	cases := []Case{
		{
			Message:    "single element",
			Definition: validator.RequiredValidatorDefinition{Required: []string{"foo"}},
			Error:      nil,
		},
		{
			Message:    "multi elements",
			Definition: validator.RequiredValidatorDefinition{Required: []string{"foo", "bar"}},
			Error:      nil,
		},
		{
			Message:    "empty slice",
			Definition: validator.RequiredValidatorDefinition{Required: []string{}},
			Error:      validator.RequiredDefinitionEmptyError,
		},
		{
			Message:    "duplicated elements",
			Definition: validator.RequiredValidatorDefinition{Required: []string{"foo", "foo"}},
			Error:      validator.RequiredDefinitionDuplicationError,
		},
		{
			Message:    "duplicated elements at first and second",
			Definition: validator.RequiredValidatorDefinition{Required: []string{"foo", "foo", "bar"}},
			Error:      validator.RequiredDefinitionDuplicationError,
		},
		{
			Message:    "duplicated elements at first and end",
			Definition: validator.RequiredValidatorDefinition{Required: []string{"foo", "bar", "foo"}},
			Error:      validator.RequiredDefinitionDuplicationError,
		},
		{
			Message:    "duplicated elements at second end end",
			Definition: validator.RequiredValidatorDefinition{Required: []string{"bar", "foo", "foo"}},
			Error:      validator.RequiredDefinitionDuplicationError,
		},
		{
			Message:    "duplicated all elements",
			Definition: validator.RequiredValidatorDefinition{Required: []string{"foo", "foo", "foo"}},
			Error:      validator.RequiredDefinitionDuplicationError,
		},
	}
	for _, c := range cases {
		_, err := validator.NewRequiredValidator(c.Definition)
		if err != c.Error {
			t.Errorf("Test with %s: fail to NewPatternValidator with error %v", c.Message, err)
		}
	}
}

func TestValidateOfRequiredValidator(t *testing.T) {
	definition := validator.RequiredValidatorDefinition{
		Required: []string{"StringValue", "IntValue", "BoolValue"},
	}

	type Native struct {
		StringValue string
		IntValue    int
		BoolValue   bool
	}

	type PtrNative struct {
		StringValue *string
		IntValue    *int
		BoolValue   *bool
	}

	type Null struct {
		StringValue dbr.NullString
		IntValue    dbr.NullInt64
		BoolValue   dbr.NullBool
	}

	type PtrNull struct {
		StringValue *dbr.NullString
		IntValue    *dbr.NullInt64
		BoolValue   *dbr.NullBool
	}

	type RequiredValidatorTestCase struct {
		Message  string
		Input    interface{}
		Expected error
	}

	stringValue := "string value"
	intValue := 1
	boolValue := true

	falsyStringValue := ""
	falsyIntValue := 0
	falsyBoolValue := false

	cases := []RequiredValidatorTestCase{
		{
			Message: "nil",
			Input:   nil,
			Expected: &validator.InvalidFieldTypeError{
				Input:      nil,
				Definition: definition,
			},
		},
		{
			Message: "non-struct",
			Input:   "foo",
			Expected: &validator.InvalidFieldTypeError{
				Input:      "foo",
				Definition: definition,
			},
		},
		{
			Message: "non-pointer struct of non-pointer native type of value",
			Input: Native{
				StringValue: "value",
				IntValue:    1,
				BoolValue:   true,
			},
			Expected: nil,
		},
		{
			Message: "non-pointer struct of non-pointer native type of falsy value",
			Input: Native{
				StringValue: "",
				IntValue:    0,
				BoolValue:   false,
			},
			Expected: nil,
		},
		{
			Message: "non-pointer struct of pointer native type of value",
			Input: PtrNative{
				StringValue: &stringValue,
				IntValue:    &intValue,
				BoolValue:   &boolValue,
			},
			Expected: nil,
		},
		{
			Message: "non-pointer struct of pointer native type of falsy value",
			Input: PtrNative{
				StringValue: &falsyStringValue,
				IntValue:    &falsyIntValue,
				BoolValue:   &falsyBoolValue,
			},
			Expected: nil,
		},
		{
			Message: "non-pointer struct of pointer native type of nil",
			Input: PtrNative{
				StringValue: nil,
				IntValue:    nil,
				BoolValue:   nil,
			},
			Expected: &validator.RequiredValidationError{
				Input: PtrNative{
					StringValue: nil,
					IntValue:    nil,
					BoolValue:   nil,
				},
				Definition: definition,
			},
		},
		{
			Message: "non-pointer struct of non-pointer nullable type of value",
			Input: Null{
				StringValue: dbr.NullString{
					NullString: sql.NullString{
						String: "value",
						Valid:  true,
					},
				},
				IntValue: dbr.NullInt64{
					NullInt64: sql.NullInt64{
						Int64: 1,
						Valid: true,
					},
				},
				BoolValue: dbr.NullBool{
					NullBool: sql.NullBool{
						Bool:  true,
						Valid: true,
					},
				},
			},
			Expected: nil,
		},
		{
			Message: "non-pointer struct of non-pointer nullable type of falsy value",
			Input: Null{
				StringValue: dbr.NullString{
					NullString: sql.NullString{
						String: "",
						Valid:  true,
					},
				},
				IntValue: dbr.NullInt64{
					NullInt64: sql.NullInt64{
						Int64: 0,
						Valid: true,
					},
				},
				BoolValue: dbr.NullBool{
					NullBool: sql.NullBool{
						Bool:  false,
						Valid: true,
					},
				},
			},
			Expected: nil,
		},
		{
			Message: "non-pointer struct of non-pointer nullable type of null value",
			Input: Null{
				StringValue: dbr.NullString{
					NullString: sql.NullString{
						String: "",
						Valid:  false,
					},
				},
				IntValue: dbr.NullInt64{
					NullInt64: sql.NullInt64{
						Int64: 0,
						Valid: false,
					},
				},
				BoolValue: dbr.NullBool{
					NullBool: sql.NullBool{
						Bool:  false,
						Valid: false,
					},
				},
			},
			Expected: &validator.RequiredValidationError{
				Input: Null{
					StringValue: dbr.NullString{
						NullString: sql.NullString{
							String: "",
							Valid:  false,
						},
					},
					IntValue: dbr.NullInt64{
						NullInt64: sql.NullInt64{
							Int64: 0,
							Valid: false,
						},
					},
					BoolValue: dbr.NullBool{
						NullBool: sql.NullBool{
							Bool:  false,
							Valid: false,
						},
					},
				},
				Definition: definition,
			},
		},
		{
			Message: "non-pointer struct of pointer nullable type of value",
			Input: PtrNull{
				StringValue: &dbr.NullString{
					NullString: sql.NullString{
						String: "value",
						Valid:  true,
					},
				},
				IntValue: &dbr.NullInt64{
					NullInt64: sql.NullInt64{
						Int64: 1,
						Valid: true,
					},
				},
				BoolValue: &dbr.NullBool{
					NullBool: sql.NullBool{
						Bool:  true,
						Valid: true,
					},
				},
			},
			Expected: nil,
		},
		{
			Message: "non-pointer struct of pointer nullable type of falsy value",
			Input: PtrNull{
				StringValue: &dbr.NullString{
					NullString: sql.NullString{
						String: "",
						Valid:  true,
					},
				},
				IntValue: &dbr.NullInt64{
					NullInt64: sql.NullInt64{
						Int64: 0,
						Valid: true,
					},
				},
				BoolValue: &dbr.NullBool{
					NullBool: sql.NullBool{
						Bool:  false,
						Valid: true,
					},
				},
			},
			Expected: nil,
		},
		{
			Message: "non-pointer struct of pointer nullable type of null value",
			Input: PtrNull{
				StringValue: &dbr.NullString{
					NullString: sql.NullString{
						String: "",
						Valid:  false,
					},
				},
				IntValue: &dbr.NullInt64{
					NullInt64: sql.NullInt64{
						Int64: 0,
						Valid: false,
					},
				},
				BoolValue: &dbr.NullBool{
					NullBool: sql.NullBool{
						Bool:  false,
						Valid: false,
					},
				},
			},
			Expected: &validator.RequiredValidationError{
				Input: PtrNull{
					StringValue: &dbr.NullString{
						NullString: sql.NullString{
							String: "",
							Valid:  false,
						},
					},
					IntValue: &dbr.NullInt64{
						NullInt64: sql.NullInt64{
							Int64: 0,
							Valid: false,
						},
					},
					BoolValue: &dbr.NullBool{
						NullBool: sql.NullBool{
							Bool:  false,
							Valid: false,
						},
					},
				},
				Definition: definition,
			},
		},
		{
			Message: "non-pointer struct of pointer nullable type of nil",
			Input: PtrNull{
				StringValue: nil,
				IntValue:    nil,
				BoolValue:   nil,
			},
			Expected: &validator.RequiredValidationError{
				Input: PtrNull{
					StringValue: nil,
					IntValue:    nil,
					BoolValue:   nil,
				},
				Definition: definition,
			},
		},

		{
			Message: "pointer struct of non-pointer native type of value",
			Input: &Native{
				StringValue: "value",
				IntValue:    1,
				BoolValue:   true,
			},
			Expected: nil,
		},
		{
			Message: "pointer struct of non-pointer native type of falsy value",
			Input: &Native{
				StringValue: "",
				IntValue:    0,
				BoolValue:   false,
			},
			Expected: nil,
		},
		{
			Message: "pointer struct of pointer native type of value",
			Input: &PtrNative{
				StringValue: &stringValue,
				IntValue:    &intValue,
				BoolValue:   &boolValue,
			},
			Expected: nil,
		},
		{
			Message: "pointer struct of pointer native type of falsy value",
			Input: &PtrNative{
				StringValue: &falsyStringValue,
				IntValue:    &falsyIntValue,
				BoolValue:   &falsyBoolValue,
			},
			Expected: nil,
		},
		{
			Message: "pointer struct of pointer native type of nil",
			Input: &PtrNative{
				StringValue: nil,
				IntValue:    nil,
				BoolValue:   nil,
			},
			Expected: &validator.RequiredValidationError{
				Input: &PtrNative{
					StringValue: nil,
					IntValue:    nil,
					BoolValue:   nil,
				},
				Definition: definition,
			},
		},

		{
			Message: "pointer struct of non-pointer nullable type of value",
			Input: &Null{
				StringValue: dbr.NullString{
					NullString: sql.NullString{
						String: "value",
						Valid:  true,
					},
				},
				IntValue: dbr.NullInt64{
					NullInt64: sql.NullInt64{
						Int64: 1,
						Valid: true,
					},
				},
				BoolValue: dbr.NullBool{
					NullBool: sql.NullBool{
						Bool:  true,
						Valid: true,
					},
				},
			},
			Expected: nil,
		},
		{
			Message: "pointer struct of non-pointer nullable type of falsy value",
			Input: &Null{
				StringValue: dbr.NullString{
					NullString: sql.NullString{
						String: "",
						Valid:  true,
					},
				},
				IntValue: dbr.NullInt64{
					NullInt64: sql.NullInt64{
						Int64: 0,
						Valid: true,
					},
				},
				BoolValue: dbr.NullBool{
					NullBool: sql.NullBool{
						Bool:  false,
						Valid: true,
					},
				},
			},
			Expected: nil,
		},
		{
			Message: "pointer struct of non-pointer nullable type of null value",
			Input: &Null{
				StringValue: dbr.NullString{
					NullString: sql.NullString{
						String: "",
						Valid:  false,
					},
				},
				IntValue: dbr.NullInt64{
					NullInt64: sql.NullInt64{
						Int64: 0,
						Valid: false,
					},
				},
				BoolValue: dbr.NullBool{
					NullBool: sql.NullBool{
						Bool:  false,
						Valid: false,
					},
				},
			},
			Expected: &validator.RequiredValidationError{
				Input: &Null{
					StringValue: dbr.NullString{
						NullString: sql.NullString{
							String: "",
							Valid:  false,
						},
					},
					IntValue: dbr.NullInt64{
						NullInt64: sql.NullInt64{
							Int64: 0,
							Valid: false,
						},
					},
					BoolValue: dbr.NullBool{
						NullBool: sql.NullBool{
							Bool:  false,
							Valid: false,
						},
					},
				},
				Definition: definition,
			},
		},
		{
			Message: "pointer struct of pointer nullable type of value",
			Input: &PtrNull{
				StringValue: &dbr.NullString{
					NullString: sql.NullString{
						String: "value",
						Valid:  true,
					},
				},
				IntValue: &dbr.NullInt64{
					NullInt64: sql.NullInt64{
						Int64: 1,
						Valid: true,
					},
				},
				BoolValue: &dbr.NullBool{
					NullBool: sql.NullBool{
						Bool:  true,
						Valid: true,
					},
				},
			},
			Expected: nil,
		},
		{
			Message: "pointer struct of pointer nullable type of falsy value",
			Input: &PtrNull{
				StringValue: &dbr.NullString{
					NullString: sql.NullString{
						String: "",
						Valid:  true,
					},
				},
				IntValue: &dbr.NullInt64{
					NullInt64: sql.NullInt64{
						Int64: 0,
						Valid: true,
					},
				},
				BoolValue: &dbr.NullBool{
					NullBool: sql.NullBool{
						Bool:  false,
						Valid: true,
					},
				},
			},
			Expected: nil,
		},
		{
			Message: "pointer struct of pointer nullable type of null value",
			Input: &PtrNull{
				StringValue: &dbr.NullString{
					NullString: sql.NullString{
						String: "",
						Valid:  false,
					},
				},
				IntValue: &dbr.NullInt64{
					NullInt64: sql.NullInt64{
						Int64: 0,
						Valid: false,
					},
				},
				BoolValue: &dbr.NullBool{
					NullBool: sql.NullBool{
						Bool:  false,
						Valid: false,
					},
				},
			},
			Expected: &validator.RequiredValidationError{
				Input: &PtrNull{
					StringValue: &dbr.NullString{
						NullString: sql.NullString{
							String: "",
							Valid:  false,
						},
					},
					IntValue: &dbr.NullInt64{
						NullInt64: sql.NullInt64{
							Int64: 0,
							Valid: false,
						},
					},
					BoolValue: &dbr.NullBool{
						NullBool: sql.NullBool{
							Bool:  false,
							Valid: false,
						},
					},
				},
				Definition: definition,
			},
		},
		{
			Message: "pointer struct of pointer nullable type of nil",
			Input: &PtrNull{
				StringValue: nil,
				IntValue:    nil,
				BoolValue:   nil,
			},
			Expected: &validator.RequiredValidationError{
				Input: &PtrNull{
					StringValue: nil,
					IntValue:    nil,
					BoolValue:   nil,
				},
				Definition: definition,
			},
		},
	}

	va, err := validator.NewRequiredValidator(definition)
	if err != nil {
		t.Error(err.Error())
	}

	for _, c := range cases {
		err := va.Validate(c.Input)
		if !reflect.DeepEqual(err, c.Expected) {
			t.Errorf("Test with %s: expected %+v, but actual %+v", c.Message, c.Expected, err)
		}
	}
}
<|endoftext|>


<commit_before>/*
Command line tool for updating Dockerfiles based on versions.yaml.
*/
package main

import (
	"bytes"
	"flag"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"text/template"

	"github.com/GoogleCloudPlatform/runtimes-common/versioning/versions"
)

type indentFormat string

// indent replaces leading spaces in the input string with the
// value of the indentFormat object.
func (f indentFormat) indent(s string) string {
	temp := strings.Split(s, "\n")
	str := ""
	for index, line := range temp {
		if index > 0 {
			str = str + "\n"
		}
		trimmed := strings.TrimLeft(line, " ")
		diff := len(line) - len(trimmed)
		prefix := strings.Repeat(string(f), diff)
		str = str + prefix + trimmed
	}
	return str
}

const keyServersRetryTemplate = `found='' && \
for server in \
 pool.sks-keyservers.net \
 na.pool.sks-keyservers.net \
 eu.pool.sks-keyservers.net \
 oc.pool.sks-keyservers.net \
 ha.pool.sks-keyservers.net \
 hkp://p80.pool.sks-keyservers.net:80 \
 hkp://keyserver.ubuntu.com:80 \
 pgp.mit.edu \
; do \
 {{ . }} \
  && found=yes && break; \
done; \
test -n "$found"`

func funcKeyServersRetryLoop(indentSequence string, cmd string) string {
	f := indentFormat(indentSequence)
	tmpl, err := template.New("retryTemplate").Parse(f.indent(keyServersRetryTemplate))
	check(err)
	var result bytes.Buffer
	tmpl.Execute(&result, cmd)
	return funcIndent(indentSequence, string(result.Bytes()))
}

func funcIndent(leading string, s string) string {
	temp := strings.Split(s, "\n")
	str := ""
	for index, line := range temp {
		if index > 0 {
			str = str + "\n" + leading
		}
		str = str + line
	}
	return str
}

func renderDockerfile(version versions.Version, tmpl template.Template) []byte {
	var result bytes.Buffer
	tmpl.Execute(&result, version)
	return result.Bytes()
}

func writeDockerfile(version versions.Version, data []byte) {
	path := filepath.Join(version.Dir, "Dockerfile")
	// Delete first to make sure file is created with the right mode.
	deleteIfFileExists(path)
	err := ioutil.WriteFile(path, data, 0644)
	check(err)
}

func findFilesToCopy(templateDir string, callback func(path string, fileInfo os.FileInfo)) {
	filepath.Walk(templateDir, func(path string, info os.FileInfo, err error) error {
		check(err)
		if strings.HasSuffix(info.Name(), ".template") || info.IsDir() {
			return nil
		}
		path, err = filepath.Rel(templateDir, path)
		check(err)
		callback(path, info)
		return nil
	})
}

func copyFiles(version versions.Version, templateDir string) {
	findFilesToCopy(templateDir, func(path string, fileInfo os.FileInfo) {
		data, err := ioutil.ReadFile(filepath.Join(templateDir, path))
		check(err)

		target := filepath.Join(version.Dir, path)
		// Delete first to make sure file is created with the right mode.
		deleteIfFileExists(target)
		err = ioutil.WriteFile(target, data, fileInfo.Mode())
		check(err)
	})
}

func deleteIfFileExists(path string) {
	if fileInfo, err := os.Stat(path); err != nil {
		if !os.IsNotExist(err) {
			log.Fatalf("File %s exists but cannot be stat'ed", path)
		}
	} else {
		if fileInfo.IsDir() {
			log.Fatalf("%s is unexpectedly a directory", path)
		}
		err = os.Remove(path)
		check(err)
	}
}

func verifyDockerfiles(spec versions.Spec, tmpl template.Template) (failureCount int) {
	foundDockerfile := make(map[string]bool)
	failureCount = 0
	warningCount := 0

	for _, version := range spec.Versions {
		data := renderDockerfile(version, tmpl)

		path := filepath.Join(version.Dir, "Dockerfile")
		dockerfile, err := ioutil.ReadFile(path)
		check(err)

		foundDockerfile[path] = true

		if string(dockerfile) == string(data) {
			log.Printf("%s: OK", path)
		} else {
			failureCount++
			log.Printf("%s: FAILED", path)
		}
	}

	err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
		check(err)
		if info.Name() == "Dockerfile" && !info.IsDir() && !foundDockerfile[path] {
			warningCount++
			log.Printf("%s: UNIDENTIFIED (warning)", path)
		}
		return nil
	})
	check(err)

	if failureCount == 0 && warningCount > 0 {
		log.Print("Dockerfile verification completed: PASSED (with warnings)")
	} else if failureCount == 0 {
		log.Print("Dockerfile verification completed: PASSED")
	} else {
		log.Print("Dockerfile verification completed: FAILED")
	}

	return
}

func verifyCopiedFiles(spec versions.Spec, templateDir string) (failureCount int) {
	failureCount = 0
	for _, version := range spec.Versions {
		tmplDirPath := filepath.Join(templateDir, version.TemplateSubDir)

		findFilesToCopy(tmplDirPath, func(path string, sourceFileInfo os.FileInfo) {
			failureCount++

			source := filepath.Join(tmplDirPath, path)
			target := filepath.Join(version.Dir, path)
			targetFileInfo, err := os.Stat(target)
			if err != nil {
				log.Printf("%s is expected but cannot be stat'ed", target)
				log.Printf("Please, check accessability of %s", source)
				return
			}

			// Check mode for owner only.
			sourcePerm := os.FileMode(sourceFileInfo.Mode().Perm() & 0700)
			targetPerm := os.FileMode(targetFileInfo.Mode().Perm() & 0700)
			if sourcePerm != targetPerm {
				log.Printf("%s has wrong file mode %v, expected %v", target, targetPerm, sourcePerm)
				return
			}

			expected, err := ioutil.ReadFile(source)
			check(err)
			actual, err := ioutil.ReadFile(filepath.Join(version.Dir, path))
			if err != nil {
				log.Printf("%s is expected but cannot be read", target)
				return
			}

			if !reflect.DeepEqual(expected, actual) {
				log.Printf("%s content is different from its template", target)
				return
			}

			log.Printf("%s: OK", target)

			failureCount--
		})
	}

	if failureCount == 0 {
		log.Print("Copied files verification completed: PASSED")
	} else {
		log.Print("Copied files verification completed: FAILED")
	}

	return
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func main() {
	defaultTemplateDirPtr := flag.String("template_dir", "templates",
		"Path to directory containing Dockerfile.template and any other files to copy over")
	verifyPtr := flag.Bool("verify_only", false, "Verify dockerfiles")
	var version = "0.01"

	// printing version of the tool; version should be bumped up
	// whenever one makes changes (initially it'll be done manually and ultimately automatically)
	log.Printf("dockerfiles version: %s", version)

	flag.Parse()

	var spec versions.Spec
	spec = versions.LoadVersions("versions.yaml")

	for _, version := range spec.Versions {
		// templatePath - path to Dockerfile.template
		templatePath := filepath.Join(*defaultTemplateDirPtr, version.TemplateSubDir, "Dockerfile.template")
		templateData, err := ioutil.ReadFile(templatePath)
		templateString := string(templateData)
		check(err)

    tmpl, err := template.
			New("dockerfileTemplate").
			Funcs(template.FuncMap{"KeyServersRetryLoop": funcKeyServersRetryLoop}).
			Parse(templateString)
		check(err)

		if *verifyPtr {
			failureCount := verifyDockerfiles(spec, *tmpl)
			failureCount += verifyCopiedFiles(spec, *defaultTemplateDirPtr)
			os.Exit(failureCount)
		} else {
			data := renderDockerfile(version, *tmpl)
			writeDockerfile(version, data)
			// if version.TemplateSubDir is empty then we default to 'templates' folder
			copyFiles(version, filepath.Join(*defaultTemplateDirPtr, version.TemplateSubDir))
		}
	}
}
<commit_msg>formatting in main() function was corrected<commit_after>/*
Command line tool for updating Dockerfiles based on versions.yaml.
*/
package main

import (
	"bytes"
	"flag"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"text/template"

	"github.com/GoogleCloudPlatform/runtimes-common/versioning/versions"
)

type indentFormat string

// indent replaces leading spaces in the input string with the
// value of the indentFormat object.
func (f indentFormat) indent(s string) string {
	temp := strings.Split(s, "\n")
	str := ""
	for index, line := range temp {
		if index > 0 {
			str = str + "\n"
		}
		trimmed := strings.TrimLeft(line, " ")
		diff := len(line) - len(trimmed)
		prefix := strings.Repeat(string(f), diff)
		str = str + prefix + trimmed
	}
	return str
}

const keyServersRetryTemplate = `found='' && \
for server in \
 pool.sks-keyservers.net \
 na.pool.sks-keyservers.net \
 eu.pool.sks-keyservers.net \
 oc.pool.sks-keyservers.net \
 ha.pool.sks-keyservers.net \
 hkp://p80.pool.sks-keyservers.net:80 \
 hkp://keyserver.ubuntu.com:80 \
 pgp.mit.edu \
; do \
 {{ . }} \
  && found=yes && break; \
done; \
test -n "$found"`

func funcKeyServersRetryLoop(indentSequence string, cmd string) string {
	f := indentFormat(indentSequence)
	tmpl, err := template.New("retryTemplate").Parse(f.indent(keyServersRetryTemplate))
	check(err)
	var result bytes.Buffer
	tmpl.Execute(&result, cmd)
	return funcIndent(indentSequence, string(result.Bytes()))
}

func funcIndent(leading string, s string) string {
	temp := strings.Split(s, "\n")
	str := ""
	for index, line := range temp {
		if index > 0 {
			str = str + "\n" + leading
		}
		str = str + line
	}
	return str
}

func renderDockerfile(version versions.Version, tmpl template.Template) []byte {
	var result bytes.Buffer
	tmpl.Execute(&result, version)
	return result.Bytes()
}

func writeDockerfile(version versions.Version, data []byte) {
	path := filepath.Join(version.Dir, "Dockerfile")
	// Delete first to make sure file is created with the right mode.
	deleteIfFileExists(path)
	err := ioutil.WriteFile(path, data, 0644)
	check(err)
}

func findFilesToCopy(templateDir string, callback func(path string, fileInfo os.FileInfo)) {
	filepath.Walk(templateDir, func(path string, info os.FileInfo, err error) error {
		check(err)
		if strings.HasSuffix(info.Name(), ".template") || info.IsDir() {
			return nil
		}
		path, err = filepath.Rel(templateDir, path)
		check(err)
		callback(path, info)
		return nil
	})
}

func copyFiles(version versions.Version, templateDir string) {
	findFilesToCopy(templateDir, func(path string, fileInfo os.FileInfo) {
		data, err := ioutil.ReadFile(filepath.Join(templateDir, path))
		check(err)

		target := filepath.Join(version.Dir, path)
		// Delete first to make sure file is created with the right mode.
		deleteIfFileExists(target)
		err = ioutil.WriteFile(target, data, fileInfo.Mode())
		check(err)
	})
}

func deleteIfFileExists(path string) {
	if fileInfo, err := os.Stat(path); err != nil {
		if !os.IsNotExist(err) {
			log.Fatalf("File %s exists but cannot be stat'ed", path)
		}
	} else {
		if fileInfo.IsDir() {
			log.Fatalf("%s is unexpectedly a directory", path)
		}
		err = os.Remove(path)
		check(err)
	}
}

func verifyDockerfiles(spec versions.Spec, tmpl template.Template) (failureCount int) {
	foundDockerfile := make(map[string]bool)
	failureCount = 0
	warningCount := 0

	for _, version := range spec.Versions {
		data := renderDockerfile(version, tmpl)

		path := filepath.Join(version.Dir, "Dockerfile")
		dockerfile, err := ioutil.ReadFile(path)
		check(err)

		foundDockerfile[path] = true

		if string(dockerfile) == string(data) {
			log.Printf("%s: OK", path)
		} else {
			failureCount++
			log.Printf("%s: FAILED", path)
		}
	}

	err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
		check(err)
		if info.Name() == "Dockerfile" && !info.IsDir() && !foundDockerfile[path] {
			warningCount++
			log.Printf("%s: UNIDENTIFIED (warning)", path)
		}
		return nil
	})
	check(err)

	if failureCount == 0 && warningCount > 0 {
		log.Print("Dockerfile verification completed: PASSED (with warnings)")
	} else if failureCount == 0 {
		log.Print("Dockerfile verification completed: PASSED")
	} else {
		log.Print("Dockerfile verification completed: FAILED")
	}

	return
}

func verifyCopiedFiles(spec versions.Spec, templateDir string) (failureCount int) {
	failureCount = 0
	for _, version := range spec.Versions {
		tmplDirPath := filepath.Join(templateDir, version.TemplateSubDir)

		findFilesToCopy(tmplDirPath, func(path string, sourceFileInfo os.FileInfo) {
			failureCount++

			source := filepath.Join(tmplDirPath, path)
			target := filepath.Join(version.Dir, path)
			targetFileInfo, err := os.Stat(target)
			if err != nil {
				log.Printf("%s is expected but cannot be stat'ed", target)
				log.Printf("Please, check accessability of %s", source)
				return
			}

			// Check mode for owner only.
			sourcePerm := os.FileMode(sourceFileInfo.Mode().Perm() & 0700)
			targetPerm := os.FileMode(targetFileInfo.Mode().Perm() & 0700)
			if sourcePerm != targetPerm {
				log.Printf("%s has wrong file mode %v, expected %v", target, targetPerm, sourcePerm)
				return
			}

			expected, err := ioutil.ReadFile(source)
			check(err)
			actual, err := ioutil.ReadFile(filepath.Join(version.Dir, path))
			if err != nil {
				log.Printf("%s is expected but cannot be read", target)
				return
			}

			if !reflect.DeepEqual(expected, actual) {
				log.Printf("%s content is different from its template", target)
				return
			}

			log.Printf("%s: OK", target)

			failureCount--
		})
	}

	if failureCount == 0 {
		log.Print("Copied files verification completed: PASSED")
	} else {
		log.Print("Copied files verification completed: FAILED")
	}

	return
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func main() {
	defaultTemplateDirPtr := flag.String("template_dir", "templates",
		"Path to directory containing Dockerfile.template and any other files to copy over")
	verifyPtr := flag.Bool("verify_only", false, "Verify dockerfiles")
	var version = "0.01"

	// printing version of the tool; version should be bumped up
	// whenever one makes changes (initially it'll be done manually and ultimately automatically)
	log.Printf("dockerfiles version: %s", version)

	flag.Parse()

	var spec versions.Spec
	spec = versions.LoadVersions("versions.yaml")

	for _, version := range spec.Versions {
		// templatePath - path to Dockerfile.template
		templatePath := filepath.Join(*defaultTemplateDirPtr, version.TemplateSubDir, "Dockerfile.template")
		templateData, err := ioutil.ReadFile(templatePath)
		templateString := string(templateData)
		check(err)

		tmpl, err := template.
			New("dockerfileTemplate").
			Funcs(template.FuncMap{"KeyServersRetryLoop": funcKeyServersRetryLoop}).
			Parse(templateString)
		check(err)

		if *verifyPtr {
			failureCount := verifyDockerfiles(spec, *tmpl)
			failureCount += verifyCopiedFiles(spec, *defaultTemplateDirPtr)
			os.Exit(failureCount)
		} else {
			data := renderDockerfile(version, *tmpl)
			writeDockerfile(version, data)
			// if version.TemplateSubDir is empty then we default to 'templates' folder
			copyFiles(version, filepath.Join(*defaultTemplateDirPtr, version.TemplateSubDir))
		}
	}
}
<|endoftext|>


<commit_before>package space

import (
	"bytes"
	"log"
	"os"
	"path/filepath"
)

type FileInfo os.FileInfo
type FileInfos map[string]FileInfo

type Filesystem interface {
	Add(string, FileInfo, string)
	Walk(string) FileInfos
	Files() []*File
}

type filesystem struct {
	files []*File
}

func NewFilesystem() Filesystem {
	return &filesystem{}
}

func (fs *filesystem) Files() []*File {
	return fs.files
}

func (fs *filesystem) Add(realpath string, fi FileInfo, basepath string) {
	// relative path
	path, _ := filepath.Rel(basepath, realpath)

	fs.files = append(fs.files, &File{
		Path:     path,
		Buffer:   &bytes.Buffer{},
		Info:     fi,
		realpath: realpath,
	})
}

func (fs *filesystem) Walk(path string) (a FileInfos) {
	a = make(FileInfos, 0)
	walker := func(name string, fi os.FileInfo, err error) error {
		if err != nil {
			log.Println("Walker: ", "Please check source dir.")
			return nil
		}

		dot := filepath.Base(name)[0]
		isDir := fi.IsDir()
		isSkip := dot == '.'

		if isSkip && isDir {
			return filepath.SkipDir
		}

		if isSkip == isDir {
			a[name] = fi
		}
		return nil
	}

	filepath.Walk(path, walker)
	return a
}
<commit_msg>support for root path is symlinks<commit_after>package space

import (
	"bytes"
	"log"
	"os"
	"path/filepath"
)

type FileInfo os.FileInfo
type FileInfos map[string]FileInfo

type Filesystem interface {
	Add(string, FileInfo, string)
	Walk(string) FileInfos
	Files() []*File
}

type filesystem struct {
	files []*File
}

func NewFilesystem() Filesystem {
	return &filesystem{}
}

func (fs *filesystem) Files() []*File {
	return fs.files
}

func (fs *filesystem) Add(realpath string, fi FileInfo, basepath string) {
	// relative path
	path, _ := filepath.Rel(basepath, realpath)

	fs.files = append(fs.files, &File{
		Path:     path,
		Buffer:   &bytes.Buffer{},
		Info:     fi,
		realpath: realpath,
	})
}

func (fs *filesystem) Walk(path string) (a FileInfos) {
	a = make(FileInfos, 0)
	path, _ = filepath.EvalSymlinks(path)
	walker := func(name string, fi os.FileInfo, err error) error {
		if err != nil {
			log.Println("Walker: ", "Please check source dir.")
			return nil
		}

		dot := filepath.Base(name)[0]
		isDir := fi.IsDir()
		isSkip := dot == '.'

		if isSkip && isDir {
			return filepath.SkipDir
		}

		if isSkip == isDir {
			a[name] = fi
		}
		return nil
	}

	filepath.Walk(path, walker)
	return a
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 The SurgeMQ Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"bufio"
	"fmt"
	"io"
	"sync/atomic"
	//"github.com/surgemq/message"
	"encoding/binary"
	"errors"
	"math"
	"sync"
	"time"
)

var (
	bufcnt            int64
	DefaultBufferSize int64

	DeviceInBufferSize  int64
	DeviceOutBufferSize int64

	MasterInBufferSize  int64
	MasterOutBufferSize int64
)

const (
	smallReadBlockSize    = 512
	defaultReadBlockSize  = 8192
	defaultWriteBlockSize = 8192
)

/**
2016.03.03 修改
bingbuffer结构体
*/
type buffer struct {
	readIndex  int64        //读序号
	writeIndex int64        //写序号
	ringBuffer []*ByteArray //环形buffer指针数组
	bufferSize int64        //初始化环形buffer指针数组大小
	mask       int64        //掩码：bufferSize-1
	done       int64        //是否完成
	rcond      *sync.Cond
	wcond      *sync.Cond
}

type ByteArray struct {
	bArray []byte
}

func (this *buffer) ReadCommit(index int64, ok bool) {
	this.rcond.L.Lock()
	defer this.rcond.L.Unlock()
	if ok {
		this.ringBuffer[index] = nil
	}
	this.wcond.Broadcast()
}

func (this *buffer) Len() int {
	cpos := this.GetCurrentReadIndex()
	ppos := this.GetCurrentWriteIndex()
	return int(ppos - cpos)
}

/**
2016.03.03 添加
初始化ringbuffer
参数bufferSize：初始化环形buffer指针数组大小
*/
func newBuffer(size int64) (*buffer, error) {
	if size < 0 {
		return nil, bufio.ErrNegativeCount
	}
	if size == 0 {
		size = DefaultBufferSize
	}
	if !powerOfTwo64(size) {
		fmt.Printf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
		return nil, fmt.Errorf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
	}

	return &buffer{
		readIndex:  int64(0),                 //读序号
		writeIndex: int64(0),                 //写序号
		ringBuffer: make([]*ByteArray, size), //环形buffer指针数组
		bufferSize: size,                     //初始化环形buffer指针数组大小
		mask:       size - 1,
		rcond:      sync.NewCond(new(sync.Mutex)),
		wcond:      sync.NewCond(new(sync.Mutex)),
	}, nil
}

/**
2016.03.03 添加
获取当前读序号
*/
func (this *buffer) GetCurrentReadIndex() int64 {
	return atomic.LoadInt64(&this.readIndex)
}

/**
2016.03.03 添加
获取当前写序号
*/
func (this *buffer) GetCurrentWriteIndex() int64 {
	return atomic.LoadInt64(&this.writeIndex)
}

/**
2016.03.03 添加
读取ringbuffer指定的buffer指针，返回该指针并清空ringbuffer该位置存在的指针内容，以及将读序号加1
*/
func (this *buffer) ReadBuffer() ([]byte, int64, bool) {
	this.rcond.L.Lock()
	defer this.rcond.L.Unlock()
	for {
		readIndex := atomic.LoadInt64(&this.readIndex)
		writeIndex := atomic.LoadInt64(&this.writeIndex)
		switch {
		case readIndex >= writeIndex:
			this.rcond.Wait()
		case writeIndex-readIndex > this.bufferSize:
			this.rcond.Wait()
		default:
			if readIndex == math.MaxInt64 {
				atomic.StoreInt64(&this.readIndex, int64(0))
			} else {
				atomic.AddInt64(&this.readIndex, int64(1))
			}

			index := readIndex & this.mask

			p_ := this.ringBuffer[index]
			//this.ringBuffer[index] = nil
			if p_ == nil {
				return nil, -1, false
			}
			p := p_.bArray

			if p == nil {
				return nil, -1, false
			}

			return p, index, true
		}
	}

}

/**
2016.03.03 添加
写入ringbuffer指针，以及将写序号加1
*/
func (this *buffer) WriteBuffer(in []byte) bool {
	this.wcond.L.Lock()
	defer func() {
		this.rcond.Broadcast()
		this.wcond.L.Unlock()
	}()

	for {
		readIndex := atomic.LoadInt64(&this.readIndex)
		writeIndex := atomic.LoadInt64(&this.writeIndex)
		switch {
		case writeIndex-readIndex < 0:
			this.wcond.Wait()
		default:
			index := writeIndex & this.mask
			if writeIndex == math.MaxInt64 {
				atomic.StoreInt64(&this.writeIndex, int64(0))
			} else {
				atomic.AddInt64(&this.writeIndex, int64(1))
			}
			if this.ringBuffer[index] == nil {
				this.ringBuffer[index] = &ByteArray{bArray: in}
				return true
			} else {
				return false
			}
		}
	}

}

/**
2016.03.03 修改
完成
*/
func (this *buffer) Close() error {
	atomic.StoreInt64(&this.done, 1)

	this.wcond.L.Lock()
	this.rcond.Broadcast()
	this.wcond.L.Unlock()

	this.rcond.L.Lock()
	this.wcond.Broadcast()
	this.rcond.L.Unlock()

	return nil
}

/*

/**
2016.03.03 修改
向ringbuffer中写数据（从connection的中向ringbuffer中写）--生产者
*/
func (this *buffer) ReadFrom(r io.Reader) (int64, error) {
	defer this.Close()
	total := int64(0)
	//for {

	//if this.isDone() {
	//	return total, io.EOF
	//}
	b := make([]byte, int64(5))
	n, err := r.Read(b[0:1])
	fmt.Println("readfrom 1st read conn:", err, "读取数量：", n)
	if n > 0 {
		total += int64(n)
		if err != nil {
			return total, err
		}
	}

	/**************************/
	cnt := 1

	// Let's read enough bytes to get the message header (msg type, remaining length)
	for {
		// If we have read 5 bytes and still not done, then there's a problem.
		if cnt > 4 {
			return 0, fmt.Errorf("sendrecv/peekMessageSize: 4th byte of remaining length has continuation bit set")
		}

		//Log.Infoc(func() string {
		//	return fmt.Sprintf("sendrecv/peekMessageSize: %d=========", cnt)
		//})
		// Peek cnt bytes from the input buffer.

		_, err := r.Read(b[cnt:(cnt + 1)])
		fmt.Println("readfrom 2th read conn:", err, "读取数量：", n)
		//fmt.Println(b)
		if err != nil {
			return 0, err
		}
		// If we got enough bytes, then check the last byte to see if the continuation
		// bit is set. If so, increment cnt and continue peeking
		if b[cnt] >= 0x80 {
			cnt++
		} else {
			break
		}
	}

	// Get the remaining length of the message
	remlen, m := binary.Uvarint(b[1:])
	//Log.Infoc(func() string {
	//	return fmt.Sprintf("b[cnt:(cnt + 1)]==end")
	//})
	// Total message length is remlen + 1 (msg type) + m (remlen bytes)

	total = int64(remlen) + int64(1) + int64(m)
	//Log.Infoc(func() string {
	//	return fmt.Sprintf("remlen===n===totle: %d===%d===%d", remlen, m, total)
	//})
	//mtype := message.MessageType(b[0] >> 4)
	/****************/
	//var msg message.Message
	//
	//msg, err = mtype.New()
	//if err != nil {
	//	return 0, err
	//}
	b__ := make([]byte, 0, total)
	b__ = append(b__, b[0:1+m]...)
	for i := int64(0); i < int64(remlen); i++ {
		b_ := make([]byte, int64(remlen))
		n, err = r.Read(b_[0:])

		if err != nil {
			Log.Errorc(func() string {
				return fmt.Sprintf("从conn读取数据失败(%s)", err)
			})
			time.Sleep(2 * time.Millisecond)
			//return total, err
		}
		b__ = append(b__, b_[0:]...)
	}

	fmt.Println("readfrom 3th read conn:", err, "读取数量：", n)
	fmt.Println(b__)
	//n, err = msg.Decode(b)
	//if err != nil {
	//	return 0, err
	//}

	/*************************/

	if !this.WriteBuffer(b__) {
		return total, err
	}

	return total, nil
	//}
}

/**
2016.03.03 修改
*/
func (this *buffer) WriteTo(w io.Writer) (int64, error) {
	defer this.Close()
	total := int64(0)
	//for {
	//if this.isDone() {
	//	return total, io.EOF
	//}
	p, index, ok := this.ReadBuffer()
	defer this.ReadCommit(index, ok)
	if !ok {
		return total, errors.New("读取过程中出现问题")
	}

	Log.Debugc(func() string {
		return fmt.Sprintf("WriteTo方法读取ringbuffer成功(%d)", index)
	})
	//Log.Debugc(func() string {
	//	return fmt.Sprintf("WriteTo函数》》读取*p：" + p)
	//})

	//
	//Log.Errorc(func() string {
	//	return fmt.Sprintf("msg::" + msg.Name())
	//})
	//
	//p := make([]byte, msg.Len())
	//_, err := msg.Encode(p)
	//if err != nil {
	//	Log.Errorc(func() string {
	//		return fmt.Sprintf("msg.Encode(p)")
	//	})
	//	return total, io.EOF
	//}
	// There's some data, let's process it first
	if len(p) > 0 {
		n, err := w.Write(p)
		total += int64(n)
		Log.Debugc(func() string {
			return fmt.Sprintf("Wrote %d bytes, totaling %d bytes", n, total)
		})

		if err != nil {
			Log.Errorc(func() string {
				return fmt.Sprintf("w.Write(p) error")
			})
			return total, err
		}
	}

	return total, nil
	//}
}

/**
2016.03.03 修改
*/
func (this *buffer) isDone() bool {
	if atomic.LoadInt64(&this.done) == 1 {
		return true
	}

	return false
}

func powerOfTwo64(n int64) bool {
	return n != 0 && (n&(n-1)) == 0
}

func roundUpPowerOfTwo64(n int64) int64 {
	n--
	n |= n >> 1
	n |= n >> 2
	n |= n >> 4
	n |= n >> 8
	n |= n >> 16
	n |= n >> 32
	n++

	return n
}
<commit_msg>添加测试代码<commit_after>// Copyright (c) 2014 The SurgeMQ Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"bufio"
	"fmt"
	"io"
	"sync/atomic"
	//"github.com/surgemq/message"
	"encoding/binary"
	"errors"
	"math"
	"sync"
	"time"
)

var (
	bufcnt            int64
	DefaultBufferSize int64

	DeviceInBufferSize  int64
	DeviceOutBufferSize int64

	MasterInBufferSize  int64
	MasterOutBufferSize int64
)

const (
	smallReadBlockSize    = 512
	defaultReadBlockSize  = 8192
	defaultWriteBlockSize = 8192
)

/**
2016.03.03 修改
bingbuffer结构体
*/
type buffer struct {
	readIndex  int64        //读序号
	writeIndex int64        //写序号
	ringBuffer []*ByteArray //环形buffer指针数组
	bufferSize int64        //初始化环形buffer指针数组大小
	mask       int64        //掩码：bufferSize-1
	done       int64        //是否完成
	rcond      *sync.Cond
	wcond      *sync.Cond
}

type ByteArray struct {
	bArray []byte
}

func (this *buffer) ReadCommit(index int64, ok bool) {
	this.rcond.L.Lock()
	defer this.rcond.L.Unlock()
	if ok {
		this.ringBuffer[index] = nil
	}
	this.wcond.Broadcast()
}

func (this *buffer) Len() int {
	cpos := this.GetCurrentReadIndex()
	ppos := this.GetCurrentWriteIndex()
	return int(ppos - cpos)
}

/**
2016.03.03 添加
初始化ringbuffer
参数bufferSize：初始化环形buffer指针数组大小
*/
func newBuffer(size int64) (*buffer, error) {
	if size < 0 {
		return nil, bufio.ErrNegativeCount
	}
	if size == 0 {
		size = DefaultBufferSize
	}
	if !powerOfTwo64(size) {
		fmt.Printf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
		return nil, fmt.Errorf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
	}

	return &buffer{
		readIndex:  int64(0),                 //读序号
		writeIndex: int64(0),                 //写序号
		ringBuffer: make([]*ByteArray, size), //环形buffer指针数组
		bufferSize: size,                     //初始化环形buffer指针数组大小
		mask:       size - 1,
		rcond:      sync.NewCond(new(sync.Mutex)),
		wcond:      sync.NewCond(new(sync.Mutex)),
	}, nil
}

/**
2016.03.03 添加
获取当前读序号
*/
func (this *buffer) GetCurrentReadIndex() int64 {
	return atomic.LoadInt64(&this.readIndex)
}

/**
2016.03.03 添加
获取当前写序号
*/
func (this *buffer) GetCurrentWriteIndex() int64 {
	return atomic.LoadInt64(&this.writeIndex)
}

/**
2016.03.03 添加
读取ringbuffer指定的buffer指针，返回该指针并清空ringbuffer该位置存在的指针内容，以及将读序号加1
*/
func (this *buffer) ReadBuffer() ([]byte, int64, bool) {
	this.rcond.L.Lock()
	defer this.rcond.L.Unlock()
	for {
		readIndex := atomic.LoadInt64(&this.readIndex)
		writeIndex := atomic.LoadInt64(&this.writeIndex)
		switch {
		case readIndex >= writeIndex:
			this.rcond.Wait()
		case writeIndex-readIndex > this.bufferSize:
			this.rcond.Wait()
		default:
			if readIndex == math.MaxInt64 {
				atomic.StoreInt64(&this.readIndex, int64(0))
			} else {
				atomic.AddInt64(&this.readIndex, int64(1))
			}

			index := readIndex & this.mask

			p_ := this.ringBuffer[index]
			//this.ringBuffer[index] = nil
			if p_ == nil {
				return nil, -1, false
			}
			p := p_.bArray

			if p == nil {
				return nil, -1, false
			}

			return p, index, true
		}
	}

}

/**
2016.03.03 添加
写入ringbuffer指针，以及将写序号加1
*/
func (this *buffer) WriteBuffer(in []byte) bool {
	this.wcond.L.Lock()
	defer func() {
		this.rcond.Broadcast()
		this.wcond.L.Unlock()
	}()

	for {
		readIndex := atomic.LoadInt64(&this.readIndex)
		writeIndex := atomic.LoadInt64(&this.writeIndex)
		switch {
		case writeIndex-readIndex < 0:
			this.wcond.Wait()
		default:
			index := writeIndex & this.mask
			if writeIndex == math.MaxInt64 {
				atomic.StoreInt64(&this.writeIndex, int64(0))
			} else {
				atomic.AddInt64(&this.writeIndex, int64(1))
			}
			if this.ringBuffer[index] == nil {
				this.ringBuffer[index] = &ByteArray{bArray: in}
				return true
			} else {
				return false
			}
		}
	}

}

/**
2016.03.03 修改
完成
*/
func (this *buffer) Close() error {
	atomic.StoreInt64(&this.done, 1)

	this.wcond.L.Lock()
	this.rcond.Broadcast()
	this.wcond.L.Unlock()

	this.rcond.L.Lock()
	this.wcond.Broadcast()
	this.rcond.L.Unlock()

	return nil
}

/*

/**
2016.03.03 修改
向ringbuffer中写数据（从connection的中向ringbuffer中写）--生产者
*/
func (this *buffer) ReadFrom(r io.Reader) (int64, error) {
	defer this.Close()
	total := int64(0)
	//for {

	//if this.isDone() {
	//	return total, io.EOF
	//}
	b := make([]byte, int64(5))
	n, err := r.Read(b[0:1])
	fmt.Println("readfrom 1st read conn:", err, "读取数量：", n)
	if n > 0 {
		total += int64(n)
		if err != nil {
			return total, err
		}
	}

	/**************************/
	cnt := 1

	// Let's read enough bytes to get the message header (msg type, remaining length)
	for {
		// If we have read 5 bytes and still not done, then there's a problem.
		if cnt > 4 {
			return 0, fmt.Errorf("sendrecv/peekMessageSize: 4th byte of remaining length has continuation bit set")
		}

		//Log.Infoc(func() string {
		//	return fmt.Sprintf("sendrecv/peekMessageSize: %d=========", cnt)
		//})
		// Peek cnt bytes from the input buffer.

		_, err := r.Read(b[cnt:(cnt + 1)])
		fmt.Println("readfrom 2th read conn:", err, "读取数量：", n)
		//fmt.Println(b)
		if err != nil {
			return 0, err
		}
		// If we got enough bytes, then check the last byte to see if the continuation
		// bit is set. If so, increment cnt and continue peeking
		if b[cnt] >= 0x80 {
			cnt++
		} else {
			break
		}
	}

	// Get the remaining length of the message
	remlen, m := binary.Uvarint(b[1:])
	//Log.Infoc(func() string {
	//	return fmt.Sprintf("b[cnt:(cnt + 1)]==end")
	//})
	// Total message length is remlen + 1 (msg type) + m (remlen bytes)

	total = int64(remlen) + int64(1) + int64(m)
	//Log.Infoc(func() string {
	//	return fmt.Sprintf("remlen===n===totle: %d===%d===%d", remlen, m, total)
	//})
	//mtype := message.MessageType(b[0] >> 4)
	/****************/
	//var msg message.Message
	//
	//msg, err = mtype.New()
	//if err != nil {
	//	return 0, err
	//}
	b__ := make([]byte, 0, total)
	b__ = append(b__, b[0:1+m]...)
	for i := int64(0); i < int64(remlen); i++ {
		b_ := make([]byte, int64(1))
		n, err = r.Read(b_[0:])

		if err != nil {
			Log.Errorc(func() string {
				return fmt.Sprintf("从conn读取数据失败(%s)", err)
			})
			time.Sleep(2 * time.Millisecond)
			//return total, err
		}
		b__ = append(b__, b_[0:]...)
	}

	fmt.Println("readfrom 3th read conn:", err, "读取数量：", n)
	fmt.Println(b__)
	//n, err = msg.Decode(b)
	//if err != nil {
	//	return 0, err
	//}

	/*************************/

	if !this.WriteBuffer(b__) {
		return total, err
	}

	return total, nil
	//}
}

/**
2016.03.03 修改
*/
func (this *buffer) WriteTo(w io.Writer) (int64, error) {
	defer this.Close()
	total := int64(0)
	//for {
	//if this.isDone() {
	//	return total, io.EOF
	//}
	p, index, ok := this.ReadBuffer()
	defer this.ReadCommit(index, ok)
	if !ok {
		return total, errors.New("读取过程中出现问题")
	}

	Log.Debugc(func() string {
		return fmt.Sprintf("WriteTo方法读取ringbuffer成功(%d)", index)
	})
	//Log.Debugc(func() string {
	//	return fmt.Sprintf("WriteTo函数》》读取*p：" + p)
	//})

	//
	//Log.Errorc(func() string {
	//	return fmt.Sprintf("msg::" + msg.Name())
	//})
	//
	//p := make([]byte, msg.Len())
	//_, err := msg.Encode(p)
	//if err != nil {
	//	Log.Errorc(func() string {
	//		return fmt.Sprintf("msg.Encode(p)")
	//	})
	//	return total, io.EOF
	//}
	// There's some data, let's process it first
	if len(p) > 0 {
		n, err := w.Write(p)
		total += int64(n)
		Log.Debugc(func() string {
			return fmt.Sprintf("Wrote %d bytes, totaling %d bytes", n, total)
		})

		if err != nil {
			Log.Errorc(func() string {
				return fmt.Sprintf("w.Write(p) error")
			})
			return total, err
		}
	}

	return total, nil
	//}
}

/**
2016.03.03 修改
*/
func (this *buffer) isDone() bool {
	if atomic.LoadInt64(&this.done) == 1 {
		return true
	}

	return false
}

func powerOfTwo64(n int64) bool {
	return n != 0 && (n&(n-1)) == 0
}

func roundUpPowerOfTwo64(n int64) int64 {
	n--
	n |= n >> 1
	n |= n >> 2
	n |= n >> 4
	n |= n >> 8
	n |= n >> 16
	n |= n >> 32
	n++

	return n
}
<|endoftext|>