<commit_before>package connector

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"sync"

	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/gorilla/mux"
	"github.com/smancke/guble/protocol"
	"github.com/smancke/guble/server/kvstore"
	"github.com/smancke/guble/server/router"
	"github.com/smancke/guble/server/service"
)

const (
	DefaultWorkers                  = 1
	SubstituteSubscriberInformation = "substitute"
)

var (
	TopicParam     = "topic"
	ConnectorParam = "connector"
)

type Sender interface {
	// Send takes a Request and returns the response or error
	Send(Request) (interface{}, error)
}

type SenderSetter interface {
	Sender() Sender
	SetSender(Sender)
}

type Metadata struct {
	Latency time.Duration
}

type ResponseHandler interface {
	// HandleResponse handles the response+error (returned by a Sender)
	HandleResponse(Request, interface{}, *Metadata, error) error
}

type ResponseHandlerSetter interface {
	ResponseHandler() ResponseHandler
	SetResponseHandler(ResponseHandler)
}

type Runner interface {
	Run(Subscriber)
}

type Connector interface {
	service.Startable
	service.Stopable
	service.Endpoint
	SenderSetter
	ResponseHandlerSetter
	Runner
	Manager() Manager
	Context() context.Context
}

type ResponsiveConnector interface {
	Connector
	ResponseHandler
}

type connector struct {
	config  Config
	sender  Sender
	handler ResponseHandler
	manager Manager
	queue   Queue
	router  router.Router
	kvstore kvstore.KVStore

	mux *mux.Router

	ctx    context.Context
	cancel context.CancelFunc

	logger *log.Entry
	wg     sync.WaitGroup
}

type Config struct {
	Name       string
	Schema     string
	Prefix     string
	URLPattern string
	Workers    int
}

func NewConnector(router router.Router, sender Sender, config Config) (Connector, error) {
	kvs, err := router.KVStore()
	if err != nil {
		return nil, err
	}

	if config.Workers <= 0 {
		config.Workers = DefaultWorkers
	}

	c := &connector{
		config:  config,
		sender:  sender,
		manager: NewManager(config.Schema, kvs),
		queue:   NewQueue(sender, config.Workers),
		router:  router,
		kvstore: kvs,
		logger:  logger.WithField("name", config.Name),
	}
	c.initMuxRouter()
	return c, nil
}

func (c *connector) initMuxRouter() {
	muxRouter := mux.NewRouter()

	baseRouter := muxRouter.PathPrefix(c.GetPrefix()).Subrouter()
	baseRouter.Methods(http.MethodGet).HandlerFunc(c.GetList)
	baseRouter.Methods(http.MethodPost).PathPrefix(SubstituteSubscriberInformation).HandlerFunc(c.Substitute)

	subRouter := baseRouter.Path(c.config.URLPattern).Subrouter()
	subRouter.Methods(http.MethodPost).HandlerFunc(c.Post)
	subRouter.Methods(http.MethodDelete).HandlerFunc(c.Delete)
	c.mux = muxRouter
}

func (c *connector) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	c.logger.WithFields(log.Fields{
		"path": req.URL.RequestURI(),
	}).Info("Handling HTTP request")
	c.mux.ServeHTTP(w, req)
}

func (c *connector) GetPrefix() string {
	return c.config.Prefix
}

// GetList returns list of subscribers
func (c *connector) GetList(w http.ResponseWriter, req *http.Request) {
	query := req.URL.Query()
	filters := make(map[string]string, len(query))

	for key, value := range query {
		if len(value) == 0 {
			continue
		}
		filters[key] = value[0]
	}

	log.WithField("filters", filters).Debug("Get list of subscriptions")
	if len(filters) == 0 {
		http.Error(w, `{"error":"Missing filters"}`, http.StatusBadRequest)
		return
	}

	subscribers := c.manager.Filter(filters)
	topics := make([]string, 0, len(subscribers))
	for _, s := range subscribers {
		topics = append(topics, s.Route().Path.RemovePrefixSlash())
	}

	encoder := json.NewEncoder(w)
	err := encoder.Encode(topics)
	if err != nil {
		http.Error(w, "Error encoding data.", http.StatusInternalServerError)
		c.logger.WithField("error", err.Error()).Error("Error encoding data.")
		return
	}
}

// Post creates a new subscriber
func (c *connector) Post(w http.ResponseWriter, req *http.Request) {
	params := mux.Vars(req)
	log.WithField("params", params).Debug("POST subscription")
	topic, ok := params[TopicParam]
	if !ok {
		fmt.Fprintf(w, "Missing topic parameter.")
		return
	}
	delete(params, TopicParam)
	params[ConnectorParam] = c.config.Name
	log.WithField("params", params).WithField("topic", topic).Debug("Creating subscription")
	subscriber, err := c.manager.Create(protocol.Path("/"+topic), params)
	if err != nil {
		if err == ErrSubscriberExists {
			fmt.Fprintf(w, `{"error":"subscription already exists"}`)
		} else {
			http.Error(w, fmt.Sprintf(`{"error":"unknown error: %s"}`, err.Error()), http.StatusInternalServerError)
		}
		return
	}
	go c.Run(subscriber)
	log.WithField("topic", topic).Debug("Subscription created")
	fmt.Fprintf(w, `{"subscribed":"/%v"}`, topic)
}

// Delete removes a subscriber
func (c *connector) Delete(w http.ResponseWriter, req *http.Request) {
	params := mux.Vars(req)
	log.WithField("params", params).Debug("DELETE subscription")
	topic, ok := params[TopicParam]
	if !ok {
		fmt.Fprintf(w, "Missing topic parameter.")
		return
	}
	delete(params, TopicParam)
	params[ConnectorParam] = c.config.Name
	log.WithField("params", params).WithField("topic", topic).Debug("Finding subscription to delete it")
	subscriber := c.manager.Find(GenerateKey("/"+topic, params))
	if subscriber == nil {
		http.Error(w, `{"error":"subscription not found"}`, http.StatusNotFound)
		return
	}
	log.WithField("params", params).WithField("topic", topic).Debug("Deleting subscription")
	err := c.manager.Remove(subscriber)
	if err != nil {
		http.Error(w, fmt.Sprintf(`{"error":"unknown error: %s"}`, err.Error()), http.StatusInternalServerError)
		return
	}
	fmt.Fprintf(w, `{"unsubscribed":"/%v"}`, topic)
}

func (c *connector) Substitute(w http.ResponseWriter, req *http.Request) {
	params := mux.Vars(req)
	log.WithField("params", params).Debug("POST subscription")
	//TODO here do the magic

}

// Start will run start all current subscriptions and workers to process the messages
func (c *connector) Start() error {
	c.queue.Start()

	c.logger.Debug("Starting connector")
	c.ctx, c.cancel = context.WithCancel(context.Background())

	c.logger.Debug("Loading subscriptions")
	err := c.manager.Load()
	if err != nil {
		return err
	}

	c.logger.Debug("Starting subscriptions")
	for _, s := range c.manager.List() {
		go c.Run(s)
	}

	c.logger.Debug("Started connector")
	return nil
}

func (c *connector) Run(s Subscriber) {
	c.wg.Add(1)
	defer c.wg.Done()

	var provideErr error
	go func() {
		err := s.Route().Provide(c.router, true)
		if err != nil {
			// cancel subscription loop if there is an error on the provider
			provideErr = err
			s.Cancel()
		}
	}()

	err := s.Loop(c.ctx, c.queue)
	if err != nil && provideErr == nil {
		c.logger.WithField("error", err.Error()).Error("Error returned by subscriber loop")
		// if context cancelled loop then unsubscribe the route from router
		// in case it's been subscribed
		if err == context.Canceled {
			c.router.Unsubscribe(s.Route())
			return
		}

		// If Route channel closed try restarting
		if err == ErrRouteChannelClosed {
			c.restart(s)
			return
		}
	}

	if provideErr != nil {
		// TODO Bogdan Treat errors where a subscription provide fails
		c.logger.WithField("error", provideErr.Error()).Error("Route provide error")

		// Router closed the route, try restart
		if provideErr == router.ErrInvalidRoute {
			c.restart(s)
			return
		}
		// Router module is stopping, exit the process
		if _, ok := provideErr.(*router.ModuleStoppingError); ok {
			return
		}
	}
}

func (c *connector) restart(s Subscriber) error {
	s.Cancel()
	err := s.Reset()
	if err != nil {
		c.logger.WithField("err", err.Error()).Error("Error reseting subscriber")
		return err
	}
	go c.Run(s)
	return nil
}

// Stop the connector (the context, the queue, the subscription loops)
func (c *connector) Stop() error {
	c.logger.Debug("Stopping connector")
	c.cancel()
	c.queue.Stop()
	c.wg.Wait()
	c.logger.Debug("Stopped connector")
	return nil
}

func (c *connector) Manager() Manager {
	return c.manager
}

func (c *connector) Context() context.Context {
	return c.ctx
}

func (c *connector) ResponseHandler() ResponseHandler {
	return c.handler
}

func (c *connector) SetResponseHandler(handler ResponseHandler) {
	c.handler = handler
	c.queue.SetResponseHandler(handler)
}

func (c *connector) Sender() Sender {
	return c.sender
}

func (c *connector) SetSender(s Sender) {
	c.sender = s
	c.queue.SetSender(s)
}
<commit_msg>Small refactoring on http methods. Added  the structure for this.<commit_after>package connector

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"sync"

	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/gorilla/mux"
	"github.com/smancke/guble/protocol"
	"github.com/smancke/guble/server/kvstore"
	"github.com/smancke/guble/server/router"
	"github.com/smancke/guble/server/service"
)

const (
	DefaultWorkers = 1
	SubstituteSubscriberInformation = "/substitute/"
)

var (
	TopicParam = "topic"
	ConnectorParam = "connector"
)

type Sender interface {
	// Send takes a Request and returns the response or error
	Send(Request) (interface{}, error)
}

type SenderSetter interface {
	Sender() Sender
	SetSender(Sender)
}

type Metadata struct {
	Latency time.Duration
}

type ResponseHandler interface {
	// HandleResponse handles the response+error (returned by a Sender)
	HandleResponse(Request, interface{}, *Metadata, error) error
}

type ResponseHandlerSetter interface {
	ResponseHandler() ResponseHandler
	SetResponseHandler(ResponseHandler)
}

type Runner interface {
	Run(Subscriber)
}

type Connector interface {
	service.Startable
	service.Stopable
	service.Endpoint
	SenderSetter
	ResponseHandlerSetter
	Runner
	Manager() Manager
	Context() context.Context
}

type ResponsiveConnector interface {
	Connector
	ResponseHandler
}

type connector struct {
	config  Config
	sender  Sender
	handler ResponseHandler
	manager Manager
	queue   Queue
	router  router.Router
	kvstore kvstore.KVStore

	mux     *mux.Router

	ctx     context.Context
	cancel  context.CancelFunc

	logger  *log.Entry
	wg      sync.WaitGroup
}

type Config struct {
	Name       string
	Schema     string
	Prefix     string
	URLPattern string
	Workers    int
}

func NewConnector(router router.Router, sender Sender, config Config) (Connector, error) {
	kvs, err := router.KVStore()
	if err != nil {
		return nil, err
	}

	if config.Workers <= 0 {
		config.Workers = DefaultWorkers
	}

	c := &connector{
		config:  config,
		sender:  sender,
		manager: NewManager(config.Schema, kvs),
		queue:   NewQueue(sender, config.Workers),
		router:  router,
		kvstore: kvs,
		logger:  logger.WithField("name", config.Name),
	}
	c.initMuxRouter()
	return c, nil
}

func (c *connector) initMuxRouter() {
	muxRouter := mux.NewRouter()

	logger.WithField("prefix", c.GetPrefix()).Info("asfafas")
	baseRouter := muxRouter.PathPrefix(c.GetPrefix()).Subrouter()
	baseRouter.Methods(http.MethodGet).HandlerFunc(c.GetList)
	baseRouter.Methods(http.MethodPost).PathPrefix(SubstituteSubscriberInformation).HandlerFunc(c.Substitute)

	subRouter := baseRouter.Path(c.config.URLPattern).Subrouter()
	subRouter.Methods(http.MethodPost).HandlerFunc(c.Post)
	subRouter.Methods(http.MethodDelete).HandlerFunc(c.Delete)
	c.mux = muxRouter
}

func (c *connector) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	c.logger.WithFields(log.Fields{
		"path": req.URL.RequestURI(),
	}).Info("Handling HTTP request")
	c.mux.ServeHTTP(w, req)
}

func (c *connector) GetPrefix() string {
	return c.config.Prefix
}

// GetList returns list of subscribers
func (c *connector) GetList(w http.ResponseWriter, req *http.Request) {
	query := req.URL.Query()
	filters := make(map[string]string, len(query))

	for key, value := range query {
		if len(value) == 0 {
			continue
		}
		filters[key] = value[0]
	}

	log.WithField("filters", filters).Debug("Get list of subscriptions")
	if len(filters) == 0 {
		http.Error(w, `{"error":"Missing filters"}`, http.StatusBadRequest)
		return
	}

	subscribers := c.manager.Filter(filters)
	topics := make([]string, 0, len(subscribers))
	for _, s := range subscribers {
		topics = append(topics, s.Route().Path.RemovePrefixSlash())
	}

	encoder := json.NewEncoder(w)
	err := encoder.Encode(topics)
	if err != nil {
		http.Error(w, "Error encoding data.", http.StatusInternalServerError)
		c.logger.WithField("error", err.Error()).Error("Error encoding data.")
		return
	}
}

// Post creates a new subscriber
func (c *connector) Post(w http.ResponseWriter, req *http.Request) {
	params := mux.Vars(req)
	log.WithField("params", params).Debug("POST subscription")
	topic, ok := params[TopicParam]
	if !ok {
		fmt.Fprintf(w, "Missing topic parameter.")
		return
	}
	delete(params, TopicParam)
	params[ConnectorParam] = c.config.Name
	log.WithField("params", params).WithField("topic", topic).Debug("Creating subscription")
	subscriber, err := c.manager.Create(protocol.Path("/" + topic), params)
	if err != nil {
		if err == ErrSubscriberExists {
			fmt.Fprintf(w, `{"error":"subscription already exists"}`)
		} else {
			http.Error(w, fmt.Sprintf(`{"error":"unknown error: %s"}`, err.Error()), http.StatusInternalServerError)
		}
		return
	}
	go c.Run(subscriber)
	log.WithField("topic", topic).Debug("Subscription created")
	fmt.Fprintf(w, `{"subscribed":"/%v"}`, topic)
}

// Delete removes a subscriber
func (c *connector) Delete(w http.ResponseWriter, req *http.Request) {
	params := mux.Vars(req)
	log.WithField("params", params).Debug("DELETE subscription")
	topic, ok := params[TopicParam]
	if !ok {
		fmt.Fprintf(w, "Missing topic parameter.")
		return
	}
	delete(params, TopicParam)
	params[ConnectorParam] = c.config.Name
	log.WithField("params", params).WithField("topic", topic).Debug("Finding subscription to delete it")
	subscriber := c.manager.Find(GenerateKey("/" + topic, params))
	if subscriber == nil {
		http.Error(w, `{"error":"subscription not found"}`, http.StatusNotFound)
		return
	}
	log.WithField("params", params).WithField("topic", topic).Debug("Deleting subscription")
	err := c.manager.Remove(subscriber)
	if err != nil {
		http.Error(w, fmt.Sprintf(`{"error":"unknown error: %s"}`, err.Error()), http.StatusInternalServerError)
		return
	}
	fmt.Fprintf(w, `{"unsubscribed":"/%v"}`, topic)
}

type Substitution struct {
	FieldName string `json:"field"`
	OldValue  string `json:"old_value"`
	NewValue  string `json:"new_value"`
}

func (c *connector) Substitute(w http.ResponseWriter, req *http.Request) {

	//TODO here do the magic

	substitutionReq := new(Substitution)
	decoder := json.NewDecoder(req.Body)
	err := decoder.Decode(&substitutionReq)
	if err != nil {
		http.Error(w, fmt.Sprintf(`{"error":"json body could not be decode: %s"}`, err.Error()), http.StatusBadRequest)
		return
	}

	filters := map[string]string{}
	filters[substitutionReq.FieldName] = substitutionReq.OldValue
	subscribers  := c.manager.Filter(filters)
	totalSubscribersUpdated := 0;
	for _,sub :=range subscribers {
		sub.Route().Set(substitutionReq.FieldName,substitutionReq.NewValue)
		err =c.manager.Update(sub)
		if err != nil {
			http.Error(w, fmt.Sprintf(`{"error":"%s"}`, err.Error()), http.StatusInternalServerError)
			return
		}
		totalSubscribersUpdated++;
	}

	log.WithField("subscribers", subscribers).WithField("req", substitutionReq).Info("POST SUBSTITUTE  subscriber info ")
	fmt.Fprintf(w, `{"modified":"%d"}`, totalSubscribersUpdated)
}

// Start will run start all current subscriptions and workers to process the messages
func (c *connector) Start() error {
	c.queue.Start()

	c.logger.Debug("Starting connector")
	c.ctx, c.cancel = context.WithCancel(context.Background())

	c.logger.Debug("Loading subscriptions")
	err := c.manager.Load()
	if err != nil {
		return err
	}

	c.logger.Debug("Starting subscriptions")
	for _, s := range c.manager.List() {
		go c.Run(s)
	}

	c.logger.Debug("Started connector")
	return nil
}

func (c *connector) Run(s Subscriber) {
	c.wg.Add(1)
	defer c.wg.Done()

	var provideErr error
	go func() {
		err := s.Route().Provide(c.router, true)
		if err != nil {
			// cancel subscription loop if there is an error on the provider
			provideErr = err
			s.Cancel()
		}
	}()

	err := s.Loop(c.ctx, c.queue)
	if err != nil && provideErr == nil {
		c.logger.WithField("error", err.Error()).Error("Error returned by subscriber loop")
		// if context cancelled loop then unsubscribe the route from router
		// in case it's been subscribed
		if err == context.Canceled {
			c.router.Unsubscribe(s.Route())
			return
		}

		// If Route channel closed try restarting
		if err == ErrRouteChannelClosed {
			c.restart(s)
			return
		}
	}

	if provideErr != nil {
		// TODO Bogdan Treat errors where a subscription provide fails
		c.logger.WithField("error", provideErr.Error()).Error("Route provide error")

		// Router closed the route, try restart
		if provideErr == router.ErrInvalidRoute {
			c.restart(s)
			return
		}
		// Router module is stopping, exit the process
		if _, ok := provideErr.(*router.ModuleStoppingError); ok {
			return
		}
	}
}

func (c *connector) restart(s Subscriber) error {
	s.Cancel()
	err := s.Reset()
	if err != nil {
		c.logger.WithField("err", err.Error()).Error("Error reseting subscriber")
		return err
	}
	go c.Run(s)
	return nil
}

// Stop the connector (the context, the queue, the subscription loops)
func (c *connector) Stop() error {
	c.logger.Debug("Stopping connector")
	c.cancel()
	c.queue.Stop()
	c.wg.Wait()
	c.logger.Debug("Stopped connector")
	return nil
}

func (c *connector) Manager() Manager {
	return c.manager
}

func (c *connector) Context() context.Context {
	return c.ctx
}

func (c *connector) ResponseHandler() ResponseHandler {
	return c.handler
}

func (c *connector) SetResponseHandler(handler ResponseHandler) {
	c.handler = handler
	c.queue.SetResponseHandler(handler)
}

func (c *connector) Sender() Sender {
	return c.sender
}

func (c *connector) SetSender(s Sender) {
	c.sender = s
	c.queue.SetSender(s)
}
<|endoftext|>


<commit_before>package connector

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"sync"

	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/gorilla/mux"
	"github.com/smancke/guble/protocol"
	"github.com/smancke/guble/server/kvstore"
	"github.com/smancke/guble/server/router"
	"github.com/smancke/guble/server/service"
)

const DefaultWorkers = 1

var (
	TopicParam = "topic"
)

type Sender interface {
	// Send takes a Request and returns the response or error
	Send(Request) (interface{}, error)
}

type SenderSetter interface {
	Sender() Sender
	SetSender(Sender)
}

type Metadata struct {
	Latency time.Duration
}

type ResponseHandler interface {
	// HandleResponse handles the response+error (returned by a Sender)
	HandleResponse(Request, interface{}, *Metadata, error) error
}

type ResponseHandlerSetter interface {
	ResponseHandler() ResponseHandler
	SetResponseHandler(ResponseHandler)
}

type Runner interface {
	Run(Subscriber)
}

type Connector interface {
	service.Startable
	service.Stopable
	service.Endpoint
	SenderSetter
	ResponseHandlerSetter
	Runner
	Manager() Manager
	Context() context.Context
}

type ResponsiveConnector interface {
	Connector
	ResponseHandler
}

type connector struct {
	config  Config
	sender  Sender
	handler ResponseHandler
	manager Manager
	queue   Queue
	router  router.Router
	kvstore kvstore.KVStore

	mux *mux.Router

	ctx    context.Context
	cancel context.CancelFunc

	logger *log.Entry
	wg     sync.WaitGroup
}

type Config struct {
	Name       string
	Schema     string
	Prefix     string
	URLPattern string
	Workers    int
}

func NewConnector(router router.Router, sender Sender, config Config) (Connector, error) {
	kvs, err := router.KVStore()
	if err != nil {
		return nil, err
	}

	if config.Workers <= 0 {
		config.Workers = DefaultWorkers
	}

	c := &connector{
		config:  config,
		sender:  sender,
		manager: NewManager(config.Schema, kvs),
		queue:   NewQueue(sender, config.Workers),
		router:  router,
		kvstore: kvs,
		logger:  logger.WithField("name", config.Name),
	}
	c.initMuxRouter()
	return c, nil
}

func (c *connector) initMuxRouter() {
	muxRouter := mux.NewRouter()

	baseRouter := muxRouter.PathPrefix(c.GetPrefix()).Subrouter()
	baseRouter.Methods("GET").HandlerFunc(c.GetList)

	subRouter := baseRouter.Path(c.config.URLPattern).Subrouter()
	subRouter.Methods("POST").HandlerFunc(c.Post)
	subRouter.Methods("DELETE").HandlerFunc(c.Delete)
	c.mux = muxRouter
}

func (c *connector) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	c.logger.WithFields(log.Fields{
		"path": req.URL.RequestURI(),
	}).Info("Handling request")
	c.mux.ServeHTTP(w, req)
}

func (c *connector) GetPrefix() string {
	return c.config.Prefix
}

// GetList returns list of subscribers
func (c *connector) GetList(w http.ResponseWriter, req *http.Request) {
	query := req.URL.Query()
	filters := make(map[string]string, len(query))

	for key, value := range query {
		if len(value) == 0 {
			continue
		}
		filters[key] = value[0]
	}

	log.WithField("filters", filters).Debug("Get list of subscriptions")
	if len(filters) == 0 {
		http.Error(w, `{"error":"Missing filters"}`, http.StatusBadRequest)
		return
	}

	subscribers := c.manager.Filter(filters)
	topics := make([]string, 0, len(subscribers))
	for _, s := range subscribers {
		topics = append(topics, s.Route().Path.RemovePrefixSlash())
	}

	encoder := json.NewEncoder(w)
	err := encoder.Encode(topics)
	if err != nil {
		http.Error(w, "Error encoding data.", http.StatusInternalServerError)
		c.logger.WithField("error", err.Error()).Error("Error encoding data.")
		return
	}
}

// Post creates a new subscriber
func (c *connector) Post(w http.ResponseWriter, req *http.Request) {
	params := mux.Vars(req)
	log.WithField("params", params).Debug("Create subscription")

	topic, ok := params[TopicParam]
	if !ok {
		fmt.Fprintf(w, "Missing topic parameter.")
		return
	}
	delete(params, TopicParam)

	subscriber, err := c.manager.Create(protocol.Path("/"+topic), params)
	if err != nil {
		if err == ErrSubscriberExists {
			fmt.Fprintf(w, `{"error":"subscription already exists"}`)
		} else {
			http.Error(w, fmt.Sprintf(`{"error":"unknown error: %s"}`, err.Error()), http.StatusInternalServerError)
		}
		return
	}

	go c.Run(subscriber)
	log.WithField("topic", topic).Debug("Subscription created")
	fmt.Fprintf(w, `{"subscribed":"/%v"}`, topic)
}

// Delete removes a subscriber
func (c *connector) Delete(w http.ResponseWriter, req *http.Request) {
	params := mux.Vars(req)
	log.WithField("params", params).Debug("Deleting subscription")
	topic, ok := params[TopicParam]
	if !ok {
		fmt.Fprintf(w, "Missing topic parameter.")
		return
	}

	delete(params, TopicParam)
	subscriber := c.manager.Find(GenerateKey("/"+topic, params))
	if subscriber == nil {
		http.Error(w, `{"error":"subscription not found"}`, http.StatusNotFound)
		return
	}

	err := c.manager.Remove(subscriber)
	if err != nil {
		http.Error(w, fmt.Sprintf(`{"error":"unknown error: %s"}`, err.Error()), http.StatusInternalServerError)
		return
	}

	fmt.Fprintf(w, `{"unsubscribed":"/%v"}`, topic)
}

// Start will run start all current subscriptions and workers to process the messages
func (c *connector) Start() error {
	c.queue.Start()

	c.logger.Debug("Starting connector")
	c.ctx, c.cancel = context.WithCancel(context.Background())

	c.logger.Debug("Loading subscriptions")
	err := c.manager.Load()
	if err != nil {
		return err
	}

	c.logger.Debug("Starting subscriptions")
	for _, s := range c.manager.List() {
		go c.Run(s)
	}

	c.logger.Debug("Started connector")
	return nil
}

func (c *connector) Run(s Subscriber) {
	c.wg.Add(1)
	defer c.wg.Done()

	var provideErr error
	go func() {
		err := s.Route().Provide(c.router, true)
		if err != nil {
			// cancel subscription loop if there is an error on the provider
			provideErr = err
			s.Cancel()
		}
	}()

	err := s.Loop(c.ctx, c.queue)
	if err != nil && provideErr == nil {
		c.logger.WithField("error", err.Error()).Error("Error returned by subscriber loop")
		// if context cancelled loop then unsubscribe the route from router
		// in case it's been subscribed
		if err == context.Canceled {
			c.router.Unsubscribe(s.Route())
			return
		}

		// If Route channel closed try restarting
		if err == ErrRouteChannelClosed {
			c.restart(s)
			return
		}
	}

	if provideErr != nil {
		// TODO Bogdan Treat errors where a subscription provide fails
		c.logger.WithField("error", provideErr.Error()).Error("Route provide error")

		// Router closed the route, try restart
		if provideErr == router.ErrInvalidRoute {
			c.restart(s)
			return
		}
		// Router module is stopping, exit the process
		if _, ok := provideErr.(*router.ModuleStoppingError); ok {
			return
		}
	}
}

func (c *connector) restart(s Subscriber) error {
	s.Cancel()
	err := s.Reset()
	if err != nil {
		c.logger.WithField("err", err.Error()).Error("Error reseting subscriber")
		return err
	}
	go c.Run(s)
	return nil
}

// Stop the connector (the context, the queue, the subscription loops)
func (c *connector) Stop() error {
	c.logger.Debug("Stopping connector")
	c.cancel()
	c.queue.Stop()
	c.wg.Wait()
	c.logger.Debug("Stopped connector")
	return nil
}

func (c *connector) Manager() Manager {
	return c.manager
}

func (c *connector) Context() context.Context {
	return c.ctx
}

func (c *connector) ResponseHandler() ResponseHandler {
	return c.handler
}

func (c *connector) SetResponseHandler(handler ResponseHandler) {
	c.handler = handler
	c.queue.SetResponseHandler(handler)
}

func (c *connector) Sender() Sender {
	return c.sender
}

func (c *connector) SetSender(s Sender) {
	c.sender = s
	c.queue.SetSender(s)
}
<commit_msg>Change log message<commit_after>package connector

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"sync"

	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/gorilla/mux"
	"github.com/smancke/guble/protocol"
	"github.com/smancke/guble/server/kvstore"
	"github.com/smancke/guble/server/router"
	"github.com/smancke/guble/server/service"
)

const DefaultWorkers = 1

var (
	TopicParam = "topic"
)

type Sender interface {
	// Send takes a Request and returns the response or error
	Send(Request) (interface{}, error)
}

type SenderSetter interface {
	Sender() Sender
	SetSender(Sender)
}

type Metadata struct {
	Latency time.Duration
}

type ResponseHandler interface {
	// HandleResponse handles the response+error (returned by a Sender)
	HandleResponse(Request, interface{}, *Metadata, error) error
}

type ResponseHandlerSetter interface {
	ResponseHandler() ResponseHandler
	SetResponseHandler(ResponseHandler)
}

type Runner interface {
	Run(Subscriber)
}

type Connector interface {
	service.Startable
	service.Stopable
	service.Endpoint
	SenderSetter
	ResponseHandlerSetter
	Runner
	Manager() Manager
	Context() context.Context
}

type ResponsiveConnector interface {
	Connector
	ResponseHandler
}

type connector struct {
	config  Config
	sender  Sender
	handler ResponseHandler
	manager Manager
	queue   Queue
	router  router.Router
	kvstore kvstore.KVStore

	mux *mux.Router

	ctx    context.Context
	cancel context.CancelFunc

	logger *log.Entry
	wg     sync.WaitGroup
}

type Config struct {
	Name       string
	Schema     string
	Prefix     string
	URLPattern string
	Workers    int
}

func NewConnector(router router.Router, sender Sender, config Config) (Connector, error) {
	kvs, err := router.KVStore()
	if err != nil {
		return nil, err
	}

	if config.Workers <= 0 {
		config.Workers = DefaultWorkers
	}

	c := &connector{
		config:  config,
		sender:  sender,
		manager: NewManager(config.Schema, kvs),
		queue:   NewQueue(sender, config.Workers),
		router:  router,
		kvstore: kvs,
		logger:  logger.WithField("name", config.Name),
	}
	c.initMuxRouter()
	return c, nil
}

func (c *connector) initMuxRouter() {
	muxRouter := mux.NewRouter()

	baseRouter := muxRouter.PathPrefix(c.GetPrefix()).Subrouter()
	baseRouter.Methods("GET").HandlerFunc(c.GetList)

	subRouter := baseRouter.Path(c.config.URLPattern).Subrouter()
	subRouter.Methods("POST").HandlerFunc(c.Post)
	subRouter.Methods("DELETE").HandlerFunc(c.Delete)
	c.mux = muxRouter
}

func (c *connector) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	c.logger.WithFields(log.Fields{
		"path": req.URL.RequestURI(),
	}).Info("Handling HTTP request")
	c.mux.ServeHTTP(w, req)
}

func (c *connector) GetPrefix() string {
	return c.config.Prefix
}

// GetList returns list of subscribers
func (c *connector) GetList(w http.ResponseWriter, req *http.Request) {
	query := req.URL.Query()
	filters := make(map[string]string, len(query))

	for key, value := range query {
		if len(value) == 0 {
			continue
		}
		filters[key] = value[0]
	}

	log.WithField("filters", filters).Debug("Get list of subscriptions")
	if len(filters) == 0 {
		http.Error(w, `{"error":"Missing filters"}`, http.StatusBadRequest)
		return
	}

	subscribers := c.manager.Filter(filters)
	topics := make([]string, 0, len(subscribers))
	for _, s := range subscribers {
		topics = append(topics, s.Route().Path.RemovePrefixSlash())
	}

	encoder := json.NewEncoder(w)
	err := encoder.Encode(topics)
	if err != nil {
		http.Error(w, "Error encoding data.", http.StatusInternalServerError)
		c.logger.WithField("error", err.Error()).Error("Error encoding data.")
		return
	}
}

// Post creates a new subscriber
func (c *connector) Post(w http.ResponseWriter, req *http.Request) {
	params := mux.Vars(req)
	log.WithField("params", params).Debug("Create subscription")

	topic, ok := params[TopicParam]
	if !ok {
		fmt.Fprintf(w, "Missing topic parameter.")
		return
	}
	delete(params, TopicParam)

	subscriber, err := c.manager.Create(protocol.Path("/"+topic), params)
	if err != nil {
		if err == ErrSubscriberExists {
			fmt.Fprintf(w, `{"error":"subscription already exists"}`)
		} else {
			http.Error(w, fmt.Sprintf(`{"error":"unknown error: %s"}`, err.Error()), http.StatusInternalServerError)
		}
		return
	}

	go c.Run(subscriber)
	log.WithField("topic", topic).Debug("Subscription created")
	fmt.Fprintf(w, `{"subscribed":"/%v"}`, topic)
}

// Delete removes a subscriber
func (c *connector) Delete(w http.ResponseWriter, req *http.Request) {
	params := mux.Vars(req)
	log.WithField("params", params).Debug("Deleting subscription")
	topic, ok := params[TopicParam]
	if !ok {
		fmt.Fprintf(w, "Missing topic parameter.")
		return
	}

	delete(params, TopicParam)
	subscriber := c.manager.Find(GenerateKey("/"+topic, params))
	if subscriber == nil {
		http.Error(w, `{"error":"subscription not found"}`, http.StatusNotFound)
		return
	}

	err := c.manager.Remove(subscriber)
	if err != nil {
		http.Error(w, fmt.Sprintf(`{"error":"unknown error: %s"}`, err.Error()), http.StatusInternalServerError)
		return
	}

	fmt.Fprintf(w, `{"unsubscribed":"/%v"}`, topic)
}

// Start will run start all current subscriptions and workers to process the messages
func (c *connector) Start() error {
	c.queue.Start()

	c.logger.Debug("Starting connector")
	c.ctx, c.cancel = context.WithCancel(context.Background())

	c.logger.Debug("Loading subscriptions")
	err := c.manager.Load()
	if err != nil {
		return err
	}

	c.logger.Debug("Starting subscriptions")
	for _, s := range c.manager.List() {
		go c.Run(s)
	}

	c.logger.Debug("Started connector")
	return nil
}

func (c *connector) Run(s Subscriber) {
	c.wg.Add(1)
	defer c.wg.Done()

	var provideErr error
	go func() {
		err := s.Route().Provide(c.router, true)
		if err != nil {
			// cancel subscription loop if there is an error on the provider
			provideErr = err
			s.Cancel()
		}
	}()

	err := s.Loop(c.ctx, c.queue)
	if err != nil && provideErr == nil {
		c.logger.WithField("error", err.Error()).Error("Error returned by subscriber loop")
		// if context cancelled loop then unsubscribe the route from router
		// in case it's been subscribed
		if err == context.Canceled {
			c.router.Unsubscribe(s.Route())
			return
		}

		// If Route channel closed try restarting
		if err == ErrRouteChannelClosed {
			c.restart(s)
			return
		}
	}

	if provideErr != nil {
		// TODO Bogdan Treat errors where a subscription provide fails
		c.logger.WithField("error", provideErr.Error()).Error("Route provide error")

		// Router closed the route, try restart
		if provideErr == router.ErrInvalidRoute {
			c.restart(s)
			return
		}
		// Router module is stopping, exit the process
		if _, ok := provideErr.(*router.ModuleStoppingError); ok {
			return
		}
	}
}

func (c *connector) restart(s Subscriber) error {
	s.Cancel()
	err := s.Reset()
	if err != nil {
		c.logger.WithField("err", err.Error()).Error("Error reseting subscriber")
		return err
	}
	go c.Run(s)
	return nil
}

// Stop the connector (the context, the queue, the subscription loops)
func (c *connector) Stop() error {
	c.logger.Debug("Stopping connector")
	c.cancel()
	c.queue.Stop()
	c.wg.Wait()
	c.logger.Debug("Stopped connector")
	return nil
}

func (c *connector) Manager() Manager {
	return c.manager
}

func (c *connector) Context() context.Context {
	return c.ctx
}

func (c *connector) ResponseHandler() ResponseHandler {
	return c.handler
}

func (c *connector) SetResponseHandler(handler ResponseHandler) {
	c.handler = handler
	c.queue.SetResponseHandler(handler)
}

func (c *connector) Sender() Sender {
	return c.sender
}

func (c *connector) SetSender(s Sender) {
	c.sender = s
	c.queue.SetSender(s)
}
<|endoftext|>


<commit_before>package bitmap

import (
	"github.com/RoaringBitmap/roaring"

	"github.com/anacrolix/missinggo"
)

// Bitmaps store the existence of values in [0,math.MaxUint32] more
// efficiently than []bool. The empty value starts with no bits set.
type Bitmap struct {
	rb *roaring.Bitmap
}

func (me *Bitmap) Len() int {
	if me.rb == nil {
		return 0
	}
	return int(me.rb.GetCardinality())
}

func (me Bitmap) ToSortedSlice() (ret []int) {
	if me.rb == nil {
		return
	}
	missinggo.CastSlice(&ret, me.rb.ToArray())
	return
}

func (me *Bitmap) lazyRB() *roaring.Bitmap {
	if me.rb == nil {
		me.rb = roaring.NewBitmap()
	}
	return me.rb
}

func (me *Bitmap) Iter(f func(interface{}) bool) {
	me.IterTyped(func(i int) bool {
		return f(i)
	})
}

func (me Bitmap) IterTyped(f func(int) bool) bool {
	if me.rb == nil {
		return true
	}
	it := me.rb.Iterator()
	for it.HasNext() {
		if !f(int(it.Next())) {
			return false
		}
	}
	return true
}

func (me *Bitmap) Add(is ...int) {
	rb := me.lazyRB()
	for _, i := range is {
		rb.AddInt(i)
	}
}

func (me *Bitmap) AddRange(begin, end int) {
	if begin >= end {
		return
	}
	me.lazyRB().AddRange(uint32(begin), uint32(end))
}

func (me *Bitmap) Remove(i int) {
	if me.rb == nil {
		return
	}
	me.rb.Remove(uint32(i))
}

func (me *Bitmap) Union(other *Bitmap) {
	me.lazyRB().Or(other.lazyRB())
}

func (me *Bitmap) Contains(i int) bool {
	if me.rb == nil {
		return false
	}
	return me.rb.Contains(uint32(i))
}

type Iter struct {
	ii roaring.IntIterable
}

func (me *Iter) Next() bool {
	if me == nil {
		return false
	}
	return me.ii.HasNext()
}

func (me *Iter) Value() interface{} {
	return me.ValueInt()
}

func (me *Iter) ValueInt() int {
	return int(me.ii.Next())
}

func (me *Iter) Stop() {}

func Sub(left, right *Bitmap) *Bitmap {
	return &Bitmap{
		rb: roaring.AndNot(left.lazyRB(), right.lazyRB()),
	}
}

func (me *Bitmap) Sub(other *Bitmap) {
	if other.rb == nil {
		return
	}
	if me.rb == nil {
		return
	}
	me.rb.AndNot(other.rb)
}

func (me *Bitmap) Clear() {
	if me.rb == nil {
		return
	}
	me.rb.Clear()
}

func (me Bitmap) Copy() (ret Bitmap) {
	ret = me
	if ret.rb != nil {
		ret.rb = ret.rb.Clone()
	}
	return
}

func (me *Bitmap) FlipRange(begin, end int) {
	me.lazyRB().FlipInt(begin, end)
}

func (me *Bitmap) Get(bit int) bool {
	return me.rb != nil && me.rb.ContainsInt(bit)
}

func (me *Bitmap) Set(bit int, value bool) {
	if value {
		me.lazyRB().AddInt(bit)
	} else {
		if me.rb != nil {
			me.rb.Remove(uint32(bit))
		}
	}
}
<commit_msg>bitmap: Add Bitmap.RemoveRange<commit_after>package bitmap

import (
	"github.com/RoaringBitmap/roaring"

	"github.com/anacrolix/missinggo"
)

// Bitmaps store the existence of values in [0,math.MaxUint32] more
// efficiently than []bool. The empty value starts with no bits set.
type Bitmap struct {
	rb *roaring.Bitmap
}

func (me *Bitmap) Len() int {
	if me.rb == nil {
		return 0
	}
	return int(me.rb.GetCardinality())
}

func (me Bitmap) ToSortedSlice() (ret []int) {
	if me.rb == nil {
		return
	}
	missinggo.CastSlice(&ret, me.rb.ToArray())
	return
}

func (me *Bitmap) lazyRB() *roaring.Bitmap {
	if me.rb == nil {
		me.rb = roaring.NewBitmap()
	}
	return me.rb
}

func (me *Bitmap) Iter(f func(interface{}) bool) {
	me.IterTyped(func(i int) bool {
		return f(i)
	})
}

func (me Bitmap) IterTyped(f func(int) bool) bool {
	if me.rb == nil {
		return true
	}
	it := me.rb.Iterator()
	for it.HasNext() {
		if !f(int(it.Next())) {
			return false
		}
	}
	return true
}

func (me *Bitmap) Add(is ...int) {
	rb := me.lazyRB()
	for _, i := range is {
		rb.AddInt(i)
	}
}

func (me *Bitmap) AddRange(begin, end int) {
	if begin >= end {
		return
	}
	me.lazyRB().AddRange(uint32(begin), uint32(end))
}

func (me *Bitmap) Remove(i int) {
	if me.rb == nil {
		return
	}
	me.rb.Remove(uint32(i))
}

func (me *Bitmap) Union(other *Bitmap) {
	me.lazyRB().Or(other.lazyRB())
}

func (me *Bitmap) Contains(i int) bool {
	if me.rb == nil {
		return false
	}
	return me.rb.Contains(uint32(i))
}

type Iter struct {
	ii roaring.IntIterable
}

func (me *Iter) Next() bool {
	if me == nil {
		return false
	}
	return me.ii.HasNext()
}

func (me *Iter) Value() interface{} {
	return me.ValueInt()
}

func (me *Iter) ValueInt() int {
	return int(me.ii.Next())
}

func (me *Iter) Stop() {}

func Sub(left, right *Bitmap) *Bitmap {
	return &Bitmap{
		rb: roaring.AndNot(left.lazyRB(), right.lazyRB()),
	}
}

func (me *Bitmap) Sub(other *Bitmap) {
	if other.rb == nil {
		return
	}
	if me.rb == nil {
		return
	}
	me.rb.AndNot(other.rb)
}

func (me *Bitmap) Clear() {
	if me.rb == nil {
		return
	}
	me.rb.Clear()
}

func (me Bitmap) Copy() (ret Bitmap) {
	ret = me
	if ret.rb != nil {
		ret.rb = ret.rb.Clone()
	}
	return
}

func (me *Bitmap) FlipRange(begin, end int) {
	me.lazyRB().FlipInt(begin, end)
}

func (me *Bitmap) Get(bit int) bool {
	return me.rb != nil && me.rb.ContainsInt(bit)
}

func (me *Bitmap) Set(bit int, value bool) {
	if value {
		me.lazyRB().AddInt(bit)
	} else {
		if me.rb != nil {
			me.rb.Remove(uint32(bit))
		}
	}
}

func (me *Bitmap) RemoveRange(begin, end int) *Bitmap {
	if me.rb == nil {
		return me
	}
	me.rb.RemoveRange(uint32(begin), uint32(end))
	return me
}
<|endoftext|>


<commit_before>package bob

import "testing"

var testCases = []struct {
	description string
	expected    string
	input       string
}{
	{
		"stating something",
		"Whatever.",
		"Tom-ay-to, tom-aaaah-to.",
	},
	{
		"shouting",
		"Woah, chill out!",
		"WATCH OUT!",
	},
	{
		"asking a question",
		"Sure.",
		"Does this cryogenic chamber make me look fat?",
	},
	{
		"asking a numeric question",
		"Sure.",
		"You are, what, like 15?",
	},
	{
		"talking forcefully",
		"Whatever.",
		"Let's go make out behind the gym!",
	},
	{
		"using acronyms in regular speech",
		"Whatever.",
		"It's OK if you don't want to go to the DMV.",
	},
	{
		"forceful questions",
		"Woah, chill out!",
		"WHAT THE HELL WERE YOU THINKING?",
	},
	{
		"shouting numbers",
		"Woah, chill out!",
		"1, 2, 3 GO!",
	},
	{
		"only numbers",
		"Whatever.",
		"1, 2, 3",
	},
	{
		"question with only numbers",
		"Sure.",
		"4?",
	},
	{
		"shouting with special characters",
		"Woah, chill out!",
		"ZOMG THE %^*@#$(*^ ZOMBIES ARE COMING!!11!!1!",
	},
	{
		"shouting with no exclamation mark",
		"Woah, chill out!",
		"I HATE YOU",
	},
	{
		"statement containing question mark",
		"Whatever.",
		"Ending with ? means a question.",
	},
	{
		"prattling on",
		"Sure.",
		"Wait! Hang on. Are you going to be OK?",
	},
	{
		"silence",
		"Fine. Be that way!",
		"",
	},
	{
		"prolonged silence",
		"Fine. Be that way!",
		"    ",
	},
	{
		"really prolonged silence",
		"Fine. Be that way!",
		"                 ",
	},
	{
		"multi line trick question",
		"Whatever.",
		"Do I ever change my mind?\nNo.",
	},
}

func TestHeyBob(t *testing.T) {
	for _, tt := range testCases {
		actual := Hey(tt.input)
		if actual != tt.expected {
			msg := `
	ALICE (%s): %s
	BOB: %s

	Expected Bob to respond: %s
			`
			t.Fatalf(msg, tt.description, tt.input, actual, tt.expected)
		}
	}
}
<commit_msg>Added benchmark in go/bob<commit_after>package bob

import "testing"

var testCases = []struct {
	description string
	expected    string
	input       string
}{
	{
		"stating something",
		"Whatever.",
		"Tom-ay-to, tom-aaaah-to.",
	},
	{
		"shouting",
		"Woah, chill out!",
		"WATCH OUT!",
	},
	{
		"asking a question",
		"Sure.",
		"Does this cryogenic chamber make me look fat?",
	},
	{
		"asking a numeric question",
		"Sure.",
		"You are, what, like 15?",
	},
	{
		"talking forcefully",
		"Whatever.",
		"Let's go make out behind the gym!",
	},
	{
		"using acronyms in regular speech",
		"Whatever.",
		"It's OK if you don't want to go to the DMV.",
	},
	{
		"forceful questions",
		"Woah, chill out!",
		"WHAT THE HELL WERE YOU THINKING?",
	},
	{
		"shouting numbers",
		"Woah, chill out!",
		"1, 2, 3 GO!",
	},
	{
		"only numbers",
		"Whatever.",
		"1, 2, 3",
	},
	{
		"question with only numbers",
		"Sure.",
		"4?",
	},
	{
		"shouting with special characters",
		"Woah, chill out!",
		"ZOMG THE %^*@#$(*^ ZOMBIES ARE COMING!!11!!1!",
	},
	{
		"shouting with no exclamation mark",
		"Woah, chill out!",
		"I HATE YOU",
	},
	{
		"statement containing question mark",
		"Whatever.",
		"Ending with ? means a question.",
	},
	{
		"prattling on",
		"Sure.",
		"Wait! Hang on. Are you going to be OK?",
	},
	{
		"silence",
		"Fine. Be that way!",
		"",
	},
	{
		"prolonged silence",
		"Fine. Be that way!",
		"    ",
	},
	{
		"really prolonged silence",
		"Fine. Be that way!",
		"                 ",
	},
	{
		"multi line trick question",
		"Whatever.",
		"Do I ever change my mind?\nNo.",
	},
}

func TestHeyBob(t *testing.T) {
	for _, tt := range testCases {
		actual := Hey(tt.input)
		if actual != tt.expected {
			msg := `
	ALICE (%s): %s
	BOB: %s

	Expected Bob to respond: %s
			`
			t.Fatalf(msg, tt.description, tt.input, actual, tt.expected)
		}
	}
}

func BenchmarkBob(b *testing.B) {
	b.StopTimer()
	for _, tt := range testCases {
		b.StartTimer()

		for i := 0; i < b.N; i++ {
			Hey(tt.input)
		}

		b.StopTimer()
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 Google, Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style license
// that can be found in the LICENSE file in the root of the source
// tree.

// Package pcapgo provides some native PCAP support, not requiring
// C libpcap to be installed.
package pcapgo

import (
	"encoding/binary"
	"fmt"
	"io"

	"github.com/google/gopacket"
	"github.com/google/gopacket/layers"
)

// Writer wraps an underlying io.Writer to write packet data in PCAP
// format.  See http://wiki.wireshark.org/Development/LibpcapFileFormat
// for information on the file format.
//
// For those that care, we currently write v2.4 files with nanosecond
// timestamp resolution and little-endian encoding.
type Writer struct {
	w io.Writer
}

const magicMicroseconds = 0xA1B2C3D4
const versionMajor = 2
const versionMinor = 4

// NewWriter returns a new writer object, for writing packet data out
// to the given writer.  If this is a new empty writer (as opposed to
// an append), you must call WriteFileHeader before WritePacket.
//
//  // Write a new file:
//  f, _ := os.Create("/tmp/file.pcap")
//  w := pcapgo.NewWriter(f)
//  w.WriteFileHeader(65536, layers.LinkTypeEthernet)  // new file, must do this.
//  w.WritePacket(gopacket.CaptureInfo{...}, data1)
//  f.Close()
//  // Append to existing file (must have same snaplen and linktype)
//  f2, _ := os.OpenFile("/tmp/file.pcap", os.O_APPEND, 0700)
//  w2 := pcapgo.NewWriter(f2)
//  // no need for file header, it's already written.
//  w2.WritePacket(gopacket.CaptureInfo{...}, data2)
//  f2.Close()
func NewWriter(w io.Writer) *Writer {
	return &Writer{w: w}
}

// WriteFileHeader writes a file header out to the writer.
// This must be called exactly once per output.
func (w *Writer) WriteFileHeader(snaplen uint32, linktype layers.LinkType) error {
	var buf [24]byte
	binary.LittleEndian.PutUint32(buf[0:4], magicMicroseconds)
	binary.LittleEndian.PutUint16(buf[4:6], versionMajor)
	binary.LittleEndian.PutUint16(buf[6:8], versionMinor)
	// bytes 8:12 stay 0 (timezone = UTC)
	// bytes 12:16 stay 0 (sigfigs is always set to zero, according to
	//   http://wiki.wireshark.org/Development/LibpcapFileFormat
	binary.LittleEndian.PutUint32(buf[16:20], snaplen)
	binary.LittleEndian.PutUint32(buf[20:24], uint32(linktype))
	_, err := w.w.Write(buf[:])
	return err
}

const nanosPerMicro = 1000
const microsPerSecond = 1000000

func (w *Writer) writePacketHeader(ci gopacket.CaptureInfo) error {
	var buf [16]byte
	micros := ci.Timestamp.UnixNano() / nanosPerMicro
	secs, usecs := uint32(micros/microsPerSecond), uint32(micros%microsPerSecond)
	binary.LittleEndian.PutUint32(buf[0:4], secs)
	binary.LittleEndian.PutUint32(buf[4:8], usecs)
	binary.LittleEndian.PutUint32(buf[8:12], uint32(ci.CaptureLength))
	binary.LittleEndian.PutUint32(buf[12:16], uint32(ci.Length))
	_, err := w.w.Write(buf[:])
	return err
}

// WritePacket writes the given packet data out to the file.
func (w *Writer) WritePacket(ci gopacket.CaptureInfo, data []byte) error {
	if ci.CaptureLength != len(data) {
		return fmt.Errorf("capture length %d does not match data length %d", ci.CaptureLength, len(data))
	}
	if ci.CaptureLength > ci.Length {
		return fmt.Errorf("invalid capture info %+v:  capture length > length", ci)
	}
	if err := w.writePacketHeader(ci); err != nil {
		return fmt.Errorf("error writing packet header: %v", err)
	}
	_, err := w.w.Write(data)
	return err
}
<commit_msg>pcapgo: Add support for writing packets with Timestamp zero<commit_after>// Copyright 2012 Google, Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style license
// that can be found in the LICENSE file in the root of the source
// tree.

// Package pcapgo provides some native PCAP support, not requiring
// C libpcap to be installed.
package pcapgo

import (
	"encoding/binary"
	"fmt"
	"io"
	"time"

	"github.com/google/gopacket"
	"github.com/google/gopacket/layers"
)

// Writer wraps an underlying io.Writer to write packet data in PCAP
// format.  See http://wiki.wireshark.org/Development/LibpcapFileFormat
// for information on the file format.
//
// For those that care, we currently write v2.4 files with nanosecond
// timestamp resolution and little-endian encoding.
type Writer struct {
	w io.Writer
}

const magicMicroseconds = 0xA1B2C3D4
const versionMajor = 2
const versionMinor = 4

// NewWriter returns a new writer object, for writing packet data out
// to the given writer.  If this is a new empty writer (as opposed to
// an append), you must call WriteFileHeader before WritePacket.
//
//  // Write a new file:
//  f, _ := os.Create("/tmp/file.pcap")
//  w := pcapgo.NewWriter(f)
//  w.WriteFileHeader(65536, layers.LinkTypeEthernet)  // new file, must do this.
//  w.WritePacket(gopacket.CaptureInfo{...}, data1)
//  f.Close()
//  // Append to existing file (must have same snaplen and linktype)
//  f2, _ := os.OpenFile("/tmp/file.pcap", os.O_APPEND, 0700)
//  w2 := pcapgo.NewWriter(f2)
//  // no need for file header, it's already written.
//  w2.WritePacket(gopacket.CaptureInfo{...}, data2)
//  f2.Close()
func NewWriter(w io.Writer) *Writer {
	return &Writer{w: w}
}

// WriteFileHeader writes a file header out to the writer.
// This must be called exactly once per output.
func (w *Writer) WriteFileHeader(snaplen uint32, linktype layers.LinkType) error {
	var buf [24]byte
	binary.LittleEndian.PutUint32(buf[0:4], magicMicroseconds)
	binary.LittleEndian.PutUint16(buf[4:6], versionMajor)
	binary.LittleEndian.PutUint16(buf[6:8], versionMinor)
	// bytes 8:12 stay 0 (timezone = UTC)
	// bytes 12:16 stay 0 (sigfigs is always set to zero, according to
	//   http://wiki.wireshark.org/Development/LibpcapFileFormat
	binary.LittleEndian.PutUint32(buf[16:20], snaplen)
	binary.LittleEndian.PutUint32(buf[20:24], uint32(linktype))
	_, err := w.w.Write(buf[:])
	return err
}

const nanosPerMicro = 1000

func (w *Writer) writePacketHeader(ci gopacket.CaptureInfo) error {
	var buf [16]byte

	t := ci.Timestamp
	if t.IsZero() {
		t = time.Now()
	}
	secs := t.Unix()
	usecs := t.Nanosecond() / nanosPerMicro
	binary.LittleEndian.PutUint32(buf[0:4], uint32(secs))
	binary.LittleEndian.PutUint32(buf[4:8], uint32(usecs))
	binary.LittleEndian.PutUint32(buf[8:12], uint32(ci.CaptureLength))
	binary.LittleEndian.PutUint32(buf[12:16], uint32(ci.Length))
	_, err := w.w.Write(buf[:])
	return err
}

// WritePacket writes the given packet data out to the file.
func (w *Writer) WritePacket(ci gopacket.CaptureInfo, data []byte) error {
	if ci.CaptureLength != len(data) {
		return fmt.Errorf("capture length %d does not match data length %d", ci.CaptureLength, len(data))
	}
	if ci.CaptureLength > ci.Length {
		return fmt.Errorf("invalid capture info %+v:  capture length > length", ci)
	}
	if err := w.writePacketHeader(ci); err != nil {
		return fmt.Errorf("error writing packet header: %v", err)
	}
	_, err := w.w.Write(data)
	return err
}
<|endoftext|>


<commit_before><commit_msg>Update gowebserver.go<commit_after>package main

import (
	"fmt"
	"net/http"
)

func handler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Hi there, I love %s!", r.URL.Path[1:])
}

func main() {
	http.HandleFunc("/", handler)
	http.ListenAndServe(":8080", nil)
}
<|endoftext|>


<commit_before>package concrete

import (
	"github.com/gonum/graph"
	"math"
)

type DenseGraph struct {
	adjacencyMatrix []float64
	numNodes        int
}

func NewDenseGraph(numNodes int, passable bool) graph.Graph {
	dg := &DenseGraph{adjacencyMatrix: make([]float64, numNodes*numNodes), numNodes: numNodes}
	if passable {
		for i := range dg.adjacencyMatrix {
			dg.adjacencyMatrix[i] = 1.0
		}
	} else {
		for i := range dg.adjacencyMatrix {
			dg.adjacencyMatrix[i] = math.Inf(1)
		}
	}

	return dg
}

func (dg *DenseGraph) NodeExists(node graph.Node) bool {
	return node.ID() < dg.numNodes
}

func (dg *DenseGraph) Degree(node graph.Node) int {
	deg := 0
	for i := 0; i < dg.numNodes; i++ {
		if dg.adjacencyMatrix[i*dg.numNodes+node.ID()] != math.Inf(1) {
			deg++
		}

		if dg.adjacencyMatrix[node.ID()*dg.numNodes+i] != math.Inf(1) {
			deg++
		}
	}

	return deg
}

func (dg *DenseGraph) NodeList() []graph.Node {
	nodes := make([]graph.Node, dg.numNodes)
	for i := 0; i < dg.numNodes; i++ {
		nodes[i] = GonumNode(i)
	}

	return nodes
}

func (dg *DenseGraph) EdgeList() []graph.Edge {
	edges := make([]graph.Edge, 0, len(dg.adjacencyMatrix))
	for i := 0; i < dg.numNodes; i++ {
		for j := 0; j < dg.numNodes; j++ {
			if dg.adjacencyMatrix[i*dg.numNodes+j] != math.Inf(1) {
				edges = append(edges, GonumEdge{GonumNode(i), GonumNode(j)})
			}
		}
	}

	return edges
}

func (dg *DenseGraph) Neighbors(node graph.Node) []graph.Node {
	neighbors := make([]graph.Node, 0)
	for i := 0; i < dg.numNodes; i++ {
		if dg.adjacencyMatrix[i*dg.numNodes+node.ID()] != math.Inf(1) ||
			dg.adjacencyMatrix[node.ID()*dg.numNodes+i] != math.Inf(1) {
			neighbors = append(neighbors, GonumNode(i))
		}
	}

	return neighbors
}

func (dg *DenseGraph) IsNeighbor(node, neighbor graph.Node) bool {
	return dg.adjacencyMatrix[neighbor.ID()*dg.numNodes+node.ID()] != math.Inf(1) ||
		dg.adjacencyMatrix[node.ID()*dg.numNodes+neighbor.ID()] != math.Inf(1)
}

func (dg *DenseGraph) Successors(node graph.Node) []graph.Node {
	neighbors := make([]graph.Node, 0)
	for i := 0; i < dg.numNodes; i++ {
		if dg.adjacencyMatrix[node.ID()*dg.numNodes+i] != math.Inf(1) {
			neighbors = append(neighbors, GonumNode(i))
		}
	}

	return neighbors
}

func (dg *DenseGraph) IsSuccessor(node, succ graph.Node) bool {
	return dg.adjacencyMatrix[node.ID()*dg.numNodes+succ.ID()] != math.Inf(1)
}

func (dg *DenseGraph) Predecessors(node graph.Node) []graph.Node {
	neighbors := make([]graph.Node, 0)
	for i := 0; i < dg.numNodes; i++ {
		if dg.adjacencyMatrix[i*dg.numNodes+node.ID()] != math.Inf(1) {
			neighbors = append(neighbors, GonumNode(i))
		}
	}

	return neighbors
}

func (dg *DenseGraph) IsPredecessor(node, pred graph.Node) bool {
	return dg.adjacencyMatrix[pred.ID()*dg.numNodes+node.ID()] != math.Inf(1)
}

// Naturally dense, we don't need to do anything
func (dg *DenseGraph) Crunch() {
}

func (dg *DenseGraph) SetEdgeCost(node, succ graph.Node, cost float64, directed bool) {
	dg.adjacencyMatrix[node.ID()*dg.numNodes+succ.ID()] = cost
	if !directed {
		dg.adjacencyMatrix[succ.ID()*dg.numNodes+node.ID()] = cost
	}
}

// More or less equivalent to SetEdgeCost(node, succ, math.Inf(1), directed)
func (dg *DenseGraph) RemoveEdge(node, succ graph.Node, directed bool) {
	dg.adjacencyMatrix[node.ID()*dg.numNodes+succ.ID()] = math.Inf(1)
	if !directed {
		dg.adjacencyMatrix[succ.ID()*dg.numNodes+node.ID()] = math.Inf(1)
	}
}
<commit_msg>Made a basic dense graph<commit_after>package concrete

import (
	"github.com/gonum/graph"
	"math"
)

// A dense graph is a graph such that all IDs are in a contiguous block from 0 to TheNumberOfNodes-1
// it uses an adjacency matrix and should be relatively fast for both access and writing.
//
// This graph implements the CrunchGraph, but since it's naturally dense this is superfluous
type DenseGraph struct {
	adjacencyMatrix []float64
	numNodes        int
}

// Creates a dense graph with the proper number of nodes. If passable is true all nodes will have
// an edge with cost 1.0, otherwise every node will start unconnected (cost of +Inf)
func NewDenseGraph(numNodes int, passable bool) *DenseGraph {
	dg := &DenseGraph{adjacencyMatrix: make([]float64, numNodes*numNodes), numNodes: numNodes}
	if passable {
		for i := range dg.adjacencyMatrix {
			dg.adjacencyMatrix[i] = 1.0
		}
	} else {
		for i := range dg.adjacencyMatrix {
			dg.adjacencyMatrix[i] = math.Inf(1)
		}
	}

	return dg
}

func (dg *DenseGraph) NodeExists(node graph.Node) bool {
	return node.ID() < dg.numNodes
}

func (dg *DenseGraph) Degree(node graph.Node) int {
	deg := 0
	for i := 0; i < dg.numNodes; i++ {
		if dg.adjacencyMatrix[i*dg.numNodes+node.ID()] != math.Inf(1) {
			deg++
		}

		if dg.adjacencyMatrix[node.ID()*dg.numNodes+i] != math.Inf(1) {
			deg++
		}
	}

	return deg
}

func (dg *DenseGraph) NodeList() []graph.Node {
	nodes := make([]graph.Node, dg.numNodes)
	for i := 0; i < dg.numNodes; i++ {
		nodes[i] = GonumNode(i)
	}

	return nodes
}

func (dg *DenseGraph) DirectedEdgeList() []graph.Edge {
	edges := make([]graph.Edge, 0, len(dg.adjacencyMatrix))
	for i := 0; i < dg.numNodes; i++ {
		for j := 0; j < dg.numNodes; j++ {
			if dg.adjacencyMatrix[i*dg.numNodes+j] != math.Inf(1) {
				edges = append(edges, GonumEdge{GonumNode(i), GonumNode(j)})
			}
		}
	}

	return edges
}

func (dg *DenseGraph) Neighbors(node graph.Node) []graph.Node {
	neighbors := make([]graph.Node, 0)
	for i := 0; i < dg.numNodes; i++ {
		if dg.adjacencyMatrix[i*dg.numNodes+node.ID()] != math.Inf(1) ||
			dg.adjacencyMatrix[node.ID()*dg.numNodes+i] != math.Inf(1) {
			neighbors = append(neighbors, GonumNode(i))
		}
	}

	return neighbors
}

func (dg *DenseGraph) IsNeighbor(node, neighbor graph.Node) bool {
	return dg.adjacencyMatrix[neighbor.ID()*dg.numNodes+node.ID()] != math.Inf(1) ||
		dg.adjacencyMatrix[node.ID()*dg.numNodes+neighbor.ID()] != math.Inf(1)
}

func (dg *DenseGraph) Successors(node graph.Node) []graph.Node {
	neighbors := make([]graph.Node, 0)
	for i := 0; i < dg.numNodes; i++ {
		if dg.adjacencyMatrix[node.ID()*dg.numNodes+i] != math.Inf(1) {
			neighbors = append(neighbors, GonumNode(i))
		}
	}

	return neighbors
}

func (dg *DenseGraph) IsSuccessor(node, succ graph.Node) bool {
	return dg.adjacencyMatrix[node.ID()*dg.numNodes+succ.ID()] != math.Inf(1)
}

func (dg *DenseGraph) Predecessors(node graph.Node) []graph.Node {
	neighbors := make([]graph.Node, 0)
	for i := 0; i < dg.numNodes; i++ {
		if dg.adjacencyMatrix[i*dg.numNodes+node.ID()] != math.Inf(1) {
			neighbors = append(neighbors, GonumNode(i))
		}
	}

	return neighbors
}

func (dg *DenseGraph) IsPredecessor(node, pred graph.Node) bool {
	return dg.adjacencyMatrix[pred.ID()*dg.numNodes+node.ID()] != math.Inf(1)
}

// DenseGraph is naturally dense, we don't need to do anything
func (dg *DenseGraph) Crunch() {
}

func (dg *DenseGraph) Cost(node, succ graph.Node) float64 {
	return dg.adjacencyMatrix[node.ID()*dg.numNodes+succ.ID()]
}

// Sets the cost of the edge between node and succ. If the cost is +Inf, it will remove the edge,
// if directed is true, it will only remove the edge one way. If it's false it will change the cost
// of the edge from succ to node as well.
func (dg *DenseGraph) SetEdgeCost(node, succ graph.Node, cost float64, directed bool) {
	dg.adjacencyMatrix[node.ID()*dg.numNodes+succ.ID()] = cost
	if !directed {
		dg.adjacencyMatrix[succ.ID()*dg.numNodes+node.ID()] = cost
	}
}

// More or less equivalent to SetEdgeCost(node, succ, math.Inf(1), directed)
func (dg *DenseGraph) RemoveEdge(node, succ graph.Node, directed bool) {
	dg.adjacencyMatrix[node.ID()*dg.numNodes+succ.ID()] = math.Inf(1)
	if !directed {
		dg.adjacencyMatrix[succ.ID()*dg.numNodes+node.ID()] = math.Inf(1)
	}
}
<|endoftext|>


<commit_before>
package atourofgo

/**
*	Uses individual import statements...
*/
import "fmt"
import "math/rand"

func PackageExample2() {
	fmt.Printf("How you have %g problems", rand.Intn(10))
}
<commit_msg>Made sure we're seeding rand<commit_after>
package atourofgo

/**
*	Uses individual import statements...
*/
import "fmt"
import "math/rand"

func PackageExample2() {
	rand.Seed(123)
	fmt.Printf("How you have %g problems", rand.Intn(10))
}
<|endoftext|>


<commit_before>package controller

import (
	"github.com/studygolang/studygolang/context"
	"github.com/studygolang/studygolang/logic"

	echo "github.com/labstack/echo/v4"
	"github.com/polaris1119/goutils"
)

type SearchController struct{}

// 注册路由
func (self SearchController) RegisterRoute(g *echo.Group) {
	g.GET("/search", self.Search)
	g.GET("/tag/:name", self.TagList)
}

// Search
func (SearchController) Search(ctx echo.Context) error {
	q := ctx.QueryParam("q")
	field := ctx.QueryParam("f")
	p := goutils.MustInt(ctx.QueryParam("p"), 1)

	rows := 50

	respBody, err := logic.DefaultSearcher.DoSearch(q, field, (p-1)*rows, rows)

	data := map[string]interface{}{
		"respBody": respBody,
		"q":        q,
		"f":        field,
	}
	if err == nil {
		uri := "/search?q=" + q + "&f=" + field + "&"
		paginator := logic.NewPaginatorWithPerPage(p, rows)
		data["pageHtml"] = paginator.SetTotal(int64(respBody.NumFound)).GetPageHtml(uri)
	}

	return render(ctx, "search.html", data)
}

// TagList
func (SearchController) TagList(ctx echo.Context) error {
	field := "tag"
	p := goutils.MustInt(ctx.QueryParam("p"), 1)
	q := ctx.Param("name")
	if q == "" {
		return render(ctx, "notfound", nil)
	}

	rows := 50

	respBody, err := logic.DefaultSearcher.DoSearch(q, field, (p-1)*rows, rows)
	users, nodes := logic.DefaultSearcher.FillNodeAndUser(context.EchoContext(ctx), respBody)

	data := map[string]interface{}{
		"respBody": respBody,
		"name":     q,
		"users":    users,
		"nodes":    nodes,
	}
	if err == nil {
		uri := "/tag/" + q + "?"
		paginator := logic.NewPaginatorWithPerPage(p, rows)
		data["pageHtml"] = paginator.SetTotal(int64(respBody.NumFound)).GetPageHtml(uri)
	}

	return render(ctx, "feed/tag.html", data)
}
<commit_msg>fix: 搜索异常时导致页面渲染异常, 搜索框 XSS 漏洞<commit_after>package controller

import (
	"html"

	"github.com/studygolang/studygolang/context"
	"github.com/studygolang/studygolang/logic"

	echo "github.com/labstack/echo/v4"
	"github.com/polaris1119/goutils"
)

type SearchController struct{}

// 注册路由
func (self SearchController) RegisterRoute(g *echo.Group) {
	g.GET("/search", self.Search)
	g.GET("/tag/:name", self.TagList)
}

// Search
func (SearchController) Search(ctx echo.Context) error {
	q := ctx.QueryParam("q")
	field := ctx.QueryParam("f")
	p := goutils.MustInt(ctx.QueryParam("p"), 1)

	rows := 50

	respBody, err := logic.DefaultSearcher.DoSearch(q, field, (p-1)*rows, rows)

	data := map[string]interface{}{
		"respBody": respBody,
		"q":        q,
		"f":        field,
	}
	if err != nil {
		return render(ctx, "500.html", nil)
	}
	uri := "/search?q=" + html.EscapeString(q) + "&f=" + field + "&"
	paginator := logic.NewPaginatorWithPerPage(p, rows)
	data["pageHtml"] = paginator.SetTotal(int64(respBody.NumFound)).GetPageHtml(uri)

	return render(ctx, "search.html", data)
}

// TagList
func (SearchController) TagList(ctx echo.Context) error {
	field := "tag"
	p := goutils.MustInt(ctx.QueryParam("p"), 1)
	q := ctx.Param("name")
	if q == "" {
		return render(ctx, "notfound", nil)
	}

	rows := 50

	respBody, err := logic.DefaultSearcher.DoSearch(q, field, (p-1)*rows, rows)
	users, nodes := logic.DefaultSearcher.FillNodeAndUser(context.EchoContext(ctx), respBody)

	data := map[string]interface{}{
		"respBody": respBody,
		"name":     q,
		"users":    users,
		"nodes":    nodes,
	}
	if err != nil {
		return render(ctx, "500.html", nil)
	}
	uri := "/tag/" + q + "?"
	paginator := logic.NewPaginatorWithPerPage(p, rows)
	data["pageHtml"] = paginator.SetTotal(int64(respBody.NumFound)).GetPageHtml(uri)

	return render(ctx, "feed/tag.html", data)
}
<|endoftext|>


<commit_before>package cluster

import (
	"strings"

	"github.com/docker/docker/pkg/stringid"
	"github.com/samalba/dockerclient"
)

// Container is exported
type Container struct {
	dockerclient.Container

	Config *ContainerConfig
	Info   dockerclient.ContainerInfo
	Engine *Engine
}

// Refresh container
func (c *Container) Refresh() (*Container, error) {
	return c.Engine.refreshContainer(c.Id, true)
}

// Containers represents a list a containers
type Containers []*Container

// Get returns a container using its ID or Name
func (containers Containers) Get(IDOrName string) *Container {
	// Abort immediately if the name is empty.
	if len(IDOrName) == 0 {
		return nil
	}

	// Match exact or short Container ID.
	for _, container := range containers {
		if container.Id == IDOrName || stringid.TruncateID(container.Id) == IDOrName {
			return container
		}
	}

	// Match exact Swarm ID.
	for _, container := range containers {
		if swarmID := container.Config.SwarmID(); swarmID == IDOrName || stringid.TruncateID(swarmID) == IDOrName {
			return container
		}
	}

	candidates := []*Container{}

	// Match name, /name or engine/name.
	for _, container := range containers {
		found := false
		for _, name := range container.Names {
			if name == IDOrName || name == "/"+IDOrName || container.Engine.ID+name == IDOrName || container.Engine.Name+name == IDOrName {
				found = true
			}
		}
		if found {
			candidates = append(candidates, container)
		}
	}

	if size := len(candidates); size == 1 {
		return candidates[0]
	} else if size > 1 {
		return nil
	}

	// Match Container ID prefix.
	for _, container := range containers {
		if strings.HasPrefix(container.Id, IDOrName) {
			candidates = append(candidates, container)
		}
	}

	// Match Swarm ID prefix.
	for _, container := range containers {
		if strings.HasPrefix(container.Config.SwarmID(), IDOrName) {
			candidates = append(candidates, container)
		}
	}

	if len(candidates) == 1 {
		return candidates[0]
	}

	return nil
}
<commit_msg>fix minor typo<commit_after>package cluster

import (
	"strings"

	"github.com/docker/docker/pkg/stringid"
	"github.com/samalba/dockerclient"
)

// Container is exported
type Container struct {
	dockerclient.Container

	Config *ContainerConfig
	Info   dockerclient.ContainerInfo
	Engine *Engine
}

// Refresh container
func (c *Container) Refresh() (*Container, error) {
	return c.Engine.refreshContainer(c.Id, true)
}

// Containers represents a list of containers
type Containers []*Container

// Get returns a container using its ID or Name
func (containers Containers) Get(IDOrName string) *Container {
	// Abort immediately if the name is empty.
	if len(IDOrName) == 0 {
		return nil
	}

	// Match exact or short Container ID.
	for _, container := range containers {
		if container.Id == IDOrName || stringid.TruncateID(container.Id) == IDOrName {
			return container
		}
	}

	// Match exact Swarm ID.
	for _, container := range containers {
		if swarmID := container.Config.SwarmID(); swarmID == IDOrName || stringid.TruncateID(swarmID) == IDOrName {
			return container
		}
	}

	candidates := []*Container{}

	// Match name, /name or engine/name.
	for _, container := range containers {
		found := false
		for _, name := range container.Names {
			if name == IDOrName || name == "/"+IDOrName || container.Engine.ID+name == IDOrName || container.Engine.Name+name == IDOrName {
				found = true
			}
		}
		if found {
			candidates = append(candidates, container)
		}
	}

	if size := len(candidates); size == 1 {
		return candidates[0]
	} else if size > 1 {
		return nil
	}

	// Match Container ID prefix.
	for _, container := range containers {
		if strings.HasPrefix(container.Id, IDOrName) {
			candidates = append(candidates, container)
		}
	}

	// Match Swarm ID prefix.
	for _, container := range containers {
		if strings.HasPrefix(container.Config.SwarmID(), IDOrName) {
			candidates = append(candidates, container)
		}
	}

	if len(candidates) == 1 {
		return candidates[0]
	}

	return nil
}
<|endoftext|>


<commit_before>package elasticthought

import (
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"reflect"
	"sync"

	"github.com/couchbaselabs/logg"
	"github.com/tleyden/go-couch"
)

// A classify job tries to classify images given by user against
// the given trained model
type ClassifyJob struct {
	ElasticThoughtDoc
	ProcessingState ProcessingState `json:"processing-state"`
	ProcessingLog   string          `json:"processing-log"`
	ClassifierID    string          `json:"classifier-id"`

	// Key: image url of image in cbfs
	// Value: the classification result for that image
	Results map[string]string `json:"results"`

	// had to make exported, due to https://github.com/gin-gonic/gin/pull/123
	// waiting for this to get merged into master branch, since go get
	// pulls from master branch.
	Configuration Configuration
}

// Create a new classify job.  If you don't use this, you must set the
// embedded ElasticThoughtDoc Type field.
func NewClassifyJob(c Configuration) *ClassifyJob {
	return &ClassifyJob{
		ElasticThoughtDoc: ElasticThoughtDoc{Type: DOC_TYPE_CLASSIFY_JOB},
		Configuration:     c,
	}
}

// Run this job
func (c *ClassifyJob) Run(wg *sync.WaitGroup) {

	defer wg.Done()

	logg.LogTo("CLASSIFY_JOB", "Run() called!")

	updatedState, err := c.UpdateProcessingState(Processing)
	if err != nil {
		c.recordProcessingError(err)
		return
	}

	if !updatedState {
		logg.LogTo("CLASSIFY_JOB", "%+v already processed.  Ignoring.", c)
		return
	}

	// TODO: add code to run job

	logg.LogTo("CLASSIFY_JOB", "lazily create dir.  images: %+v", c.Results)

	if err := c.createWorkDirectory(); err != nil {
		c.recordProcessingError(err)
		return
	}

	if err := c.downloadWorkAssets(); err != nil {
		c.recordProcessingError(err)
		return
	}

	// invoke caffe
	resultsMap, err := c.invokeCaffe()
	if err != nil {
		c.recordProcessingError(err)
		return
	}

	logg.LogTo("CLASSIFY_JOB", "resultsMap: %+v", resultsMap)

	// extract results

	// update classifyjob with results

}

func (c ClassifyJob) invokeCaffe() (map[string]interface{}, error) {

	// call "python classifier.py"
	// build command args
	cmdArgs := []string{"classifier.py"}
	python := "python"

	// debugging
	logg.LogTo("CLASSIFY_JOB", "Running %v with args %v", python, cmdArgs)
	logg.LogTo("CLASSIFY_JOB", "Path %v", os.Getenv("PATH"))

	// TEMP DEBUGGING -- remove this
	exec.Command("ls", "-alh", "/usr/local/bin").Run()

	// explicitly check if caffe binary found on the PATH
	lookPathResult, err := exec.LookPath("python")
	if err != nil {
		logg.LogError(fmt.Errorf("python not found on path: %v", err))
	}
	logg.LogTo("CLASSIFY_JOB", "python found on path: %v", lookPathResult)

	// Create command, but don't actually run it yet
	cmd := exec.Command(python, cmdArgs...)

	// set the directory where the command will be run in (important
	// because we depend on relative file paths to work)
	cmd.Dir = c.getWorkDirectory()

	// Run command
	if err := cmd.Run(); err != nil {
		return nil, err
	}

	// read output.json file into map
	result := map[string]interface{}{}
	resultFile, err := os.Open("result.json")
	if err != nil {
		return nil, err
	}
	jsonParser := json.NewDecoder(resultFile)
	if err = jsonParser.Decode(&result); err != nil {
		return nil, err
	}

	return result, nil

}

func (c ClassifyJob) getWorkDirectory() string {
	return filepath.Join(c.Configuration.WorkDirectory, c.Id)
}

func (c ClassifyJob) getWorkImagesDirectory() string {
	return filepath.Join(c.getWorkDirectory(), "images")
}

// lazily create work dir
func (c ClassifyJob) createWorkDirectory() error {
	workDir := c.getWorkDirectory()
	logg.LogTo("CLASSIFY_JOB", "Creating dir: %v", workDir)
	if err := Mkdir(workDir); err != nil {
		return err
	}

	imagesSubdir := c.getWorkImagesDirectory()
	if err := Mkdir(imagesSubdir); err != nil {
		return err
	}

	return nil

}

func (c ClassifyJob) getClassifier() (*Classifier, error) {

	classifier := NewClassifier(c.Configuration)
	err := classifier.Find(c.ClassifierID)
	return classifier, err
}

func (c ClassifyJob) getTrainingJob() (*TrainingJob, error) {

	classifier, err := c.getClassifier()
	if err != nil {
		return nil, err
	}

	trainingJob := NewTrainingJob(c.Configuration)
	err = trainingJob.Find(classifier.TrainingJobID)
	return trainingJob, err

}

func (c ClassifyJob) downloadWorkAssets() error {

	// caffe model
	if err := c.downloadCaffeModel(); err != nil {
		return err
	}

	// prototxt
	if err := c.downloadPrototxt(); err != nil {
		return err
	}

	// images
	if err := c.downloadImagesToClassify(); err != nil {
		return err
	}

	// python classify script
	workDirectory := c.getWorkDirectory()
	if err := c.copyPythonClassifier(workDirectory); err != nil {
		return err
	}

	return nil

}

func (c ClassifyJob) copyPythonClassifier(destDirPath string) error {

	// find out $GOPATH env variable
	gopath := os.Getenv("GOPATH")

	// find path to python classifier file
	elasticThoughtRoot := path.Join(
		gopath,
		"src",
		"github.com",
		"tleyden",
		"elastic-thought",
	)

	pythonScript := path.Join(
		elasticThoughtRoot,
		"scripts",
		"python-classifier",
		"classifier.py",
	)

	destPath := path.Join(destDirPath, "classifier.py")

	if err := CopyFileContents(pythonScript, destPath); err != nil {
		return err
	}

	return nil

}

func (c ClassifyJob) downloadCaffeModel() error {

	// get training job id, instantiate object
	trainingJob, err := c.getTrainingJob()
	if err != nil {
		return err
	}

	// make sure that training job state == Finished
	if trainingJob.GetProcessingState() != FinishedSuccessfully {
		return fmt.Errorf("TrainingJob is not finished yet")
	}

	// make sure the trained model url starts with cbfs
	trainedModelUrl := trainingJob.TrainedModelUrl

	// download from cbfs to local file system
	cbfs, err := c.Configuration.NewCbfsClient()
	if err != nil {
		return err
	}
	destPath := path.Join(c.getWorkDirectory(), "caffe.model")
	if err := downloadFromCbfs(cbfs, trainedModelUrl, destPath); err != nil {
		return err
	}

	return nil

}

func (c ClassifyJob) downloadPrototxt() error {

	classifier, err := c.getClassifier()
	if err != nil {
		return err
	}

	cbfs, err := c.Configuration.NewCbfsClient()
	if err != nil {
		return err
	}
	destPath := path.Join(c.getWorkDirectory(), "classifier.prototxt")
	if err := downloadFromCbfs(cbfs, classifier.SpecificationUrl, destPath); err != nil {
		return err
	}

	return nil
}

func (c ClassifyJob) downloadImagesToClassify() error {

	// for now, just download a single image
	// (if multiple images specified, all bust last will be clobbered)
	// TODO: fix this and download all images

	cbfs, err := c.Configuration.NewCbfsClient()
	if err != nil {
		return err
	}

	i := 0
	for imageUrl, _ := range c.Results {

		// url will be cbfs://<classify_job_id>/<imageurl_sha1_hash>
		_, imageSha1Hash := path.Split(imageUrl)
		destPath := path.Join(c.getWorkImagesDirectory(), imageSha1Hash)
		if err := downloadFromCbfs(cbfs, imageUrl, destPath); err != nil {
			return err
		}
		i += 1
	}

	return nil

}

// Update the processing state to new state.
func (c *ClassifyJob) UpdateProcessingState(newState ProcessingState) (bool, error) {

	updater := func(classifyJob *ClassifyJob) {
		classifyJob.ProcessingState = newState
	}

	doneMetric := func(classifyJob ClassifyJob) bool {
		return classifyJob.ProcessingState == newState
	}

	return c.casUpdate(updater, doneMetric)

}

func (c *ClassifyJob) SetResults(results map[string]string) (bool, error) {

	updater := func(classifyJob *ClassifyJob) {
		classifyJob.Results = results
	}

	doneMetric := func(classifyJob ClassifyJob) bool {
		return reflect.DeepEqual(results, classifyJob.Results)
	}

	return c.casUpdate(updater, doneMetric)

}

func (c *ClassifyJob) UpdateProcessingLog(val string) (bool, error) {

	updater := func(classifyJob *ClassifyJob) {
		classifyJob.ProcessingLog = val
	}

	doneMetric := func(classifyJob ClassifyJob) bool {
		return classifyJob.ProcessingLog == val
	}

	return c.casUpdate(updater, doneMetric)

}

func (c *ClassifyJob) casUpdate(updater func(*ClassifyJob), doneMetric func(ClassifyJob) bool) (bool, error) {

	db := c.Configuration.DbConnection()

	genUpdater := func(classifyJobPtr interface{}) {
		cjp := classifyJobPtr.(*ClassifyJob)
		updater(cjp)
	}

	genDoneMetric := func(classifyJobPtr interface{}) bool {
		cjp := classifyJobPtr.(*ClassifyJob)
		return doneMetric(*cjp)
	}

	refresh := func(classifyJobPtr interface{}) error {
		cjp := classifyJobPtr.(*ClassifyJob)
		return cjp.RefreshFromDB(db)
	}

	return casUpdate(db, c, genUpdater, genDoneMetric, refresh)

}

// Insert into database (only call this if you know it doesn't arleady exist,
// or else you'll end up w/ unwanted dupes)
func (c *ClassifyJob) Insert() error {

	db := c.Configuration.DbConnection()

	id, rev, err := db.Insert(c)
	if err != nil {
		err := fmt.Errorf("Error inserting classify job: %v.  Err: %v", c, err)
		return err
	}

	c.Id = id
	c.Revision = rev

	return nil

}

// CodeReview: duplication with RefreshFromDB in many places
func (c *ClassifyJob) RefreshFromDB(db couch.Database) error {
	classifyJob := ClassifyJob{}
	err := db.Retrieve(c.Id, &classifyJob)
	if err != nil {
		return err
	}
	*c = classifyJob
	return nil
}

// Find a classify Job in the db with the given id, or error if not found
// CodeReview: duplication with Find in many places
func (c *ClassifyJob) Find(id string) error {
	db := c.Configuration.DbConnection()
	c.Id = id
	if err := c.RefreshFromDB(db); err != nil {
		return err
	}
	return nil
}

// Codereview: de-dupe
func (c ClassifyJob) recordProcessingError(err error) {
	logg.LogError(err)
	db := c.Configuration.DbConnection()
	if err := c.Failed(db, err); err != nil {
		errMsg := fmt.Errorf("Error setting training job as failed: %v", err)
		logg.LogError(errMsg)
	}
}

func (c ClassifyJob) Failed(db couch.Database, processingErr error) error {

	_, err := c.UpdateProcessingState(Failed)
	if err != nil {
		return err
	}

	logg.LogTo("CLASSIFY_JOB", "updating processing log")

	logValue := fmt.Sprintf("%v", processingErr)
	_, err = c.UpdateProcessingLog(logValue)
	if err != nil {
		return err
	}

	return nil

}
<commit_msg>classify job calls runCmdTeeStdio<commit_after>package elasticthought

import (
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"reflect"
	"sync"

	"github.com/couchbaselabs/logg"
	"github.com/tleyden/go-couch"
)

// A classify job tries to classify images given by user against
// the given trained model
type ClassifyJob struct {
	ElasticThoughtDoc
	ProcessingState ProcessingState `json:"processing-state"`
	ProcessingLog   string          `json:"processing-log"`
	StdOutUrl       string          `json:"std-out-url"`
	StdErrUrl       string          `json:"std-err-url"`
	ClassifierID    string          `json:"classifier-id"`

	// Key: image url of image in cbfs
	// Value: the classification result for that image
	Results map[string]string `json:"results"`

	// had to make exported, due to https://github.com/gin-gonic/gin/pull/123
	// waiting for this to get merged into master branch, since go get
	// pulls from master branch.
	Configuration Configuration
}

// Create a new classify job.  If you don't use this, you must set the
// embedded ElasticThoughtDoc Type field.
func NewClassifyJob(c Configuration) *ClassifyJob {
	return &ClassifyJob{
		ElasticThoughtDoc: ElasticThoughtDoc{Type: DOC_TYPE_CLASSIFY_JOB},
		Configuration:     c,
	}
}

// Run this job
func (c *ClassifyJob) Run(wg *sync.WaitGroup) {

	defer wg.Done()

	logg.LogTo("CLASSIFY_JOB", "Run() called!")

	updatedState, err := c.UpdateProcessingState(Processing)
	if err != nil {
		c.recordProcessingError(err)
		return
	}

	if !updatedState {
		logg.LogTo("CLASSIFY_JOB", "%+v already processed.  Ignoring.", c)
		return
	}

	// TODO: add code to run job

	logg.LogTo("CLASSIFY_JOB", "lazily create dir.  images: %+v", c.Results)

	if err := c.createWorkDirectory(); err != nil {
		c.recordProcessingError(err)
		return
	}

	if err := c.downloadWorkAssets(); err != nil {
		c.recordProcessingError(err)
		return
	}

	// invoke caffe
	resultsMap, err := c.invokeCaffe()
	if err != nil {
		c.recordProcessingError(err)
		return
	}

	logg.LogTo("CLASSIFY_JOB", "resultsMap: %+v", resultsMap)

	// extract results

	// update classifyjob with results

}

func (c ClassifyJob) invokeCaffe() (map[string]interface{}, error) {

	// call "python classifier.py"
	// build command args
	cmdArgs := []string{"classifier.py"}
	python := "python"

	// debugging
	logg.LogTo("CLASSIFY_JOB", "Running %v with args %v", python, cmdArgs)
	logg.LogTo("CLASSIFY_JOB", "Path %v", os.Getenv("PATH"))

	// TEMP DEBUGGING -- remove this
	exec.Command("ls", "-alh", "/usr/local/bin").Run()

	// explicitly check if caffe binary found on the PATH
	lookPathResult, err := exec.LookPath("python")
	if err != nil {
		logg.LogError(fmt.Errorf("python not found on path: %v", err))
	}
	logg.LogTo("CLASSIFY_JOB", "python found on path: %v", lookPathResult)

	// Create command, but don't actually run it yet
	cmd := exec.Command(python, cmdArgs...)

	// set the directory where the command will be run in (important
	// because we depend on relative file paths to work)
	cmd.Dir = c.getWorkDirectory()

	// run the command and save stdio to files and tee to stdio streams
	if err := runCmdTeeStdio(cmd, c.getStdOutPath(), c.getStdErrPath()); err != nil {
		return nil, err
	}

	/*
		// read from temp files and write to cbfs.
		// initially I tried to write the stdout/stderr streams directly
		// to cbfs, but ran into an error related to the io.Seeker interface.
		if err := c.saveCmdOutputToCbfs(c.getStdOutPath()); err != nil {
			return fmt.Errorf("Error running caffe: could not save output to cbfs. Err: %v", err)
		}

		if err := c.saveCmdOutputToCbfs(c.getStdErrPath()); err != nil {
			return fmt.Errorf("Error running caffe: could not save output to cbfs. Err: %v", err)
		}
	*/

	// read output.json file into map
	result := map[string]interface{}{}
	resultFile, err := os.Open("result.json")
	if err != nil {
		return nil, err
	}
	jsonParser := json.NewDecoder(resultFile)
	if err = jsonParser.Decode(&result); err != nil {
		return nil, err
	}

	return result, nil

}

func (c ClassifyJob) getStdOutPath() string {
	return path.Join(c.getWorkDirectory(), "stdout")
}

func (c ClassifyJob) getStdErrPath() string {
	return path.Join(c.getWorkDirectory(), "stderr")
}

func (c ClassifyJob) getWorkDirectory() string {
	return filepath.Join(c.Configuration.WorkDirectory, c.Id)
}

func (c ClassifyJob) getWorkImagesDirectory() string {
	return filepath.Join(c.getWorkDirectory(), "images")
}

// lazily create work dir
func (c ClassifyJob) createWorkDirectory() error {
	workDir := c.getWorkDirectory()
	logg.LogTo("CLASSIFY_JOB", "Creating dir: %v", workDir)
	if err := Mkdir(workDir); err != nil {
		return err
	}

	imagesSubdir := c.getWorkImagesDirectory()
	if err := Mkdir(imagesSubdir); err != nil {
		return err
	}

	return nil

}

func (c ClassifyJob) getClassifier() (*Classifier, error) {

	classifier := NewClassifier(c.Configuration)
	err := classifier.Find(c.ClassifierID)
	return classifier, err
}

func (c ClassifyJob) getTrainingJob() (*TrainingJob, error) {

	classifier, err := c.getClassifier()
	if err != nil {
		return nil, err
	}

	trainingJob := NewTrainingJob(c.Configuration)
	err = trainingJob.Find(classifier.TrainingJobID)
	return trainingJob, err

}

func (c ClassifyJob) downloadWorkAssets() error {

	// caffe model
	if err := c.downloadCaffeModel(); err != nil {
		return err
	}

	// prototxt
	if err := c.downloadPrototxt(); err != nil {
		return err
	}

	// images
	if err := c.downloadImagesToClassify(); err != nil {
		return err
	}

	// python classify script
	workDirectory := c.getWorkDirectory()
	if err := c.copyPythonClassifier(workDirectory); err != nil {
		return err
	}

	return nil

}

func (c ClassifyJob) copyPythonClassifier(destDirPath string) error {

	// find out $GOPATH env variable
	gopath := os.Getenv("GOPATH")

	// find path to python classifier file
	elasticThoughtRoot := path.Join(
		gopath,
		"src",
		"github.com",
		"tleyden",
		"elastic-thought",
	)

	pythonScript := path.Join(
		elasticThoughtRoot,
		"scripts",
		"python-classifier",
		"classifier.py",
	)

	destPath := path.Join(destDirPath, "classifier.py")

	if err := CopyFileContents(pythonScript, destPath); err != nil {
		return err
	}

	return nil

}

func (c ClassifyJob) downloadCaffeModel() error {

	// get training job id, instantiate object
	trainingJob, err := c.getTrainingJob()
	if err != nil {
		return err
	}

	// make sure that training job state == Finished
	if trainingJob.GetProcessingState() != FinishedSuccessfully {
		return fmt.Errorf("TrainingJob is not finished yet")
	}

	// make sure the trained model url starts with cbfs
	trainedModelUrl := trainingJob.TrainedModelUrl

	// download from cbfs to local file system
	cbfs, err := c.Configuration.NewCbfsClient()
	if err != nil {
		return err
	}
	destPath := path.Join(c.getWorkDirectory(), "caffe.model")
	if err := downloadFromCbfs(cbfs, trainedModelUrl, destPath); err != nil {
		return err
	}

	return nil

}

func (c ClassifyJob) downloadPrototxt() error {

	classifier, err := c.getClassifier()
	if err != nil {
		return err
	}

	cbfs, err := c.Configuration.NewCbfsClient()
	if err != nil {
		return err
	}
	destPath := path.Join(c.getWorkDirectory(), "classifier.prototxt")
	if err := downloadFromCbfs(cbfs, classifier.SpecificationUrl, destPath); err != nil {
		return err
	}

	return nil
}

func (c ClassifyJob) downloadImagesToClassify() error {

	// for now, just download a single image
	// (if multiple images specified, all bust last will be clobbered)
	// TODO: fix this and download all images

	cbfs, err := c.Configuration.NewCbfsClient()
	if err != nil {
		return err
	}

	i := 0
	for imageUrl, _ := range c.Results {

		// url will be cbfs://<classify_job_id>/<imageurl_sha1_hash>
		_, imageSha1Hash := path.Split(imageUrl)
		destPath := path.Join(c.getWorkImagesDirectory(), imageSha1Hash)
		if err := downloadFromCbfs(cbfs, imageUrl, destPath); err != nil {
			return err
		}
		i += 1
	}

	return nil

}

// Update the processing state to new state.
func (c *ClassifyJob) UpdateProcessingState(newState ProcessingState) (bool, error) {

	updater := func(classifyJob *ClassifyJob) {
		classifyJob.ProcessingState = newState
	}

	doneMetric := func(classifyJob ClassifyJob) bool {
		return classifyJob.ProcessingState == newState
	}

	return c.casUpdate(updater, doneMetric)

}

func (c *ClassifyJob) SetResults(results map[string]string) (bool, error) {

	updater := func(classifyJob *ClassifyJob) {
		classifyJob.Results = results
	}

	doneMetric := func(classifyJob ClassifyJob) bool {
		return reflect.DeepEqual(results, classifyJob.Results)
	}

	return c.casUpdate(updater, doneMetric)

}

func (c *ClassifyJob) UpdateProcessingLog(val string) (bool, error) {

	updater := func(classifyJob *ClassifyJob) {
		classifyJob.ProcessingLog = val
	}

	doneMetric := func(classifyJob ClassifyJob) bool {
		return classifyJob.ProcessingLog == val
	}

	return c.casUpdate(updater, doneMetric)

}

func (c *ClassifyJob) casUpdate(updater func(*ClassifyJob), doneMetric func(ClassifyJob) bool) (bool, error) {

	db := c.Configuration.DbConnection()

	genUpdater := func(classifyJobPtr interface{}) {
		cjp := classifyJobPtr.(*ClassifyJob)
		updater(cjp)
	}

	genDoneMetric := func(classifyJobPtr interface{}) bool {
		cjp := classifyJobPtr.(*ClassifyJob)
		return doneMetric(*cjp)
	}

	refresh := func(classifyJobPtr interface{}) error {
		cjp := classifyJobPtr.(*ClassifyJob)
		return cjp.RefreshFromDB(db)
	}

	return casUpdate(db, c, genUpdater, genDoneMetric, refresh)

}

// Insert into database (only call this if you know it doesn't arleady exist,
// or else you'll end up w/ unwanted dupes)
func (c *ClassifyJob) Insert() error {

	db := c.Configuration.DbConnection()

	id, rev, err := db.Insert(c)
	if err != nil {
		err := fmt.Errorf("Error inserting classify job: %v.  Err: %v", c, err)
		return err
	}

	c.Id = id
	c.Revision = rev

	return nil

}

// CodeReview: duplication with RefreshFromDB in many places
func (c *ClassifyJob) RefreshFromDB(db couch.Database) error {
	classifyJob := ClassifyJob{}
	err := db.Retrieve(c.Id, &classifyJob)
	if err != nil {
		return err
	}
	*c = classifyJob
	return nil
}

// Find a classify Job in the db with the given id, or error if not found
// CodeReview: duplication with Find in many places
func (c *ClassifyJob) Find(id string) error {
	db := c.Configuration.DbConnection()
	c.Id = id
	if err := c.RefreshFromDB(db); err != nil {
		return err
	}
	return nil
}

// Codereview: de-dupe
func (c ClassifyJob) recordProcessingError(err error) {
	logg.LogError(err)
	db := c.Configuration.DbConnection()
	if err := c.Failed(db, err); err != nil {
		errMsg := fmt.Errorf("Error setting training job as failed: %v", err)
		logg.LogError(errMsg)
	}
}

func (c ClassifyJob) Failed(db couch.Database, processingErr error) error {

	_, err := c.UpdateProcessingState(Failed)
	if err != nil {
		return err
	}

	logg.LogTo("CLASSIFY_JOB", "updating processing log")

	logValue := fmt.Sprintf("%v", processingErr)
	_, err = c.UpdateProcessingLog(logValue)
	if err != nil {
		return err
	}

	return nil

}
<|endoftext|>


<commit_before>package clis

import (
	"fmt"
	"octlink/rstore/modules/blobs"
	"octlink/rstore/modules/blobsmanifest"
	"octlink/rstore/modules/config"
	"octlink/rstore/modules/manifest"
	"octlink/rstore/utils"
	"octlink/rstore/utils/configuration"
	"octlink/rstore/utils/merrors"
	"os"

	"github.com/spf13/cobra"
)

func init() {
	pullCmd.Flags().StringVarP(&installpath, "installpath", "i", "", "Install path like 'rstore://xxxxxx/xxxxxxxx'")
	pullCmd.Flags().StringVarP(&outpath, "outpath", "o", "./out.qcow2", "Output file path")
	pullCmd.Flags().StringVarP(&address, "address", "a", "localhost:5000", "Rstore Server Address")
}

func getBlob(name string, blobhash string) ([]byte, int, error) {

	if blobhash == config.ZeroDataDigest8M {
		// zero data no need fetch from remote
		fmt.Printf("zero data no need fetch from remote\n")
		return config.ZeroData8M, configuration.BlobSize, nil
	}

	url := fmt.Sprintf("http://%s/v1/%s/blobs/%s", address, name, blobhash)
	data, len, err := blobs.HTTPGetBlob(url)
	if err != nil {
		fmt.Printf("got blob from url %s error\n", url)
		return nil, 0, err
	}

	return data, len, nil
}

func pullImage() int {

	name, digest := manifest.ParseInstallPath(installpath)
	if name == "" || digest == "" {
		fmt.Printf("name and digest must be specified\n")
		return merrors.ErrBadParas
	}

	url := fmt.Sprintf("http://%s/v1/%s/manifests/%s", address, name, digest)
	manifest, err := manifest.HTTPGetManifest(url)
	if err != nil {
		fmt.Printf("get manifest from url %s error, manifest may not exist\n", url)
		return merrors.ErrSegmentNotExist
	}

	fmt.Printf("got manifest %s\n", utils.JSON2String(manifest))

	url = fmt.Sprintf("http://%s/v1/%s/blobsmanifest/%s", address, name, manifest.BlobSum)
	blobs, err := blobsmanifest.HTTPGetBlobsManifest(url)
	if err != nil {
		fmt.Printf("got blobsmanifest error from url %s\n", url)
		return merrors.ErrUserNotExist
	}
	fmt.Printf("got blobsmanifest %s\n", utils.JSON2String(blobs))

	fd, err := os.Create(outpath)
	if err != nil {
		fmt.Printf("create output file %s error\n", outpath)
		return merrors.ErrCmdErr
	}

	defer fd.Close()

	// Start to downloan blobs
	var total int
	for _, blob := range blobs.Chunks {
		data, len, err := getBlob(name, blob)
		if err != nil {
			fmt.Printf("got blob of %s error\n", blob)
			fmt.Printf("Pull Image of %s error\n", installpath)
			return merrors.ErrSyscallErr
		}
		fd.Write(data)
		total += len
	}

	fmt.Printf("pull image of %s to %s length[%d] OK\n", installpath, outpath, total)

	return merrors.ErrSuccess
}

var pullCmd = &cobra.Command{

	Use:   "pull",
	Short: "Pull image from rstore.",

	Run: func(cmd *cobra.Command, args []string) {

		fmt.Printf("running in pull service\n")

		if installpath != "" {
			os.Exit(pullImage())
		}

		cmd.Usage()

		os.Exit(merrors.ErrBadParas)
	},
}
<commit_msg>for image size checking<commit_after>package clis

import (
	"fmt"
	"octlink/rstore/modules/blobs"
	"octlink/rstore/modules/blobsmanifest"
	"octlink/rstore/modules/config"
	"octlink/rstore/modules/manifest"
	"octlink/rstore/utils"
	"octlink/rstore/utils/configuration"
	"octlink/rstore/utils/merrors"
	"os"

	"github.com/spf13/cobra"
)

func init() {
	pullCmd.Flags().StringVarP(&installpath, "installpath", "i", "", "Install path like 'rstore://xxxxxx/xxxxxxxx'")
	pullCmd.Flags().StringVarP(&outpath, "outpath", "o", "./out.qcow2", "Output file path")
	pullCmd.Flags().StringVarP(&address, "address", "a", "localhost:5000", "Rstore Server Address")
}

func getBlob(name string, blobhash string) ([]byte, int, error) {

	if blobhash == config.ZeroDataDigest8M {
		// zero data no need fetch from remote
		fmt.Printf("zero data no need fetch from remote\n")
		return config.ZeroData8M, configuration.BlobSize, nil
	}

	url := fmt.Sprintf("http://%s/v1/%s/blobs/%s", address, name, blobhash)
	data, len, err := blobs.HTTPGetBlob(url)
	if err != nil {
		fmt.Printf("got blob from url %s error\n", url)
		return nil, 0, err
	}

	return data, len, nil
}

func pullImage() int {

	name, digest := manifest.ParseInstallPath(installpath)
	if name == "" || digest == "" {
		fmt.Printf("name and digest must be specified\n")
		return merrors.ErrBadParas
	}

	url := fmt.Sprintf("http://%s/v1/%s/manifests/%s", address, name, digest)
	manifest, err := manifest.HTTPGetManifest(url)
	if err != nil {
		fmt.Printf("get manifest from url %s error, manifest may not exist\n", url)
		return merrors.ErrSegmentNotExist
	}

	fmt.Printf("got manifest %s\n", utils.JSON2String(manifest))

	url = fmt.Sprintf("http://%s/v1/%s/blobsmanifest/%s", address, name, manifest.BlobSum)
	blobs, err := blobsmanifest.HTTPGetBlobsManifest(url)
	if err != nil {
		fmt.Printf("got blobsmanifest error from url %s\n", url)
		return merrors.ErrUserNotExist
	}
	fmt.Printf("got blobsmanifest %s\n", utils.JSON2String(blobs))

	fd, err := os.Create(outpath)
	if err != nil {
		fmt.Printf("create output file %s error\n", outpath)
		return merrors.ErrCmdErr
	}

	defer fd.Close()

	// Start to downloan blobs
	var total int64
	for _, blob := range blobs.Chunks {
		data, len, err := getBlob(name, blob)
		if err != nil {
			fmt.Printf("got blob of %s error\n", blob)
			fmt.Printf("Pull Image of %s error\n", installpath)
			return merrors.ErrSyscallErr
		}
		fd.Write(data)
		total += len
	}

	if total != blobs.Size {
		fmt.Printf("size fetched %lld not match size %lld\n", total, blobs.Size)
		return merrors.ErrCommonErr
	}

	fmt.Printf("pull image of %s to %s length[%d] OK\n", installpath, outpath, total)

	return merrors.ErrSuccess
}

var pullCmd = &cobra.Command{

	Use:   "pull",
	Short: "Pull image from rstore.",

	Run: func(cmd *cobra.Command, args []string) {

		fmt.Printf("running in pull service\n")

		if installpath != "" {
			os.Exit(pullImage())
		}

		cmd.Usage()

		os.Exit(merrors.ErrBadParas)
	},
}
<|endoftext|>