<commit_before>package api

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"strconv"
	"sync"
	"time"

	"github.com/grafana/metrictank/api/middleware"
	"github.com/grafana/metrictank/api/models"
	"github.com/grafana/metrictank/api/response"
	"github.com/grafana/metrictank/cluster"
	"github.com/grafana/metrictank/stats"
	"github.com/raintank/worldping-api/pkg/log"
	"github.com/tinylib/msgp/msgp"
)

var NotFoundErr = errors.New("not found")

var (

	// metric api.cluster.speculative.attempts is how many peer queries resulted in speculation
	speculativeAttempts = stats.NewCounter32("api.cluster.speculative.attempts")

	// metric api.cluster.speculative.wins is how many peer queries were improved due to speculation
	speculativeWins = stats.NewCounter32("api.cluster.speculative.wins")

	// metric api.cluster.speculative.requests is how many speculative http requests made to peers
	speculativeRequests = stats.NewCounter32("api.cluster.speculative.requests")
)

func (s *Server) explainPriority(ctx *middleware.Context) {
	var data []interface{}
	for _, p := range s.prioritySetters {
		data = append(data, p.ExplainPriority())
	}
	response.Write(ctx, response.NewJson(200, data, ""))
}

func (s *Server) getNodeStatus(ctx *middleware.Context) {
	response.Write(ctx, response.NewJson(200, cluster.Manager.ThisNode(), ""))
}

func (s *Server) setNodeStatus(ctx *middleware.Context, status models.NodeStatus) {
	primary, err := strconv.ParseBool(status.Primary)
	if err != nil {
		response.Write(ctx, response.NewError(http.StatusBadRequest, fmt.Sprintf(
			"could not parse status to bool. %s",
			err.Error())),
		)
		return
	}
	cluster.Manager.SetPrimary(primary)
	ctx.PlainText(200, []byte("OK"))
}

func (s *Server) appStatus(ctx *middleware.Context) {
	if cluster.Manager.IsReady() {
		ctx.PlainText(200, []byte("OK"))
		return
	}

	response.Write(ctx, response.NewError(http.StatusServiceUnavailable, "node not ready"))
}

func (s *Server) getClusterStatus(ctx *middleware.Context) {
	status := models.ClusterStatus{
		ClusterName: cluster.ClusterName,
		NodeName:    cluster.Manager.ThisNode().GetName(),
		Members:     cluster.Manager.MemberList(),
	}
	response.Write(ctx, response.NewJson(200, status, ""))
}

func (s *Server) postClusterMembers(ctx *middleware.Context, req models.ClusterMembers) {
	memberNames := make(map[string]struct{})
	var toJoin []string

	for _, memberNode := range cluster.Manager.MemberList() {
		memberNames[memberNode.GetName()] = struct{}{}
	}

	for _, peerName := range req.Members {
		if _, ok := memberNames[peerName]; !ok {
			toJoin = append(toJoin, peerName)
		}
	}

	resp := models.ClusterMembersResp{
		Status:       "ok",
		MembersAdded: 0,
	}

	if len(toJoin) == 0 {
		response.Write(ctx, response.NewJson(200, resp, ""))
		return
	}

	n, err := cluster.Manager.Join(toJoin)
	if err != nil {
		response.Write(ctx, response.NewError(http.StatusBadRequest, fmt.Sprintf(
			"error when joining cluster members: %s", err.Error())),
		)
		return
	}
	resp.MembersAdded = n
	response.Write(ctx, response.NewJson(200, resp, ""))
}

// IndexFind returns a sequence of msgp encoded idx.Node's
func (s *Server) indexFind(ctx *middleware.Context, req models.IndexFind) {
	resp := models.NewIndexFindResp()

	for _, pattern := range req.Patterns {
		nodes, err := s.MetricIndex.Find(req.OrgId, pattern, req.From)
		if err != nil {
			response.Write(ctx, response.WrapError(err))
			return
		}
		resp.Nodes[pattern] = nodes
	}
	response.Write(ctx, response.NewMsgp(200, resp))
}

func (s *Server) indexTagDetails(ctx *middleware.Context, req models.IndexTagDetails) {
	values, err := s.MetricIndex.TagDetails(req.OrgId, req.Tag, req.Filter, req.From)
	if err != nil {
		response.Write(ctx, response.NewError(http.StatusBadRequest, err.Error()))
		return
	}
	response.Write(ctx, response.NewMsgp(200, &models.IndexTagDetailsResp{Values: values}))
}

func (s *Server) indexTags(ctx *middleware.Context, req models.IndexTags) {
	tags, err := s.MetricIndex.Tags(req.OrgId, req.Filter, req.From)
	if err != nil {
		response.Write(ctx, response.NewError(http.StatusBadRequest, err.Error()))
		return
	}
	response.Write(ctx, response.NewMsgp(200, &models.IndexTagsResp{Tags: tags}))
}

func (s *Server) indexAutoCompleteTags(ctx *middleware.Context, req models.IndexAutoCompleteTags) {
	tags, err := s.MetricIndex.FindTags(req.OrgId, req.Prefix, req.Expr, req.From, req.Limit)
	if err != nil {
		response.Write(ctx, response.NewError(http.StatusBadRequest, err.Error()))
		return
	}
	response.Write(ctx, response.NewMsgp(200, models.StringList(tags)))
}

func (s *Server) indexAutoCompleteTagValues(ctx *middleware.Context, req models.IndexAutoCompleteTagValues) {
	tags, err := s.MetricIndex.FindTagValues(req.OrgId, req.Tag, req.Prefix, req.Expr, req.From, req.Limit)
	if err != nil {
		response.Write(ctx, response.NewError(http.StatusBadRequest, err.Error()))
		return
	}
	response.Write(ctx, response.NewMsgp(200, models.StringList(tags)))
}

func (s *Server) indexTagDelSeries(ctx *middleware.Context, request models.IndexTagDelSeries) {
	deleted, err := s.MetricIndex.DeleteTagged(request.OrgId, request.Paths)
	if err != nil {
		response.Write(ctx, response.WrapErrorForTagDB(err))
		return
	}

	res := models.IndexTagDelSeriesResp{}
	res.Count = len(deleted)

	response.Write(ctx, response.NewMsgp(200, res))
}

func (s *Server) indexFindByTag(ctx *middleware.Context, req models.IndexFindByTag) {
	metrics, err := s.MetricIndex.FindByTag(req.OrgId, req.Expr, req.From)
	if err != nil {
		response.Write(ctx, response.NewError(http.StatusBadRequest, err.Error()))
		return
	}
	response.Write(ctx, response.NewMsgp(200, &models.IndexFindByTagResp{Metrics: metrics}))
}

// IndexGet returns a msgp encoded schema.MetricDefinition
func (s *Server) indexGet(ctx *middleware.Context, req models.IndexGet) {
	def, ok := s.MetricIndex.Get(req.MKey)
	if !ok {
		response.Write(ctx, response.NewError(http.StatusNotFound, "Not Found"))
		return
	}

	response.Write(ctx, response.NewMsgp(200, &def))
}

// IndexList returns msgp encoded schema.MetricDefinition's
func (s *Server) indexList(ctx *middleware.Context, req models.IndexList) {
	defs := s.MetricIndex.List(req.OrgId)
	resp := make([]msgp.Marshaler, len(defs))
	for i := range defs {
		d := defs[i]
		resp[i] = &d
	}
	response.Write(ctx, response.NewMsgpArray(200, resp))
}

func (s *Server) getData(ctx *middleware.Context, request models.GetData) {
	series, err := s.getTargetsLocal(ctx.Req.Context(), request.Requests)
	if err != nil {
		// the only errors returned are from us catching panics, so we should treat them
		// all as internalServerErrors
		log.Error(3, "HTTP getData() %s", err.Error())
		response.Write(ctx, response.WrapError(err))
		return
	}
	response.Write(ctx, response.NewMsgp(200, &models.GetDataResp{Series: series}))
}

func (s *Server) indexDelete(ctx *middleware.Context, req models.IndexDelete) {
	defs, err := s.MetricIndex.Delete(req.OrgId, req.Query)
	if err != nil {
		// errors can only be caused by bad request.
		response.Write(ctx, response.NewError(http.StatusBadRequest, err.Error()))
		return
	}

	resp := models.MetricsDeleteResp{
		DeletedDefs: len(defs),
	}
	response.Write(ctx, response.NewMsgp(200, &resp))
}

type PeerResponse struct {
	peer cluster.Node
	buf  []byte
}

// peerQuery takes a request and the path to request it on, then fans it out
// across the cluster, except to the local peer. If any peer fails requests to
// other peers are aborted.
// ctx:          request context
// data:         request to be submitted
// name:         name to be used in logging & tracing
// path:         path to request on
func (s *Server) peerQuery(ctx context.Context, data cluster.Traceable, name, path string, allPeers bool) (map[string]PeerResponse, error) {
	var peers []cluster.Node
	var err error

	if allPeers {
		peers = cluster.Manager.MemberList()
	} else {
		peers, err = cluster.MembersForQuery()
		if err != nil {
			log.Error(3, "HTTP peerQuery unable to get peers, %s", err)
			return nil, err
		}
	}
	log.Debug("HTTP %s across %d instances", name, len(peers)-1)

	reqCtx, cancel := context.WithCancel(ctx)
	defer cancel()

	responses := make(chan struct {
		data PeerResponse
		err  error
	}, 1)
	var wg sync.WaitGroup
	for _, peer := range peers {
		if peer.IsLocal() {
			continue
		}
		wg.Add(1)
		go func(peer cluster.Node) {
			defer wg.Done()
			log.Debug("HTTP Render querying %s%s", peer.GetName(), path)
			buf, err := peer.Post(reqCtx, name, path, data)
			if err != nil {
				cancel()
				log.Error(4, "HTTP Render error querying %s%s: %q", peer.GetName(), path, err)
			}
			responses <- struct {
				data PeerResponse
				err  error
			}{PeerResponse{peer, buf}, err}
		}(peer)
	}
	// wait for all list goroutines to end, then close our responses channel
	go func() {
		wg.Wait()
		close(responses)
	}()

	result := make(map[string]PeerResponse)
	for resp := range responses {
		if resp.err != nil {
			return nil, err
		}
		result[resp.data.peer.GetName()] = resp.data
	}

	return result, nil
}

// peerQuerySpeculative takes a request and the path to request it on, then fans it out
// across the cluster, except to the local peer. If any peer fails requests to
// other peers are aborted. If enough peers have been heard from (based on
// speculation-threshold configuration), and we are missing the others, try to
// speculatively query each other member of each shard group.
// ctx:          request context
// data:         request to be submitted
// name:         name to be used in logging & tracing
// path:         path to request on
func (s *Server) peerQuerySpeculative(ctx context.Context, data cluster.Traceable, name, path string) (map[string]PeerResponse, error) {
	peerGroups, err := cluster.MembersForSpeculativeQuery()
	if err != nil {
		log.Error(3, "HTTP peerQuery unable to get peers, %s", err)
		return nil, err
	}
	log.Debug("HTTP %s across %d instances", name, len(peerGroups)-1)

	reqCtx, cancel := context.WithCancel(ctx)
	defer cancel()

	originalPeers := make(map[string]struct{}, len(peerGroups))
	pendingResponses := make(map[int32]struct{}, len(peerGroups))
	receivedResponses := make(map[int32]struct{}, len(peerGroups))

	responses := make(chan struct {
		shardGroup int32
		data       PeerResponse
		err        error
	}, 1)

	askPeer := func(shardGroup int32, peer cluster.Node) {
		log.Debug("HTTP Render querying %s%s", peer.GetName(), path)
		buf, err := peer.Post(reqCtx, name, path, data)

		select {
		case <-ctx.Done():
			return
		default:
			// Not canceled, continue
		}

		if err != nil {
			cancel()
			log.Error(4, "HTTP Render error querying %s%s: %q", peer.GetName(), path, err)
		}
		responses <- struct {
			shardGroup int32
			data       PeerResponse
			err        error
		}{shardGroup, PeerResponse{peer, buf}, err}
	}

	for group, peers := range peerGroups {
		peer := peers[0]
		originalPeers[peer.GetName()] = struct{}{}
		pendingResponses[group] = struct{}{}
		go askPeer(group, peer)
	}

	result := make(map[string]PeerResponse)

	specCheckTicker := time.NewTicker(5 * time.Millisecond)

	for len(pendingResponses) > 0 {
		select {
		case resp := <-responses:
			if _, ok := receivedResponses[resp.shardGroup]; ok {
				// already received this response (possibly speculatively)
				continue
			}

			if resp.err != nil {
				return nil, err
			}

			result[resp.data.peer.GetName()] = resp.data
			receivedResponses[resp.shardGroup] = struct{}{}
			delete(pendingResponses, resp.shardGroup)
			delete(originalPeers, resp.data.peer.GetName())

		case <-specCheckTicker.C:
			// Check if it's time to speculate!
			percentReceived := 1 - (float64(len(pendingResponses)) / float64(len(peerGroups)))
			if percentReceived > speculationThreshold {
				// kick off speculative queries to other members now
				specCheckTicker.Stop()
				speculativeAttempts.Inc()
				for shardGroup := range pendingResponses {
					eligiblePeers := peerGroups[shardGroup][1:]
					for _, peer := range eligiblePeers {
						speculativeRequests.Inc()
						go askPeer(shardGroup, peer)
					}
				}
			}
		}
	}

	if len(originalPeers) > 0 {
		speculativeWins.Inc()
	}

	return result, nil
}
<commit_msg>implement speculation-threshold by its spec<commit_after>package api

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"strconv"
	"sync"
	"time"

	"github.com/grafana/metrictank/api/middleware"
	"github.com/grafana/metrictank/api/models"
	"github.com/grafana/metrictank/api/response"
	"github.com/grafana/metrictank/cluster"
	"github.com/grafana/metrictank/stats"
	"github.com/raintank/worldping-api/pkg/log"
	"github.com/tinylib/msgp/msgp"
)

var NotFoundErr = errors.New("not found")

var (

	// metric api.cluster.speculative.attempts is how many peer queries resulted in speculation
	speculativeAttempts = stats.NewCounter32("api.cluster.speculative.attempts")

	// metric api.cluster.speculative.wins is how many peer queries were improved due to speculation
	speculativeWins = stats.NewCounter32("api.cluster.speculative.wins")

	// metric api.cluster.speculative.requests is how many speculative http requests made to peers
	speculativeRequests = stats.NewCounter32("api.cluster.speculative.requests")
)

func (s *Server) explainPriority(ctx *middleware.Context) {
	var data []interface{}
	for _, p := range s.prioritySetters {
		data = append(data, p.ExplainPriority())
	}
	response.Write(ctx, response.NewJson(200, data, ""))
}

func (s *Server) getNodeStatus(ctx *middleware.Context) {
	response.Write(ctx, response.NewJson(200, cluster.Manager.ThisNode(), ""))
}

func (s *Server) setNodeStatus(ctx *middleware.Context, status models.NodeStatus) {
	primary, err := strconv.ParseBool(status.Primary)
	if err != nil {
		response.Write(ctx, response.NewError(http.StatusBadRequest, fmt.Sprintf(
			"could not parse status to bool. %s",
			err.Error())),
		)
		return
	}
	cluster.Manager.SetPrimary(primary)
	ctx.PlainText(200, []byte("OK"))
}

func (s *Server) appStatus(ctx *middleware.Context) {
	if cluster.Manager.IsReady() {
		ctx.PlainText(200, []byte("OK"))
		return
	}

	response.Write(ctx, response.NewError(http.StatusServiceUnavailable, "node not ready"))
}

func (s *Server) getClusterStatus(ctx *middleware.Context) {
	status := models.ClusterStatus{
		ClusterName: cluster.ClusterName,
		NodeName:    cluster.Manager.ThisNode().GetName(),
		Members:     cluster.Manager.MemberList(),
	}
	response.Write(ctx, response.NewJson(200, status, ""))
}

func (s *Server) postClusterMembers(ctx *middleware.Context, req models.ClusterMembers) {
	memberNames := make(map[string]struct{})
	var toJoin []string

	for _, memberNode := range cluster.Manager.MemberList() {
		memberNames[memberNode.GetName()] = struct{}{}
	}

	for _, peerName := range req.Members {
		if _, ok := memberNames[peerName]; !ok {
			toJoin = append(toJoin, peerName)
		}
	}

	resp := models.ClusterMembersResp{
		Status:       "ok",
		MembersAdded: 0,
	}

	if len(toJoin) == 0 {
		response.Write(ctx, response.NewJson(200, resp, ""))
		return
	}

	n, err := cluster.Manager.Join(toJoin)
	if err != nil {
		response.Write(ctx, response.NewError(http.StatusBadRequest, fmt.Sprintf(
			"error when joining cluster members: %s", err.Error())),
		)
		return
	}
	resp.MembersAdded = n
	response.Write(ctx, response.NewJson(200, resp, ""))
}

// IndexFind returns a sequence of msgp encoded idx.Node's
func (s *Server) indexFind(ctx *middleware.Context, req models.IndexFind) {
	resp := models.NewIndexFindResp()

	for _, pattern := range req.Patterns {
		nodes, err := s.MetricIndex.Find(req.OrgId, pattern, req.From)
		if err != nil {
			response.Write(ctx, response.WrapError(err))
			return
		}
		resp.Nodes[pattern] = nodes
	}
	response.Write(ctx, response.NewMsgp(200, resp))
}

func (s *Server) indexTagDetails(ctx *middleware.Context, req models.IndexTagDetails) {
	values, err := s.MetricIndex.TagDetails(req.OrgId, req.Tag, req.Filter, req.From)
	if err != nil {
		response.Write(ctx, response.NewError(http.StatusBadRequest, err.Error()))
		return
	}
	response.Write(ctx, response.NewMsgp(200, &models.IndexTagDetailsResp{Values: values}))
}

func (s *Server) indexTags(ctx *middleware.Context, req models.IndexTags) {
	tags, err := s.MetricIndex.Tags(req.OrgId, req.Filter, req.From)
	if err != nil {
		response.Write(ctx, response.NewError(http.StatusBadRequest, err.Error()))
		return
	}
	response.Write(ctx, response.NewMsgp(200, &models.IndexTagsResp{Tags: tags}))
}

func (s *Server) indexAutoCompleteTags(ctx *middleware.Context, req models.IndexAutoCompleteTags) {
	tags, err := s.MetricIndex.FindTags(req.OrgId, req.Prefix, req.Expr, req.From, req.Limit)
	if err != nil {
		response.Write(ctx, response.NewError(http.StatusBadRequest, err.Error()))
		return
	}
	response.Write(ctx, response.NewMsgp(200, models.StringList(tags)))
}

func (s *Server) indexAutoCompleteTagValues(ctx *middleware.Context, req models.IndexAutoCompleteTagValues) {
	tags, err := s.MetricIndex.FindTagValues(req.OrgId, req.Tag, req.Prefix, req.Expr, req.From, req.Limit)
	if err != nil {
		response.Write(ctx, response.NewError(http.StatusBadRequest, err.Error()))
		return
	}
	response.Write(ctx, response.NewMsgp(200, models.StringList(tags)))
}

func (s *Server) indexTagDelSeries(ctx *middleware.Context, request models.IndexTagDelSeries) {
	deleted, err := s.MetricIndex.DeleteTagged(request.OrgId, request.Paths)
	if err != nil {
		response.Write(ctx, response.WrapErrorForTagDB(err))
		return
	}

	res := models.IndexTagDelSeriesResp{}
	res.Count = len(deleted)

	response.Write(ctx, response.NewMsgp(200, res))
}

func (s *Server) indexFindByTag(ctx *middleware.Context, req models.IndexFindByTag) {
	metrics, err := s.MetricIndex.FindByTag(req.OrgId, req.Expr, req.From)
	if err != nil {
		response.Write(ctx, response.NewError(http.StatusBadRequest, err.Error()))
		return
	}
	response.Write(ctx, response.NewMsgp(200, &models.IndexFindByTagResp{Metrics: metrics}))
}

// IndexGet returns a msgp encoded schema.MetricDefinition
func (s *Server) indexGet(ctx *middleware.Context, req models.IndexGet) {
	def, ok := s.MetricIndex.Get(req.MKey)
	if !ok {
		response.Write(ctx, response.NewError(http.StatusNotFound, "Not Found"))
		return
	}

	response.Write(ctx, response.NewMsgp(200, &def))
}

// IndexList returns msgp encoded schema.MetricDefinition's
func (s *Server) indexList(ctx *middleware.Context, req models.IndexList) {
	defs := s.MetricIndex.List(req.OrgId)
	resp := make([]msgp.Marshaler, len(defs))
	for i := range defs {
		d := defs[i]
		resp[i] = &d
	}
	response.Write(ctx, response.NewMsgpArray(200, resp))
}

func (s *Server) getData(ctx *middleware.Context, request models.GetData) {
	series, err := s.getTargetsLocal(ctx.Req.Context(), request.Requests)
	if err != nil {
		// the only errors returned are from us catching panics, so we should treat them
		// all as internalServerErrors
		log.Error(3, "HTTP getData() %s", err.Error())
		response.Write(ctx, response.WrapError(err))
		return
	}
	response.Write(ctx, response.NewMsgp(200, &models.GetDataResp{Series: series}))
}

func (s *Server) indexDelete(ctx *middleware.Context, req models.IndexDelete) {
	defs, err := s.MetricIndex.Delete(req.OrgId, req.Query)
	if err != nil {
		// errors can only be caused by bad request.
		response.Write(ctx, response.NewError(http.StatusBadRequest, err.Error()))
		return
	}

	resp := models.MetricsDeleteResp{
		DeletedDefs: len(defs),
	}
	response.Write(ctx, response.NewMsgp(200, &resp))
}

type PeerResponse struct {
	peer cluster.Node
	buf  []byte
}

// peerQuery takes a request and the path to request it on, then fans it out
// across the cluster, except to the local peer. If any peer fails requests to
// other peers are aborted.
// ctx:          request context
// data:         request to be submitted
// name:         name to be used in logging & tracing
// path:         path to request on
func (s *Server) peerQuery(ctx context.Context, data cluster.Traceable, name, path string, allPeers bool) (map[string]PeerResponse, error) {
	var peers []cluster.Node
	var err error

	if allPeers {
		peers = cluster.Manager.MemberList()
	} else {
		peers, err = cluster.MembersForQuery()
		if err != nil {
			log.Error(3, "HTTP peerQuery unable to get peers, %s", err)
			return nil, err
		}
	}
	log.Debug("HTTP %s across %d instances", name, len(peers)-1)

	reqCtx, cancel := context.WithCancel(ctx)
	defer cancel()

	responses := make(chan struct {
		data PeerResponse
		err  error
	}, 1)
	var wg sync.WaitGroup
	for _, peer := range peers {
		if peer.IsLocal() {
			continue
		}
		wg.Add(1)
		go func(peer cluster.Node) {
			defer wg.Done()
			log.Debug("HTTP Render querying %s%s", peer.GetName(), path)
			buf, err := peer.Post(reqCtx, name, path, data)
			if err != nil {
				cancel()
				log.Error(4, "HTTP Render error querying %s%s: %q", peer.GetName(), path, err)
			}
			responses <- struct {
				data PeerResponse
				err  error
			}{PeerResponse{peer, buf}, err}
		}(peer)
	}
	// wait for all list goroutines to end, then close our responses channel
	go func() {
		wg.Wait()
		close(responses)
	}()

	result := make(map[string]PeerResponse)
	for resp := range responses {
		if resp.err != nil {
			return nil, err
		}
		result[resp.data.peer.GetName()] = resp.data
	}

	return result, nil
}

// peerQuerySpeculative takes a request and the path to request it on, then fans it out
// across the cluster, except to the local peer. If any peer fails requests to
// other peers are aborted. If enough peers have been heard from (based on
// speculation-threshold configuration), and we are missing the others, try to
// speculatively query each other member of each shard group.
// ctx:          request context
// data:         request to be submitted
// name:         name to be used in logging & tracing
// path:         path to request on
func (s *Server) peerQuerySpeculative(ctx context.Context, data cluster.Traceable, name, path string) (map[string]PeerResponse, error) {
	peerGroups, err := cluster.MembersForSpeculativeQuery()
	if err != nil {
		log.Error(3, "HTTP peerQuery unable to get peers, %s", err)
		return nil, err
	}
	log.Debug("HTTP %s across %d instances", name, len(peerGroups)-1)

	reqCtx, cancel := context.WithCancel(ctx)
	defer cancel()

	originalPeers := make(map[string]struct{}, len(peerGroups))
	pendingResponses := make(map[int32]struct{}, len(peerGroups))
	receivedResponses := make(map[int32]struct{}, len(peerGroups))

	responses := make(chan struct {
		shardGroup int32
		data       PeerResponse
		err        error
	}, 1)

	askPeer := func(shardGroup int32, peer cluster.Node) {
		log.Debug("HTTP Render querying %s%s", peer.GetName(), path)
		buf, err := peer.Post(reqCtx, name, path, data)

		select {
		case <-ctx.Done():
			return
		default:
			// Not canceled, continue
		}

		if err != nil {
			cancel()
			log.Error(4, "HTTP Render error querying %s%s: %q", peer.GetName(), path, err)
		}
		responses <- struct {
			shardGroup int32
			data       PeerResponse
			err        error
		}{shardGroup, PeerResponse{peer, buf}, err}
	}

	for group, peers := range peerGroups {
		peer := peers[0]
		originalPeers[peer.GetName()] = struct{}{}
		pendingResponses[group] = struct{}{}
		go askPeer(group, peer)
	}

	result := make(map[string]PeerResponse)

	var ticker *time.Ticker
	var tickChan <-chan time.Time
	if speculationThreshold != 1 {
		ticker = time.NewTicker(5 * time.Millisecond)
		tickChan = ticker.C
	}

	for len(pendingResponses) > 0 {
		select {
		case resp := <-responses:
			if _, ok := receivedResponses[resp.shardGroup]; ok {
				// already received this response (possibly speculatively)
				continue
			}

			if resp.err != nil {
				return nil, err
			}

			result[resp.data.peer.GetName()] = resp.data
			receivedResponses[resp.shardGroup] = struct{}{}
			delete(pendingResponses, resp.shardGroup)
			delete(originalPeers, resp.data.peer.GetName())

		case <-tickChan:
			// Check if it's time to speculate!
			percentReceived := 1 - (float64(len(pendingResponses)) / float64(len(peerGroups)))
			if percentReceived >= speculationThreshold {
				// kick off speculative queries to other members now
				ticker.Stop()
				speculativeAttempts.Inc()
				for shardGroup := range pendingResponses {
					eligiblePeers := peerGroups[shardGroup][1:]
					for _, peer := range eligiblePeers {
						speculativeRequests.Inc()
						go askPeer(shardGroup, peer)
					}
				}
			}
		}
	}

	if len(originalPeers) > 0 {
		speculativeWins.Inc()
	}

	return result, nil
}
<|endoftext|>


<commit_before>package api

import (
	"net/http"

	"github.com/labstack/echo"
)

type Prefix struct {
	Background string   `json:"background"`
	Color      string   `json:"color"`
	Words      []string `json:"words"`
	TimedEvent bool     `json:"timedEvent"`
	Default    bool     `json:"default"`
}

var DefaultPrefixes = []Prefix{
	Prefix{
		Background: "4C6C9B",
		Color:      "FFFFFF",
		Words:      []string{"HW", "Read", "Reading"},
		Default:    true,
	},
	Prefix{
		Background: "9ACD32",
		Color:      "FFFFFF",
		Words:      []string{"Project"},
		Default:    true,
	},
	Prefix{
		Background: "FFD700",
		Color:      "FFFFFF",
		Words:      []string{"Report", "Essay", "Paper"},
		Default:    true,
	},
	Prefix{
		Background: "FFA500",
		Color:      "FFFFFF",
		Words:      []string{"Quiz", "PopQuiz"},
		Default:    true,
	},
	Prefix{
		Background: "DC143C",
		Color:      "FFFFFF",
		Words:      []string{"Test", "Final", "Exam", "Midterm"},
		Default:    true,
	},
	Prefix{
		Background: "2AC0F1",
		Color:      "FFFFFF",
		Words:      []string{"ICA"},
		Default:    true,
	},
	Prefix{
		Background: "2AF15E",
		Color:      "FFFFFF",
		Words:      []string{"Lab", "Study", "Memorize"},
		TimedEvent: true,
		Default:    true,
	},
	Prefix{
		Background: "003DAD",
		Color:      "FFFFFF",
		Words:      []string{"DocID"},
		Default:    true,
	},
	Prefix{
		Background: "000000",
		Color:      "00FF00",
		Words:      []string{"Trojun", "Hex"},
		Default:    true,
	},
	Prefix{
		Background: "FCF8E3",
		Color:      "000000",
		Words:      []string{"Trojun", "Hex"},
		Default:    true,
	},
	Prefix{
		Background: "5000BC",
		Color:      "FFFFFF",
		Words:      []string{"OptionalHW", "Challenge"},
		Default:    true,
	},
	Prefix{
		Background: "000099",
		Color:      "FFFFFF",
		Words:      []string{"Presentation", "Prez"},
		Default:    true,
	},
	Prefix{
		Background: "123456",
		Color:      "FFFFFF",
		Words:      []string{"BuildSession", "Build"},
		TimedEvent: true,
		Default:    true,
	},
	Prefix{
		Background: "5A1B87",
		Color:      "FFFFFF",
		Words:      []string{"Meeting", "Meet"},
		TimedEvent: true,
		Default:    true,
	},
}

type PrefixesResponse struct {
	Status             string   `json:"status"`
	Prefixes           []Prefix `json:"prefixes"`
	FallbackBackground string   `json:"fallbackBackground"`
	FallbackColor      string   `json:"fallbackColor"`
}

func InitPrefixesAPI(e *echo.Echo) {
	e.GET("/prefixes/getList", func(c echo.Context) error {
		if GetSessionUserID(&c) == -1 {
			return c.JSON(http.StatusUnauthorized, ErrorResponse{"error", "logged_out"})
		}

		return c.JSON(http.StatusOK, PrefixesResponse{"ok", DefaultPrefixes, "FFD3BD", "000000"})
	})
}
<commit_msg>remove duplicate<commit_after>package api

import (
	"net/http"

	"github.com/labstack/echo"
)

type Prefix struct {
	Background string   `json:"background"`
	Color      string   `json:"color"`
	Words      []string `json:"words"`
	TimedEvent bool     `json:"timedEvent"`
	Default    bool     `json:"default"`
}

var DefaultPrefixes = []Prefix{
	Prefix{
		Background: "4C6C9B",
		Color:      "FFFFFF",
		Words:      []string{"HW", "Read", "Reading"},
		Default:    true,
	},
	Prefix{
		Background: "9ACD32",
		Color:      "FFFFFF",
		Words:      []string{"Project"},
		Default:    true,
	},
	Prefix{
		Background: "FFD700",
		Color:      "FFFFFF",
		Words:      []string{"Report", "Essay", "Paper"},
		Default:    true,
	},
	Prefix{
		Background: "FFA500",
		Color:      "FFFFFF",
		Words:      []string{"Quiz", "PopQuiz"},
		Default:    true,
	},
	Prefix{
		Background: "DC143C",
		Color:      "FFFFFF",
		Words:      []string{"Test", "Final", "Exam", "Midterm"},
		Default:    true,
	},
	Prefix{
		Background: "2AC0F1",
		Color:      "FFFFFF",
		Words:      []string{"ICA"},
		Default:    true,
	},
	Prefix{
		Background: "2AF15E",
		Color:      "FFFFFF",
		Words:      []string{"Lab", "Study", "Memorize"},
		TimedEvent: true,
		Default:    true,
	},
	Prefix{
		Background: "003DAD",
		Color:      "FFFFFF",
		Words:      []string{"DocID"},
		Default:    true,
	},
	Prefix{
		Background: "000000",
		Color:      "00FF00",
		Words:      []string{"Trojun", "Hex"},
		Default:    true,
	},
	Prefix{
		Background: "5000BC",
		Color:      "FFFFFF",
		Words:      []string{"OptionalHW", "Challenge"},
		Default:    true,
	},
	Prefix{
		Background: "000099",
		Color:      "FFFFFF",
		Words:      []string{"Presentation", "Prez"},
		Default:    true,
	},
	Prefix{
		Background: "123456",
		Color:      "FFFFFF",
		Words:      []string{"BuildSession", "Build"},
		TimedEvent: true,
		Default:    true,
	},
	Prefix{
		Background: "5A1B87",
		Color:      "FFFFFF",
		Words:      []string{"Meeting", "Meet"},
		TimedEvent: true,
		Default:    true,
	},
}

type PrefixesResponse struct {
	Status             string   `json:"status"`
	Prefixes           []Prefix `json:"prefixes"`
	FallbackBackground string   `json:"fallbackBackground"`
	FallbackColor      string   `json:"fallbackColor"`
}

func InitPrefixesAPI(e *echo.Echo) {
	e.GET("/prefixes/getList", func(c echo.Context) error {
		if GetSessionUserID(&c) == -1 {
			return c.JSON(http.StatusUnauthorized, ErrorResponse{"error", "logged_out"})
		}

		return c.JSON(http.StatusOK, PrefixesResponse{"ok", DefaultPrefixes, "FFD3BD", "000000"})
	})
}
<|endoftext|>


<commit_before>package authy

import (
	"fmt"
	"log"
	"net/url"
	"time"

	"golang.org/x/crypto/ssh"

	"github.com/dcu/go-authy"
)

type authyClient struct {
	Config struct {
		APIKey string `long:"challenger-authy-apikey" description:"Authy API Key" env:"SSHPIPERD_CHALLENGER_AUTHY_APIKEY" ini-name:"challenger-authy-apikey"`
		Method string `long:"challenger-authy-method" default:"token" description:"Authy authentication method" env:"SSHPIPERD_CHALLENGER_AUTHY_METHOD" ini-name:"challenger-authy-method" choice:"token" choice:"onetouch"`
		File   string `long:"challenger-authy-idfile" description:"Path to a file with ssh_name [space] authy_id per line (first line win if duplicate)" env:"SSHPIPERD_CHALLENGER_AUTHY_IDFILE" ini-name:"challenger-authy-idfile"`
	}

	authyAPI *authy.Authy
	logger   *log.Logger
}

func (a *authyClient) Init(logger *log.Logger) error {
	a.logger = logger
	a.authyAPI = authy.NewAuthyAPI(a.Config.APIKey)

	return nil
}

func (a *authyClient) challenge(conn ssh.ConnMetadata, client ssh.KeyboardInteractiveChallenge) (ssh.AdditionalChallengeContext, error) {
	user := conn.User()

	authyID, err := a.findAuthyID(user)

	if err != nil {
		return nil, err
	}

	switch a.Config.Method {
	case "token":

		ans, err := client(user, "", []string{"Please input your Authy token: "}, []bool{true})
		if err != nil {
			return nil, err
		}

		verification, err := a.authyAPI.VerifyToken(authyID, ans[0], url.Values{})
		if err != nil {
			return nil, err
		}

		if verification.Valid() {
			return nil, nil
		}

		return nil, fmt.Errorf(verification.Message)
	case "onetouch":
		_, err = client(conn.User(), "Please verify login on your Authy app", nil, nil)
		if err != nil {
			return nil, err
		}

		details := authy.Details{
			"User":     user,
			"ClientIP": conn.RemoteAddr().String(),
		}

		approvalRequest, err := a.authyAPI.SendApprovalRequest(authyID, "Log to SSH server", details, url.Values{})
		if err != nil {
			return nil, err
		}

		status, err := a.authyAPI.WaitForApprovalRequest(approvalRequest.UUID, time.Second*30, url.Values{})
		if err != nil {
			return nil, err
		}

		if status == authy.OneTouchStatusApproved {
			return nil, nil
		}

		_, err = client(conn.User(), "Authy OneTouch failed", nil, nil)
		if err != nil {
			return nil, err
		}

		return nil, fmt.Errorf("one touch faild code: %v", status)

	default:
		return nil, fmt.Errorf("unsupported authy method")
	}
}
<commit_msg>echo error if authy token failed<commit_after>package authy

import (
	"fmt"
	"log"
	"net/url"
	"time"

	"golang.org/x/crypto/ssh"

	"github.com/dcu/go-authy"
)

type authyClient struct {
	Config struct {
		APIKey string `long:"challenger-authy-apikey" description:"Authy API Key" env:"SSHPIPERD_CHALLENGER_AUTHY_APIKEY" ini-name:"challenger-authy-apikey"`
		Method string `long:"challenger-authy-method" default:"token" description:"Authy authentication method" env:"SSHPIPERD_CHALLENGER_AUTHY_METHOD" ini-name:"challenger-authy-method" choice:"token" choice:"onetouch"`
		File   string `long:"challenger-authy-idfile" description:"Path to a file with ssh_name [space] authy_id per line (first line win if duplicate)" env:"SSHPIPERD_CHALLENGER_AUTHY_IDFILE" ini-name:"challenger-authy-idfile"`
	}

	authyAPI *authy.Authy
	logger   *log.Logger
}

func (a *authyClient) Init(logger *log.Logger) error {
	a.logger = logger
	a.authyAPI = authy.NewAuthyAPI(a.Config.APIKey)

	return nil
}

func (a *authyClient) challenge(conn ssh.ConnMetadata, client ssh.KeyboardInteractiveChallenge) (ssh.AdditionalChallengeContext, error) {
	user := conn.User()

	authyID, err := a.findAuthyID(user)

	if err != nil {
		return nil, err
	}

	switch a.Config.Method {
	case "token":

		ans, err := client(user, "", []string{"Please input your Authy token: "}, []bool{true})
		if err != nil {
			return nil, err
		}

		verification, err := a.authyAPI.VerifyToken(authyID, ans[0], url.Values{})
		if err != nil {
			return nil, err
		}

		if verification.Valid() {
			return nil, nil
		}

		_, err = client(conn.User(), verification.Message, nil, nil)
		if err != nil {
			return nil, err
		}

		return nil, fmt.Errorf("failed to auth with authy: %v", verification.Message)

	case "onetouch":
		_, err = client(conn.User(), "Please verify login on your Authy app", nil, nil)
		if err != nil {
			return nil, err
		}

		details := authy.Details{
			"User":     user,
			"ClientIP": conn.RemoteAddr().String(),
		}

		approvalRequest, err := a.authyAPI.SendApprovalRequest(authyID, "Log to SSH server", details, url.Values{})
		if err != nil {
			return nil, err
		}

		status, err := a.authyAPI.WaitForApprovalRequest(approvalRequest.UUID, time.Second*30, url.Values{})
		if err != nil {
			return nil, err
		}

		if status == authy.OneTouchStatusApproved {
			return nil, nil
		}

		_, err = client(conn.User(), "Authy OneTouch failed", nil, nil)
		if err != nil {
			return nil, err
		}

		return nil, fmt.Errorf("one touch faild code: %v", status)

	default:
		return nil, fmt.Errorf("unsupported authy method")
	}
}
<|endoftext|>


<commit_before>package database

import (
	"github.com/jinzhu/gorm"
	_ "github.com/jinzhu/gorm/dialects/sqlite" // gorm dialect

	upstreamprovider "github.com/tg123/sshpiper/sshpiperd/upstream"
)

type sqliteplugin struct {
	plugin

	Config struct {
		File string `long:"sqlite-dbfile" default:"file:sshpiper.sqlite" description:"databasefile for sqlite" env:"SSHPIPERD_UPSTREAM_SQLITE_FILE" ini-name:"upstream-sqlite-file"`
	}
}

func (p *sqliteplugin) create() (*gorm.DB, error) {

	db, err := gorm.Open("sqlite3", p.Config.File)
	if err != nil {
		return nil, err
	}

	return db, nil
}

func (sqliteplugin) GetName() string {
	return "sqlite"
}

func (p *sqliteplugin) GetOpts() interface{} {
	return &p.Config
}

func init() {
	p := &sqliteplugin{}
	p.createdb = p
	upstreamprovider.Register("sqlite", p)
}
<commit_msg>unify opt for sqlite<commit_after>package database

import (
	"github.com/jinzhu/gorm"
	_ "github.com/jinzhu/gorm/dialects/sqlite" // gorm dialect

	upstreamprovider "github.com/tg123/sshpiper/sshpiperd/upstream"
)

type sqliteplugin struct {
	plugin

	Config struct {
		File string `long:"upstream-sqlite-dbfile" default:"file:sshpiper.sqlite" description:"databasefile for sqlite" env:"SSHPIPERD_UPSTREAM_SQLITE_FILE" ini-name:"upstream-sqlite-file"`
	}
}

func (p *sqliteplugin) create() (*gorm.DB, error) {

	db, err := gorm.Open("sqlite3", p.Config.File)
	if err != nil {
		return nil, err
	}

	return db, nil
}

func (sqliteplugin) GetName() string {
	return "sqlite"
}

func (p *sqliteplugin) GetOpts() interface{} {
	return &p.Config
}

func init() {
	p := &sqliteplugin{}
	p.createdb = p
	upstreamprovider.Register("sqlite", p)
}
<|endoftext|>


<commit_before>//Package ngorm is a Go Object relation mapper that focus on performance,
//maintainability, modularity,	battle testing, extensibility , safety and
//developer frindliness.
//
// To achieve all of the goals, the project is divided into many components. The
// components are desined in a functional style API, whereby objects are
// explicitly passed around as arguments to functions that operate on them.
//
// This tries to avoid defining methods on structs. This comes at a cost of
// limiting chaining, this cost is intential. I intend to work really hard on
// improving performance and thus avoiding spaghetti is not an option.
//
// Installation
//
// You can install  with go get
//   go get -u github.com/gernest/ngorm
//
//
// The package is divided into two phases, Query building and Query execution
// phase.
//
// Query Building
//
// The subpackage engine exposes a structure named Engine. This structure has
// everything necessary to build a query. Most of the functions defined in this
// package subpackages operate on this struct by accepting it as the first
// argument.
//
// Having this as a separate layer helps fine tuning the generated querries and
// also it make easy to test and very that the ORM is doing the right thing. So,
// the generated query can be easily optimised without adding a lot of overhead.
//
// Query execution
//
// This is s the phase where the generated sql query is executed. This phase is as generic as
// possible in a way that you can easily implement adoptes for non SQL database
// and still reap all the benefits of this package.
//
// Table of Ccntents
//
// The following are links to packages under this project.
//   [engine] https://godoc.org/github.com/gernest/ngorm/engine
// This is what drives the whole project, helps with query building and provides
// conveinet structure to help with query execution.
//
//   [scope] https://godoc.org/github.com/gernest/ngorm/scope
// Functions to help with model manipulations.
//
//   [search] https://godoc.org/github.com/gernest/ngorm/search
// Functions to help with search  querries building.
//
//   [hooks] https://godoc.org/github.com/gernest/ngorm/hooks
// Callbacks executed byt ngorm. You can easily overide and provide custom ones
// to suit your needs.
//
//   [logger] https://godoc.org/github.com/gernest/ngorm/logger
// The logger used by ngorm for logging. It is an interface, and a reference
// implementation is provided.
//
//   [dialects] https://godoc.org/github.com/gernest/ngorm/dialects
// Adopts to different SQL databases supported by ngorm. For now ngorm support
// mysql, mssql, postgresql, sqlite and ql.
package ngorm

// DB contains information for current db connection
type DB struct {
}
<commit_msg>Update godoc<commit_after>//Package ngorm is a Go Object relation mapper that focus on performance,
//maintainability, modularity,	battle testing, extensibility , safety and
//developer frindliness.
//
// To achieve all of the goals, the project is divided into many components. The
// components are desined in a functional style API, whereby objects are
// explicitly passed around as arguments to functions that operate on them.
//
// This tries to avoid defining methods on structs. This comes at a cost of
// limiting chaining, this cost is intential. I intend to work really hard on
// improving performance and thus avoiding spaghetti is not an option.
//
// Installation
//
// You can install  with go get
//   go get -u github.com/gernest/ngorm
//
//
// The package is divided into two phases, Query building and Query execution
// phase.
//
// Query Building
//
// The subpackage engine exposes a structure named Engine. This structure has
// everything necessary to build a query. Most of the functions defined in this
// package subpackages operate on this struct by accepting it as the first
// argument.
//
// Having this as a separate layer helps fine tuning the generated querries and
// also it make easy to test and very that the ORM is doing the right thing. So,
// the generated query can be easily optimised without adding a lot of overhead.
//
// Query execution
//
// This is s the phase where the generated sql query is executed. This phase is as generic as
// possible in a way that you can easily implement adoptes for non SQL database
// and still reap all the benefits of this package.
//
// Table of Ccntents
//
// The following are links to packages under this project.
//   [engine] https://godoc.org/github.com/gernest/ngorm/engine
// This is what drives the whole project, helps with query building and provides
// conveinet structure to help with query execution.
//
//   [scope] https://godoc.org/github.com/gernest/ngorm/scope
// Functions to help with model manipulations.
//
//   [search] https://godoc.org/github.com/gernest/ngorm/search
// Functions to help with search  querries building.
//
//   [hooks] https://godoc.org/github.com/gernest/ngorm/hooks
// Callbacks executed by ngorm. You can easily overide and provide custom ones
// to suit your needs.
//
//   [logger] https://godoc.org/github.com/gernest/ngorm/logger
// The logger used by ngorm for logging. It is an interface, and a reference
// implementation is provided.
//
//   [dialects] https://godoc.org/github.com/gernest/ngorm/dialects
// Adopts to different SQL databases supported by ngorm. For now ngorm support
// ql ,
package ngorm

// DB contains information for current db connection
type DB struct {
}
<|endoftext|>


<commit_before>//Package ngorm is a Go Object relation mapper that foucs on performance,
//maintainability, modularity,	battle testing, extensibility , safety and
//developer frindliness.
//
// To achieve all of the goals, the project is divided into many components. The
// components are desined in a functional style API, whereby objects are
// explicitly passed around as arguments to functions that operate on them.
//
// This tries to avoid defining methods on structs. This comes at a cost of
// limiting chaining, this cost is intential. I intend to work really hard on
// improving perfomance and thus avoiding spaghetti is not an option.
//
// The package is divided into two phases, Query building and Query execution
// phase.
//
// Query Building
//
// The subpackage engine exposes a structure named Engine. This structure has
// everything necessary to build a query. Most of the functions defined in this
// package subpackages operate on this struct by accepting it as the first
// argument.
//
// Having this as a separate layer helps fine tuning the generated querries and
// also it make easy to test and very that the ORM is doing the right thing. So,
// the generated query can be easily optimised without adding a lot of overhead.
//
// Query execution
//
// This is s the phase where the generated sql query is executed. This phase is as generic as
// possible in a way that you can easily implement adoptes for non SQL database
// and still reap all the benefits of this package.
//
// Table of Ccntents
//
// The following are links to packages under this project.
//   [engine] https://godoc.org/github.com/gernest/ngorm/engine
// This is what drives the whole project, helps with query building and provides
// conveinet structure to help with query execution.
package ngorm

// DB contains information for current db connection
type DB struct {
}
<commit_msg>Add documentation<commit_after>//Package ngorm is a Go Object relation mapper that foucs on performance,
//maintainability, modularity,	battle testing, extensibility , safety and
//developer frindliness.
//
// To achieve all of the goals, the project is divided into many components. The
// components are desined in a functional style API, whereby objects are
// explicitly passed around as arguments to functions that operate on them.
//
// This tries to avoid defining methods on structs. This comes at a cost of
// limiting chaining, this cost is intential. I intend to work really hard on
// improving perfomance and thus avoiding spaghetti is not an option.
//
// Installation
//
// You can install  with go get
//   go get -u github.com/gernest/ngorm
//
//
// The package is divided into two phases, Query building and Query execution
// phase.
//
// Query Building
//
// The subpackage engine exposes a structure named Engine. This structure has
// everything necessary to build a query. Most of the functions defined in this
// package subpackages operate on this struct by accepting it as the first
// argument.
//
// Having this as a separate layer helps fine tuning the generated querries and
// also it make easy to test and very that the ORM is doing the right thing. So,
// the generated query can be easily optimised without adding a lot of overhead.
//
// Query execution
//
// This is s the phase where the generated sql query is executed. This phase is as generic as
// possible in a way that you can easily implement adoptes for non SQL database
// and still reap all the benefits of this package.
//
// Table of Ccntents
//
// The following are links to packages under this project.
//   [engine] https://godoc.org/github.com/gernest/ngorm/engine
// This is what drives the whole project, helps with query building and provides
// conveinet structure to help with query execution.
//
//   [scope] https://godoc.org/github.com/gernest/ngorm/scope
// Functions to help with model manipulations.
//
//   [search] https://godoc.org/github.com/gernest/ngorm/search
// Functions to help with search  querries building.
//
//   [hooks] https://godoc.org/github.com/gernest/ngorm/hooks
// Callbacks executed byt ngorm. You can easily overide and provide custom ones
// to suit your needs.
//
//   [logger] https://godoc.org/github.com/gernest/ngorm/logger
// The logger used by ngorm for logging. It is an interface, and a reference
// implementation is provided.
//
//   [dialects] https://godoc.org/github.com/gernest/ngorm/dialects
// Adopts to different SQL databases supported by ngorm. For now ngorm support
// mysql, mssql, postgresql, sqlite and ql.
package ngorm

// DB contains information for current db connection
type DB struct {
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"runtime"

	"github.com/shurcooL/trayhost"
)

func main() {
	runtime.LockOSThread()

	menuItems := []trayhost.MenuItem{
		trayhost.MenuItem{
			Title: "Instant Share",
			Handler: func() {
				fmt.Println("TODO: grab content, content-type of clipboard")
				fmt.Println("TODO: request URL")
				fmt.Println("TODO: display/put URL in clipboard")
				fmt.Println("TODO: upload image in background")
			},
		},
		trayhost.SeparatorMenuItem(),
		trayhost.MenuItem{
			Title:   "Quit",
			Handler: trayhost.Exit,
		},
	}

	// TODO: Create a real icon and bake it into the binary.
	iconData, err := ioutil.ReadFile("./icon.png")
	if err != nil {
		panic(err)
	}

	trayhost.Initialize("InstantShare", iconData, menuItems)

	trayhost.EnterLoop()
}
<commit_msg>First working prototype.<commit_after>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"runtime"

	"github.com/shurcooL/trayhost"
)

func instantShareHandler() {
	fmt.Println("grab content, content-type of clipboard")

	img, err := trayhost.GetClipboardImage()
	if err != nil {
		log.Println(err)
		return
	}

	fmt.Println("request URL")

	resp, err := http.Get("http://localhost:27080/api/getfilename?ext=png")
	if err != nil {
		log.Println(err)
		return
	}
	defer resp.Body.Close()
	filename, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Println(err)
		return
	}

	fmt.Println("display/put URL in clipboard")

	url := "http://localhost:27080/" + string(filename)
	trayhost.SetClipboardString(url)
	// TODO: Notification? Or not?

	fmt.Println("upload image in background of size", len(img))

	go func() {
		req, err := http.NewRequest("PUT", url, bytes.NewReader(img))
		if err != nil {
			log.Println(err)
			return
		}
		req.Header.Set("Content-Type", "image/png")
		resp, err := http.DefaultClient.Do(req)
		if err != nil {
			log.Println(err)
			return
		}
		_ = resp.Body.Close()
	}()
}

func main() {
	runtime.LockOSThread()

	menuItems := []trayhost.MenuItem{
		trayhost.MenuItem{
			Title:   "Instant Share",
			Handler: instantShareHandler,
		},
		trayhost.MenuItem{
			Title: "Debug: Get Clipboard String",
			Handler: func() {
				str, err := trayhost.GetClipboardString()
				fmt.Printf("GetClipboardString(): %q %v\n", str, err)
			},
		},
		trayhost.MenuItem{
			Title: "Debug: Get Clipboard Image",
			Handler: func() {
				img, err := trayhost.GetClipboardImage()
				fmt.Printf("GetClipboardImage(): len(%v) %v\n", len(img), err)
			},
		},
		trayhost.MenuItem{
			Title: "Debug: Set Clipboard",
			Handler: func() {
				trayhost.SetClipboardString("http://www.example.org/image.png")
			},
		},
		trayhost.SeparatorMenuItem(),
		trayhost.MenuItem{
			Title:   "Quit",
			Handler: trayhost.Exit,
		},
	}

	// TODO: Create a real icon and bake it into the binary.
	iconData, err := ioutil.ReadFile("./icon.png")
	if err != nil {
		panic(err)
	}

	fmt.Println("Starting.")

	trayhost.Initialize("InstantShare", iconData, menuItems)

	trayhost.EnterLoop()

	fmt.Println("Exiting.")
}
<|endoftext|>


<commit_before>package main

const (
	msgMainUsage     = "mole [options] <command> [command-options]"
	msgDigUsage      = "mole [global-options] dig [options] <tunnel> [host]"
	msgInstallUsage  = "mole [global-options] install [package]"
	msgLsUsage       = "mole [global-options] ls [options] [regexp]"
	msgPushUsage     = "mole [global-options] push <tunnelfile>"
	msgRegisterUsage = "mole [global-options] register [options] <server>"
	msgShowUsage     = "mole [global-options] show [options] <tunnel>"
	msgTestUsage     = "mole [global-options] test [options] <tunnel>"
	msgUpgradeUsage  = "mole [global-options] upgrade [options]"
	msgVersionUsage  = "mole [global-options] version [options]"

	msgDigShort      = "Dig tunnel"
	msgInstallShort  = "Install package"
	msgLsShort       = "List tunnels"
	msgPushShort     = "Push tunnel"
	msgRegisterShort = "Register with server"
	msgRmShort       = "Delete tunnel"
	msgShowShort     = "Show tunnel"
	msgTestShort     = "Test tunnel"
	msgTicketShort   = "Explain current ticket"
	msgUpgradeShort  = "Upgrade mole"
	msgVersionShort  = "Show version"

	msgDebugEnabled = "Debug output enabled."

	msgErrGainRoot         = "Error: missing root privileges to execute %q.\nTo give mole root access, execute it using sudo. Mole itself drops root privilege on startup and executes as the non privileged user. However, child processes such as ifconfig will inherit the saved user ID and have the ability to become root as necessary."
	msgErrNoVPN            = "No VPN provider for %q available. Try 'mole install' to see what packages are available or use the packaging system native to your platform."
	msgErrIncorrectFwd     = "Badly formatted fwd command %q."
	msgErrIncorrectFwdSrc  = "Badly formatted fwd source %q."
	msgErrIncorrectFwdDst  = "Badly formatted fwd destination %q."
	msgErrIncorrectFwdIP   = "Cannot forward from non-existent local IP %q."
	msgErrIncorrectFwdPriv = "Cannot forward from privileged port %q (<1024)."
	msgErrNoSuchCommand    = `No such command %q. Try "help".`
	msgErrNoHome           = "No home directory that I could find; cannot proceed."
	msgErrPEMNoKey         = "No ssh key found after PEM decode."

	msgVpncStart     = "vpnc: Started (pid %d)."
	msgVpncStopping  = "vpnc: Stopping (pid %d)."
	msgVpncWait      = "vpnc: Waiting for connect..."
	msgVpncConnected = "vpnc: Connected."
	msgVpncStopped   = "vpnc: Stopped."

	msgOpncStart     = "openconnect: Started (pid %d)."
	msgOpncStopping  = "openconnect: Stopping (pid %d)."
	msgOpncWait      = "openconnect: Waiting for connect..."
	msgOpncConnected = "openconnect: Connected."
	msgOpncStopped   = "openconnect: Stopped."

	msgDownloadingUpgrade = "Downloading upgrade..."
	msgUpgraded           = "Upgraded your mole to %s."

	msgFileNotInit    = "File %q should have .ini extension"
	msgOkPushed       = "Pushed %q"
	msgErrNoTunModule = "Required tunnel module (kernel extension) not available and not loadable."

	msgNeedsAuth       = "Authentication required. Enter your LDAP credentials."
	msgUsername        = "Username: "
	msgPassword        = "Password for %q: "
	msgPasswordVisible = "Password will be visible when typed."

	msgNoHost = "No server hostname is configured. Have you run 'mole register'?"

	msgNoPackages = "There are no packages available for installation on your OS/architecture."

	msgRegistered = "Registered with %q. Consider running 'mole install' to see what extra packages, such as VPN providers, are available."

	msgOkDeleted = "Deleted %q."

	msg530 = "530 Version Unacceptable\nYour client is either too new or too old to talk to this server. Make sure you are in fact registered with the correct server and try 'mole upgrade' to get the newest client."

	msgLatest       = "You are running the latest version."
	msgAutoUpgrades = "Mole uses automatic upgrades to keep your client up to date. To disable these automatic upgrades (which is a bad idea for most users) or silence this message, see 'mole upgrade -help'."
	msgUpdatedHost  = "Updated configured server name to %q."

	msgLsFlags = `  o····  Requires OpenConnect
  v····  Requires vpnc
  ·k···  Uses SSH with key authentication
  ··p··  Uses SSH with password authentication
  ···l·  Uses local (non-SSH) forwards
  ···s·  Uses SOCKS proxy
  ····E  Parse or access error reading tunnel
  ····U  Unknown or unsupported features required
`
	msgTesting            = "Connected; verifying connectivity in background..."
	msgTunnelRtt          = "Tunnel RTT ~%.0f ms; %d of %d forwards connect OK"
	msgKeepaliveTimeout   = "SSH server alive check failed"
	msgTunnelVerifyFailed = "No forwards (out of %d) could connect. Aborting."

	msgTicketExplanation = "Ticket valid for %s\nUntil %s\nFor the following IPs:"

	msgDigWarnMainHost = "Using non-default main host; some or all tunnels may be nonfunctional."
	msgDigNoHost       = "Host %q does not exist in tunnel configuration."
)
<commit_msg>Remove message about dropping privs<commit_after>package main

const (
	msgMainUsage     = "mole [options] <command> [command-options]"
	msgDigUsage      = "mole [global-options] dig [options] <tunnel> [host]"
	msgInstallUsage  = "mole [global-options] install [package]"
	msgLsUsage       = "mole [global-options] ls [options] [regexp]"
	msgPushUsage     = "mole [global-options] push <tunnelfile>"
	msgRegisterUsage = "mole [global-options] register [options] <server>"
	msgShowUsage     = "mole [global-options] show [options] <tunnel>"
	msgTestUsage     = "mole [global-options] test [options] <tunnel>"
	msgUpgradeUsage  = "mole [global-options] upgrade [options]"
	msgVersionUsage  = "mole [global-options] version [options]"

	msgDigShort      = "Dig tunnel"
	msgInstallShort  = "Install package"
	msgLsShort       = "List tunnels"
	msgPushShort     = "Push tunnel"
	msgRegisterShort = "Register with server"
	msgRmShort       = "Delete tunnel"
	msgShowShort     = "Show tunnel"
	msgTestShort     = "Test tunnel"
	msgTicketShort   = "Explain current ticket"
	msgUpgradeShort  = "Upgrade mole"
	msgVersionShort  = "Show version"

	msgDebugEnabled = "Debug output enabled."

	msgErrGainRoot         = "Error: missing root privileges to execute %q.\nTo give mole root access, execute it using sudo."
	msgErrNoVPN            = "No VPN provider for %q available. Try 'mole install' to see what packages are available or use the packaging system native to your platform."
	msgErrIncorrectFwd     = "Badly formatted fwd command %q."
	msgErrIncorrectFwdSrc  = "Badly formatted fwd source %q."
	msgErrIncorrectFwdDst  = "Badly formatted fwd destination %q."
	msgErrIncorrectFwdIP   = "Cannot forward from non-existent local IP %q."
	msgErrIncorrectFwdPriv = "Cannot forward from privileged port %q (<1024)."
	msgErrNoSuchCommand    = `No such command %q. Try "help".`
	msgErrNoHome           = "No home directory that I could find; cannot proceed."
	msgErrPEMNoKey         = "No ssh key found after PEM decode."

	msgVpncStart     = "vpnc: Started (pid %d)."
	msgVpncStopping  = "vpnc: Stopping (pid %d)."
	msgVpncWait      = "vpnc: Waiting for connect..."
	msgVpncConnected = "vpnc: Connected."
	msgVpncStopped   = "vpnc: Stopped."

	msgOpncStart     = "openconnect: Started (pid %d)."
	msgOpncStopping  = "openconnect: Stopping (pid %d)."
	msgOpncWait      = "openconnect: Waiting for connect..."
	msgOpncConnected = "openconnect: Connected."
	msgOpncStopped   = "openconnect: Stopped."

	msgDownloadingUpgrade = "Downloading upgrade..."
	msgUpgraded           = "Upgraded your mole to %s."

	msgFileNotInit    = "File %q should have .ini extension"
	msgOkPushed       = "Pushed %q"
	msgErrNoTunModule = "Required tunnel module (kernel extension) not available and not loadable."

	msgNeedsAuth       = "Authentication required. Enter your LDAP credentials."
	msgUsername        = "Username: "
	msgPassword        = "Password for %q: "
	msgPasswordVisible = "Password will be visible when typed."

	msgNoHost = "No server hostname is configured. Have you run 'mole register'?"

	msgNoPackages = "There are no packages available for installation on your OS/architecture."

	msgRegistered = "Registered with %q. Consider running 'mole install' to see what extra packages, such as VPN providers, are available."

	msgOkDeleted = "Deleted %q."

	msg530 = "530 Version Unacceptable\nYour client is either too new or too old to talk to this server. Make sure you are in fact registered with the correct server and try 'mole upgrade' to get the newest client."

	msgLatest       = "You are running the latest version."
	msgAutoUpgrades = "Mole uses automatic upgrades to keep your client up to date. To disable these automatic upgrades (which is a bad idea for most users) or silence this message, see 'mole upgrade -help'."
	msgUpdatedHost  = "Updated configured server name to %q."

	msgLsFlags = `  o····  Requires OpenConnect
  v····  Requires vpnc
  ·k···  Uses SSH with key authentication
  ··p··  Uses SSH with password authentication
  ···l·  Uses local (non-SSH) forwards
  ···s·  Uses SOCKS proxy
  ····E  Parse or access error reading tunnel
  ····U  Unknown or unsupported features required
`
	msgTesting            = "Connected; verifying connectivity in background..."
	msgTunnelRtt          = "Tunnel RTT ~%.0f ms; %d of %d forwards connect OK"
	msgKeepaliveTimeout   = "SSH server alive check failed"
	msgTunnelVerifyFailed = "No forwards (out of %d) could connect. Aborting."

	msgTicketExplanation = "Ticket valid for %s\nUntil %s\nFor the following IPs:"

	msgDigWarnMainHost = "Using non-default main host; some or all tunnels may be nonfunctional."
	msgDigNoHost       = "Host %q does not exist in tunnel configuration."
)
<|endoftext|>


<commit_before>package netest

import (
	"bytes"
	"encoding/binary"
	"net"
)

var MTU = 1500

type PacketHeader struct {
	Length   uint16
	Sequence uint16
}

type Packet struct {
	PacketHeader
	Payload []byte
}

type Connection struct {
	err      error
	sequence uint16
	socket   *net.UDPConn
}

func (c *Connection) getAddress(address string) *net.UDPAddr {
	var udpAddress *net.UDPAddr
	if c.err == nil {
		udpAddress, c.err = net.ResolveUDPAddr("udp", address)
		return udpAddress
	}
	return nil
}

func NewSink(addr string) (*Connection, error) {
	c := new(Connection)
	address := c.getAddress(addr)
	if c.err == nil {
		c.socket, c.err = net.ListenUDP("udp", address)
	}
	return c, c.err
}

func NewSrc(laddr, raddr string) (*Connection, error) {
	c := new(Connection)
	localAddr := c.getAddress(laddr)
	remoteAddr := c.getAddress(raddr)
	if c.err == nil {
		c.socket, c.err = net.DialUDP("udp", localAddr, remoteAddr)
	}
	return c, c.err
}

func (c *Connection) SendMsg(payload []byte) error {
	if c.err != nil {
		return c.err
	}
	h := new(PacketHeader)
	h.Length = uint16(len(payload) + binary.Size(h))
	h.Sequence = c.sequence

	buf := make([]byte, 0)
	buffer := bytes.NewBuffer(buf)

	binary.Write(buffer, binary.BigEndian, h)
	buffer.Write(payload)
	_, c.err = buffer.WriteTo(c.socket)

	if c.err != nil {
		c.sequence++
	}
	return c.err
}

func (c *Connection) ReceiveMsg() (*Packet, error) {
	if c.err != nil {
		return nil, c.err
	}

	buf := make([]byte, MTU)
	_, c.err = c.socket.Read(buf)

	if c.err != nil {
		return nil, c.err
	}

	h := new(PacketHeader)
	buffer := bytes.NewBuffer(buf)
	binary.Read(buffer, binary.BigEndian, h)

	p := new(Packet)
	p.PacketHeader = *h
	p.Payload = buf[4 : h.Length-1]
	return p, c.err
}

func (c *Connection) Close() error {
	return c.socket.Close()
}
<commit_msg>Fixes payload too large issue as well as incrementing the sequence number<commit_after>package netest

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"net"
)

var MTU = 1500

type PacketHeader struct {
	Length   uint16
	Sequence uint16
}

type Packet struct {
	PacketHeader
	Payload []byte
}

type Connection struct {
	err      error
	sequence uint16
	socket   *net.UDPConn
}

func (c *Connection) getAddress(address string) *net.UDPAddr {
	var udpAddress *net.UDPAddr
	if c.err == nil {
		udpAddress, c.err = net.ResolveUDPAddr("udp", address)
		return udpAddress
	}
	return nil
}

func NewSink(addr string) (*Connection, error) {
	c := new(Connection)
	address := c.getAddress(addr)
	if c.err == nil {
		c.socket, c.err = net.ListenUDP("udp", address)
	}
	return c, c.err
}

func NewSrc(laddr, raddr string) (*Connection, error) {
	c := new(Connection)
	localAddr := c.getAddress(laddr)
	remoteAddr := c.getAddress(raddr)
	if c.err == nil {
		c.socket, c.err = net.DialUDP("udp", localAddr, remoteAddr)
	}
	return c, c.err
}

func (c *Connection) SendMsg(payload []byte) error {
	if c.err != nil {
		return c.err
	}
	if len(payload) > MTU {
		return fmt.Errorf("payload cannot be greater than %d bytes", MTU)
	}
	h := new(PacketHeader)
	h.Length = uint16(len(payload) + binary.Size(h))
	h.Sequence = c.sequence

	buf := make([]byte, 0)
	buffer := bytes.NewBuffer(buf)

	binary.Write(buffer, binary.BigEndian, h)
	buffer.Write(payload)
	_, c.err = buffer.WriteTo(c.socket)

	if c.err == nil {
		c.sequence++
	}
	return c.err
}

func (c *Connection) ReceiveMsg() (*Packet, error) {
	if c.err != nil {
		return nil, c.err
	}

	buf := make([]byte, MTU)
	_, c.err = c.socket.Read(buf)

	if c.err != nil {
		return nil, c.err
	}

	h := new(PacketHeader)
	buffer := bytes.NewBuffer(buf)
	binary.Read(buffer, binary.BigEndian, h)

	p := new(Packet)
	p.PacketHeader = *h
	p.Payload = buf[4 : h.Length-1]
	return p, c.err
}

func (c *Connection) Close() error {
	return c.socket.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"crypto/rand"
	"encoding/base64"
	"encoding/hex"
	"fmt"
	"io"
	"net"
	"net/http"
	"strings"
)

const ProxyAuthenticationRequired = "HTTP/1.0 407 Proxy authentication required\r\n\r\n"

type connection struct {
	id       string
	incoming net.Conn
	outgoing net.Conn
	proxy
	localAddr net.Addr
}

func (c *connection) Dial(network, address string) (net.Conn, error) {
	if config.UseIncomingLocalAddr {
		if c.localAddr == nil {
			logger.Warn.Println(c.id, "Missing local net.Addr: a default local net.Addr will be used")
			goto fallback
		}

		// Ensure the TCPAddr has its Port set to 0, which is way of telling the dialer to use
		// and random port.
		switch tcpAddr := c.localAddr.(type) {
		case *net.TCPAddr:
			tcpAddr.Port = 0
		default:
			logger.Warn.Println(c.id, "Ignoring local net.Addr", c.localAddr, "because net.TCPAddr was expected")
			goto fallback
		}

		dialer := &net.Dialer{LocalAddr: c.localAddr}

		// Try to dial with the incoming LocalAddr to keep the incoming and outgoing IPs the same.
		conn, err := dialer.Dial(network, address)
		if err == nil {
			return conn, nil
		}

		// If an error occurs, fallback to the default interface. This might happen if you connected
		// via a loopback interace, like testing on the same machine. We should be more specifc about
		// error handling, but falling back is fine for now.
		logger.Warn.Println(c.id, "Ignoring local net.Addr for", c.localAddr, "dialing due to error:", err)
	}

fallback:
	return net.Dial(network, address)
}

func (c *connection) Handle() {
	logger.Info.Println(c.id, "Handling new connection.")

	reader := bufio.NewReader(c.incoming)
	request, err := http.ReadRequest(reader)
	if err == io.EOF {
		logger.Warn.Println(c.id, "Incoming connection disconnected.")
		return
	}
	if err != nil {
		logger.Warn.Println(c.id, "Could not parse or read request from incoming connection:", err)
		return
	}

	defer request.Body.Close()

	if !isAuthenticated(request) {
		logger.Fatal.Println(c.id, "Invalid credentials.")
		c.incoming.Write([]byte(ProxyAuthenticationRequired))
		return
	}

	// Delete the auth and proxy headers.
	if config.AuthenticationRequired() {
		request.Header.Del("Proxy-Authorization")
	}

	// Delete any other proxy related thing if enabled.
	if config.StripProxyHeaders {
		request.Header.Del("Forwarded")
		request.Header.Del("Proxy-Connection")
		request.Header.Del("Via")
		request.Header.Del("X-Forwarded-For")
		request.Header.Del("X-Forwarded-Host")
		request.Header.Del("X-Forwarded-Proto")
	}

	logger.Info.Println(c.id, "Processing connection to:", request.Method, request.Host)

	if request.Method == "CONNECT" {
		c.proxy = &httpsProxy{}
	} else {
		c.proxy = &httpProxy{}
	}

	err = c.proxy.SetupOutgoing(c, request)
	if err != nil {
		logger.Warn.Println(c.id, err)
		return
	}

	// Spawn incoming->outgoing and outgoing->incoming streams.
	signal := make(chan error, 1)
	go streamBytes(c.incoming, c.outgoing, signal)
	go streamBytes(c.outgoing, c.incoming, signal)

	// Wait for either stream to complete and finish. The second will always be an error.
	err = <-signal
	if err != nil {
		logger.Warn.Println(c.id, "Error reading or writing data", request.Host, err)
		return
	}
}

func (c *connection) Close() {
	if c.incoming != nil {
		c.incoming.Close()
	}

	if c.outgoing != nil {
		c.outgoing.Close()
	}

	logger.Info.Println(c.id, "Connection closed.")
}

// COPIED FROM STD LIB TO USE WITH PROXY-AUTH HEADER
// parseBasicAuth parses an HTTP Basic Authentication string.
// "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==" returns ("Aladdin", "open sesame", true).
func parseBasicAuth(auth string) (username, password string, ok bool) {
	const prefix = "Basic "
	if !strings.HasPrefix(auth, prefix) {
		return
	}
	c, err := base64.StdEncoding.DecodeString(auth[len(prefix):])
	if err != nil {
		return
	}
	cs := string(c)
	s := strings.IndexByte(cs, ':')
	if s < 0 {
		return
	}
	return cs[:s], cs[s+1:], true
}

func isAuthenticated(request *http.Request) bool {
	if !config.AuthenticationRequired() {
		return true
	}

	proxyAuthHeader := request.Header.Get("Proxy-Authorization")
	if proxyAuthHeader == "" {
		return false
	}

	username, password, ok := parseBasicAuth(proxyAuthHeader)
	if !ok {
		return false
	}

	return config.IsAuthenticated(username, password)
}

func newConnectionId() string {
	bytes := make([]byte, 3) // 6 characters long.
	if _, err := rand.Read(bytes); err != nil {
		return "[ERROR-MAKING-ID]"
	}
	return "[" + hex.EncodeToString(bytes) + "]"
}

func NewConnection(incoming net.Conn) *connection {
	newId := fmt.Sprint(newConnectionId(), " [", incoming.RemoteAddr().String(), "]")
	localAddr := incoming.LocalAddr()

	return &connection{
		id:        newId,
		incoming:  incoming,
		localAddr: localAddr,
	}
}
<commit_msg>Fix typo in comments about Port=0<commit_after>package main

import (
	"bufio"
	"crypto/rand"
	"encoding/base64"
	"encoding/hex"
	"fmt"
	"io"
	"net"
	"net/http"
	"strings"
)

const ProxyAuthenticationRequired = "HTTP/1.0 407 Proxy authentication required\r\n\r\n"

type connection struct {
	id       string
	incoming net.Conn
	outgoing net.Conn
	proxy
	localAddr net.Addr
}

func (c *connection) Dial(network, address string) (net.Conn, error) {
	if config.UseIncomingLocalAddr {
		if c.localAddr == nil {
			logger.Warn.Println(c.id, "Missing local net.Addr: a default local net.Addr will be used")
			goto fallback
		}

		switch tcpAddr := c.localAddr.(type) {
		case *net.TCPAddr:
			// Ensure the TCPAddr has its Port set to 0, which is way of telling the dialer to
			// use any random port. If you don't change this, you'll get a bind error.
			tcpAddr.Port = 0
		default:
			logger.Warn.Println(c.id, "Ignoring local net.Addr", c.localAddr, "because net.TCPAddr was expected")
			goto fallback
		}

		dialer := &net.Dialer{LocalAddr: c.localAddr}

		// Try to dial with the incoming LocalAddr to keep the incoming and outgoing IPs the same.
		conn, err := dialer.Dial(network, address)
		if err == nil {
			return conn, nil
		}

		// If an error occurs, fallback to the default interface. This might happen if you connected
		// via a loopback interace, like testing on the same machine. We should be more specifc about
		// error handling, but falling back is fine for now.
		logger.Warn.Println(c.id, "Ignoring local net.Addr for", c.localAddr, "dialing due to error:", err)
	}

fallback:
	return net.Dial(network, address)
}

func (c *connection) Handle() {
	logger.Info.Println(c.id, "Handling new connection.")

	reader := bufio.NewReader(c.incoming)
	request, err := http.ReadRequest(reader)
	if err == io.EOF {
		logger.Warn.Println(c.id, "Incoming connection disconnected.")
		return
	}
	if err != nil {
		logger.Warn.Println(c.id, "Could not parse or read request from incoming connection:", err)
		return
	}

	defer request.Body.Close()

	if !isAuthenticated(request) {
		logger.Fatal.Println(c.id, "Invalid credentials.")
		c.incoming.Write([]byte(ProxyAuthenticationRequired))
		return
	}

	// Delete the auth and proxy headers.
	if config.AuthenticationRequired() {
		request.Header.Del("Proxy-Authorization")
	}

	// Delete any other proxy related thing if enabled.
	if config.StripProxyHeaders {
		request.Header.Del("Forwarded")
		request.Header.Del("Proxy-Connection")
		request.Header.Del("Via")
		request.Header.Del("X-Forwarded-For")
		request.Header.Del("X-Forwarded-Host")
		request.Header.Del("X-Forwarded-Proto")
	}

	logger.Info.Println(c.id, "Processing connection to:", request.Method, request.Host)

	if request.Method == "CONNECT" {
		c.proxy = &httpsProxy{}
	} else {
		c.proxy = &httpProxy{}
	}

	err = c.proxy.SetupOutgoing(c, request)
	if err != nil {
		logger.Warn.Println(c.id, err)
		return
	}

	// Spawn incoming->outgoing and outgoing->incoming streams.
	signal := make(chan error, 1)
	go streamBytes(c.incoming, c.outgoing, signal)
	go streamBytes(c.outgoing, c.incoming, signal)

	// Wait for either stream to complete and finish. The second will always be an error.
	err = <-signal
	if err != nil {
		logger.Warn.Println(c.id, "Error reading or writing data", request.Host, err)
		return
	}
}

func (c *connection) Close() {
	if c.incoming != nil {
		c.incoming.Close()
	}

	if c.outgoing != nil {
		c.outgoing.Close()
	}

	logger.Info.Println(c.id, "Connection closed.")
}

// COPIED FROM STD LIB TO USE WITH PROXY-AUTH HEADER
// parseBasicAuth parses an HTTP Basic Authentication string.
// "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==" returns ("Aladdin", "open sesame", true).
func parseBasicAuth(auth string) (username, password string, ok bool) {
	const prefix = "Basic "
	if !strings.HasPrefix(auth, prefix) {
		return
	}
	c, err := base64.StdEncoding.DecodeString(auth[len(prefix):])
	if err != nil {
		return
	}
	cs := string(c)
	s := strings.IndexByte(cs, ':')
	if s < 0 {
		return
	}
	return cs[:s], cs[s+1:], true
}

func isAuthenticated(request *http.Request) bool {
	if !config.AuthenticationRequired() {
		return true
	}

	proxyAuthHeader := request.Header.Get("Proxy-Authorization")
	if proxyAuthHeader == "" {
		return false
	}

	username, password, ok := parseBasicAuth(proxyAuthHeader)
	if !ok {
		return false
	}

	return config.IsAuthenticated(username, password)
}

func newConnectionId() string {
	bytes := make([]byte, 3) // 6 characters long.
	if _, err := rand.Read(bytes); err != nil {
		return "[ERROR-MAKING-ID]"
	}
	return "[" + hex.EncodeToString(bytes) + "]"
}

func NewConnection(incoming net.Conn) *connection {
	newId := fmt.Sprint(newConnectionId(), " [", incoming.RemoteAddr().String(), "]")
	localAddr := incoming.LocalAddr()

	return &connection{
		id:        newId,
		incoming:  incoming,
		localAddr: localAddr,
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 The go-github AUTHORS. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package github

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"reflect"
	"testing"
)

func TestRepositoriesService_CreateHook(t *testing.T) {
	client, mux, _, teardown := setup()
	defer teardown()

	input := &Hook{CreatedAt: &referenceTime}

	mux.HandleFunc("/repos/o/r/hooks", func(w http.ResponseWriter, r *http.Request) {
		v := new(createHookRequest)
		json.NewDecoder(r.Body).Decode(v)

		testMethod(t, r, "POST")
		want := &createHookRequest{Name: "web"}
		if !reflect.DeepEqual(v, want) {
			t.Errorf("Request body = %+v, want %+v", v, want)
		}

		fmt.Fprint(w, `{"id":1}`)
	})

	ctx := context.Background()
	hook, _, err := client.Repositories.CreateHook(ctx, "o", "r", input)
	if err != nil {
		t.Errorf("Repositories.CreateHook returned error: %v", err)
	}

	want := &Hook{ID: Int64(1)}
	if !reflect.DeepEqual(hook, want) {
		t.Errorf("Repositories.CreateHook returned %+v, want %+v", hook, want)
	}
}

func TestRepositoriesService_ListHooks(t *testing.T) {
	client, mux, _, teardown := setup()
	defer teardown()

	mux.HandleFunc("/repos/o/r/hooks", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		testFormValues(t, r, values{"page": "2"})
		fmt.Fprint(w, `[{"id":1}, {"id":2}]`)
	})

	opt := &ListOptions{Page: 2}

	ctx := context.Background()
	hooks, _, err := client.Repositories.ListHooks(ctx, "o", "r", opt)
	if err != nil {
		t.Errorf("Repositories.ListHooks returned error: %v", err)
	}

	want := []*Hook{{ID: Int64(1)}, {ID: Int64(2)}}
	if !reflect.DeepEqual(hooks, want) {
		t.Errorf("Repositories.ListHooks returned %+v, want %+v", hooks, want)
	}
}

func TestRepositoriesService_ListHooks_invalidOwner(t *testing.T) {
	client, _, _, teardown := setup()
	defer teardown()

	ctx := context.Background()
	_, _, err := client.Repositories.ListHooks(ctx, "%", "%", nil)
	testURLParseError(t, err)
}

func TestRepositoriesService_GetHook(t *testing.T) {
	client, mux, _, teardown := setup()
	defer teardown()

	mux.HandleFunc("/repos/o/r/hooks/1", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		fmt.Fprint(w, `{"id":1}`)
	})

	ctx := context.Background()
	hook, _, err := client.Repositories.GetHook(ctx, "o", "r", 1)
	if err != nil {
		t.Errorf("Repositories.GetHook returned error: %v", err)
	}

	want := &Hook{ID: Int64(1)}
	if !reflect.DeepEqual(hook, want) {
		t.Errorf("Repositories.GetHook returned %+v, want %+v", hook, want)
	}
}

func TestRepositoriesService_GetHook_invalidOwner(t *testing.T) {
	client, _, _, teardown := setup()
	defer teardown()

	ctx := context.Background()
	_, _, err := client.Repositories.GetHook(ctx, "%", "%", 1)
	testURLParseError(t, err)
}

func TestRepositoriesService_EditHook(t *testing.T) {
	client, mux, _, teardown := setup()
	defer teardown()

	input := &Hook{}

	mux.HandleFunc("/repos/o/r/hooks/1", func(w http.ResponseWriter, r *http.Request) {
		v := new(Hook)
		json.NewDecoder(r.Body).Decode(v)

		testMethod(t, r, "PATCH")
		if !reflect.DeepEqual(v, input) {
			t.Errorf("Request body = %+v, want %+v", v, input)
		}

		fmt.Fprint(w, `{"id":1}`)
	})

	ctx := context.Background()
	hook, _, err := client.Repositories.EditHook(ctx, "o", "r", 1, input)
	if err != nil {
		t.Errorf("Repositories.EditHook returned error: %v", err)
	}

	want := &Hook{ID: Int64(1)}
	if !reflect.DeepEqual(hook, want) {
		t.Errorf("Repositories.EditHook returned %+v, want %+v", hook, want)
	}
}

func TestRepositoriesService_EditHook_invalidOwner(t *testing.T) {
	client, _, _, teardown := setup()
	defer teardown()

	ctx := context.Background()
	_, _, err := client.Repositories.EditHook(ctx, "%", "%", 1, nil)
	testURLParseError(t, err)
}

func TestRepositoriesService_DeleteHook(t *testing.T) {
	client, mux, _, teardown := setup()
	defer teardown()

	mux.HandleFunc("/repos/o/r/hooks/1", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "DELETE")
	})

	ctx := context.Background()
	_, err := client.Repositories.DeleteHook(ctx, "o", "r", 1)
	if err != nil {
		t.Errorf("Repositories.DeleteHook returned error: %v", err)
	}
}

func TestRepositoriesService_DeleteHook_invalidOwner(t *testing.T) {
	client, _, _, teardown := setup()
	defer teardown()

	ctx := context.Background()
	_, err := client.Repositories.DeleteHook(ctx, "%", "%", 1)
	testURLParseError(t, err)
}

func TestRepositoriesService_PingHook(t *testing.T) {
	client, mux, _, teardown := setup()
	defer teardown()

	mux.HandleFunc("/repos/o/r/hooks/1/pings", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "POST")
	})

	ctx := context.Background()
	_, err := client.Repositories.PingHook(ctx, "o", "r", 1)
	if err != nil {
		t.Errorf("Repositories.PingHook returned error: %v", err)
	}
}

func TestRepositoriesService_TestHook(t *testing.T) {
	client, mux, _, teardown := setup()
	defer teardown()

	mux.HandleFunc("/repos/o/r/hooks/1/tests", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "POST")
	})

	ctx := context.Background()
	_, err := client.Repositories.TestHook(ctx, "o", "r", 1)
	if err != nil {
		t.Errorf("Repositories.TestHook returned error: %v", err)
	}
}

func TestRepositoriesService_TestHook_invalidOwner(t *testing.T) {
	client, _, _, teardown := setup()
	defer teardown()

	ctx := context.Background()
	_, err := client.Repositories.TestHook(ctx, "%", "%", 1)
	testURLParseError(t, err)
}
<commit_msg>Improve repos_hooks.go coverage (#1750)<commit_after>// Copyright 2013 The go-github AUTHORS. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package github

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"reflect"
	"testing"
)

func TestRepositoriesService_CreateHook(t *testing.T) {
	client, mux, _, teardown := setup()
	defer teardown()

	input := &Hook{CreatedAt: &referenceTime}

	mux.HandleFunc("/repos/o/r/hooks", func(w http.ResponseWriter, r *http.Request) {
		v := new(createHookRequest)
		json.NewDecoder(r.Body).Decode(v)

		testMethod(t, r, "POST")
		want := &createHookRequest{Name: "web"}
		if !reflect.DeepEqual(v, want) {
			t.Errorf("Request body = %+v, want %+v", v, want)
		}

		fmt.Fprint(w, `{"id":1}`)
	})

	ctx := context.Background()
	hook, _, err := client.Repositories.CreateHook(ctx, "o", "r", input)
	if err != nil {
		t.Errorf("Repositories.CreateHook returned error: %v", err)
	}

	want := &Hook{ID: Int64(1)}
	if !reflect.DeepEqual(hook, want) {
		t.Errorf("Repositories.CreateHook returned %+v, want %+v", hook, want)
	}

	const methodName = "CreateHook"
	testBadOptions(t, methodName, func() (err error) {
		_, _, err = client.Repositories.CreateHook(ctx, "\n", "\n", input)
		return err
	})

	testNewRequestAndDoFailure(t, methodName, client, func() (*Response, error) {
		got, resp, err := client.Repositories.CreateHook(ctx, "o", "r", input)
		if got != nil {
			t.Errorf("testNewRequestAndDoFailure %v = %#v, want nil", methodName, got)
		}
		return resp, err
	})
}

func TestRepositoriesService_ListHooks(t *testing.T) {
	client, mux, _, teardown := setup()
	defer teardown()

	mux.HandleFunc("/repos/o/r/hooks", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		testFormValues(t, r, values{"page": "2"})
		fmt.Fprint(w, `[{"id":1}, {"id":2}]`)
	})

	opt := &ListOptions{Page: 2}

	ctx := context.Background()
	hooks, _, err := client.Repositories.ListHooks(ctx, "o", "r", opt)
	if err != nil {
		t.Errorf("Repositories.ListHooks returned error: %v", err)
	}

	want := []*Hook{{ID: Int64(1)}, {ID: Int64(2)}}
	if !reflect.DeepEqual(hooks, want) {
		t.Errorf("Repositories.ListHooks returned %+v, want %+v", hooks, want)
	}

	const methodName = "ListHooks"
	testBadOptions(t, methodName, func() (err error) {
		_, _, err = client.Repositories.ListHooks(ctx, "\n", "\n", opt)
		return err
	})

	testNewRequestAndDoFailure(t, methodName, client, func() (*Response, error) {
		got, resp, err := client.Repositories.ListHooks(ctx, "o", "r", opt)
		if got != nil {
			t.Errorf("testNewRequestAndDoFailure %v = %#v, want nil", methodName, got)
		}
		return resp, err
	})
}

func TestRepositoriesService_ListHooks_invalidOwner(t *testing.T) {
	client, _, _, teardown := setup()
	defer teardown()

	ctx := context.Background()
	_, _, err := client.Repositories.ListHooks(ctx, "%", "%", nil)
	testURLParseError(t, err)
}

func TestRepositoriesService_GetHook(t *testing.T) {
	client, mux, _, teardown := setup()
	defer teardown()

	mux.HandleFunc("/repos/o/r/hooks/1", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		fmt.Fprint(w, `{"id":1}`)
	})

	ctx := context.Background()
	hook, _, err := client.Repositories.GetHook(ctx, "o", "r", 1)
	if err != nil {
		t.Errorf("Repositories.GetHook returned error: %v", err)
	}

	want := &Hook{ID: Int64(1)}
	if !reflect.DeepEqual(hook, want) {
		t.Errorf("Repositories.GetHook returned %+v, want %+v", hook, want)
	}

	const methodName = "GetHook"
	testBadOptions(t, methodName, func() (err error) {
		_, _, err = client.Repositories.GetHook(ctx, "\n", "\n", -1)
		return err
	})

	testNewRequestAndDoFailure(t, methodName, client, func() (*Response, error) {
		got, resp, err := client.Repositories.GetHook(ctx, "o", "r", 1)
		if got != nil {
			t.Errorf("testNewRequestAndDoFailure %v = %#v, want nil", methodName, got)
		}
		return resp, err
	})
}

func TestRepositoriesService_GetHook_invalidOwner(t *testing.T) {
	client, _, _, teardown := setup()
	defer teardown()

	ctx := context.Background()
	_, _, err := client.Repositories.GetHook(ctx, "%", "%", 1)
	testURLParseError(t, err)
}

func TestRepositoriesService_EditHook(t *testing.T) {
	client, mux, _, teardown := setup()
	defer teardown()

	input := &Hook{}

	mux.HandleFunc("/repos/o/r/hooks/1", func(w http.ResponseWriter, r *http.Request) {
		v := new(Hook)
		json.NewDecoder(r.Body).Decode(v)

		testMethod(t, r, "PATCH")
		if !reflect.DeepEqual(v, input) {
			t.Errorf("Request body = %+v, want %+v", v, input)
		}

		fmt.Fprint(w, `{"id":1}`)
	})

	ctx := context.Background()
	hook, _, err := client.Repositories.EditHook(ctx, "o", "r", 1, input)
	if err != nil {
		t.Errorf("Repositories.EditHook returned error: %v", err)
	}

	want := &Hook{ID: Int64(1)}
	if !reflect.DeepEqual(hook, want) {
		t.Errorf("Repositories.EditHook returned %+v, want %+v", hook, want)
	}

	const methodName = "EditHook"
	testBadOptions(t, methodName, func() (err error) {
		_, _, err = client.Repositories.EditHook(ctx, "\n", "\n", -1, input)
		return err
	})

	testNewRequestAndDoFailure(t, methodName, client, func() (*Response, error) {
		got, resp, err := client.Repositories.EditHook(ctx, "o", "r", 1, input)
		if got != nil {
			t.Errorf("testNewRequestAndDoFailure %v = %#v, want nil", methodName, got)
		}
		return resp, err
	})
}

func TestRepositoriesService_EditHook_invalidOwner(t *testing.T) {
	client, _, _, teardown := setup()
	defer teardown()

	ctx := context.Background()
	_, _, err := client.Repositories.EditHook(ctx, "%", "%", 1, nil)
	testURLParseError(t, err)
}

func TestRepositoriesService_DeleteHook(t *testing.T) {
	client, mux, _, teardown := setup()
	defer teardown()

	mux.HandleFunc("/repos/o/r/hooks/1", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "DELETE")
	})

	ctx := context.Background()
	_, err := client.Repositories.DeleteHook(ctx, "o", "r", 1)
	if err != nil {
		t.Errorf("Repositories.DeleteHook returned error: %v", err)
	}

	const methodName = "DeleteHook"
	testBadOptions(t, methodName, func() (err error) {
		_, err = client.Repositories.DeleteHook(ctx, "\n", "\n", -1)
		return err
	})

	testNewRequestAndDoFailure(t, methodName, client, func() (*Response, error) {
		return client.Repositories.DeleteHook(ctx, "o", "r", 1)
	})
}

func TestRepositoriesService_DeleteHook_invalidOwner(t *testing.T) {
	client, _, _, teardown := setup()
	defer teardown()

	ctx := context.Background()
	_, err := client.Repositories.DeleteHook(ctx, "%", "%", 1)
	testURLParseError(t, err)
}

func TestRepositoriesService_PingHook(t *testing.T) {
	client, mux, _, teardown := setup()
	defer teardown()

	mux.HandleFunc("/repos/o/r/hooks/1/pings", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "POST")
	})

	ctx := context.Background()
	_, err := client.Repositories.PingHook(ctx, "o", "r", 1)
	if err != nil {
		t.Errorf("Repositories.PingHook returned error: %v", err)
	}

	const methodName = "PingHook"
	testBadOptions(t, methodName, func() (err error) {
		_, err = client.Repositories.PingHook(ctx, "\n", "\n", -1)
		return err
	})

	testNewRequestAndDoFailure(t, methodName, client, func() (*Response, error) {
		return client.Repositories.PingHook(ctx, "o", "r", 1)
	})
}

func TestRepositoriesService_TestHook(t *testing.T) {
	client, mux, _, teardown := setup()
	defer teardown()

	mux.HandleFunc("/repos/o/r/hooks/1/tests", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "POST")
	})

	ctx := context.Background()
	_, err := client.Repositories.TestHook(ctx, "o", "r", 1)
	if err != nil {
		t.Errorf("Repositories.TestHook returned error: %v", err)
	}

	const methodName = "TestHook"
	testBadOptions(t, methodName, func() (err error) {
		_, err = client.Repositories.TestHook(ctx, "\n", "\n", -1)
		return err
	})

	testNewRequestAndDoFailure(t, methodName, client, func() (*Response, error) {
		return client.Repositories.TestHook(ctx, "o", "r", 1)
	})
}

func TestRepositoriesService_TestHook_invalidOwner(t *testing.T) {
	client, _, _, teardown := setup()
	defer teardown()

	ctx := context.Background()
	_, err := client.Repositories.TestHook(ctx, "%", "%", 1)
	testURLParseError(t, err)
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"unicode/utf8"

	log "github.com/Sirupsen/logrus"
	"github.com/caarlos0/env"
	"github.com/fgrehm/go-dockerpty"
	"github.com/fsouza/go-dockerclient"
)

type environment struct {
	Image              string `env:"DUPLICITY_DOCKER_IMAGE" envDefault:"camptocamp/duplicity:latest"`
	DuplicityTargetURL string `env:"DUPLICITY_TARGET_URL"`
	AWSAccessKeyID     string `env:"AWS_ACCESS_KEY_ID"`
	AWSSecretAccessKey string `env:"AWS_SECRET_ACCESS_KEY"`
	SwiftUsername      string `env:"SWIFT_USERNAME"`
	SwiftPassword      string `env:"SWIFT_PASSWORD"`
	SwiftAuthURL       string `env:"SWIFT_AUTHURL"`
	SwiftTenantName    string `env:"SWIFT_TENANTNAME"`
	SwiftRegionName    string `env:"SWIFT_REGIONNAME"`
}

type conplicity struct {
	*docker.Client
	*environment
	Hostname string
}

func main() {
	log.Infof("Starting backup...")

	var err error

	c := &conplicity{}

	c.getEnv()

	c.Hostname, err = os.Hostname()
	checkErr(err, "Failed to get hostname: %v", 1)

	endpoint := "unix:///var/run/docker.sock"

	c.Client, err = docker.NewClient(endpoint)
	checkErr(err, "Failed to create Docker client: %v", 1)

	vols, err := c.ListVolumes(docker.ListVolumesOptions{})
	checkErr(err, "Failed to list Docker volumes: %v", 1)

	err = c.pullImage()
	checkErr(err, "Failed to pull image: %v", 1)

	for _, vol := range vols {
		err = c.backupVolume(vol)
		checkErr(err, "Failed to process volume "+vol.Name+": %v", -1)
	}

	log.Infof("End backup...")
}

func (c *conplicity) getEnv() (err error) {
	c.environment = &environment{}
	env.Parse(c.environment)

	return
}

func (c *conplicity) backupVolume(vol docker.Volume) (err error) {
	if utf8.RuneCountInString(vol.Name) == 64 {
		log.Infof("Ignoring volume " + vol.Name)
		return
	}

	// TODO: detect if it's a Database volume (PostgreSQL, MySQL, OpenLDAP...) and launch DUPLICITY_PRECOMMAND instead of backuping the volume
	log.Infof("ID: " + vol.Name)
	log.Infof("Driver: " + vol.Driver)
	log.Infof("Mountpoint: " + vol.Mountpoint)
	log.Infof("Creating duplicity container...")
	container, err := c.CreateContainer(
		docker.CreateContainerOptions{
			Config: &docker.Config{
				Cmd: []string{
					"--full-if-older-than", "15D",
					"--s3-use-new-style",
					"--no-encryption",
					"--allow-source-mismatch",
					"/var/backups",
					c.DuplicityTargetURL + "/" + c.Hostname + "/" + vol.Name,
				},
				Env: []string{
					"AWS_ACCESS_KEY_ID=" + c.AWSAccessKeyID,
					"AWS_SECRET_ACCESS_KEY=" + c.AWSSecretAccessKey,
					"SWIFT_USERNAME=" + c.SwiftUsername,
					"SWIFT_PASSWORD=" + c.SwiftPassword,
					"SWIFT_AUTHURL=" + c.SwiftAuthURL,
					"SWIFT_TENANTNAME=" + c.SwiftTenantName,
					"SWIFT_REGIONNAME=" + c.SwiftRegionName,
					"SWIFT_AUTHVERSION=2",
				},
				Image:        c.Image,
				OpenStdin:    true,
				StdinOnce:    true,
				AttachStdin:  true,
				AttachStdout: true,
				AttachStderr: true,
				Tty:          true,
			},
		},
	)

	checkErr(err, "Failed to create container for volume "+vol.Name+": %v", 1)

	defer func() {
		c.RemoveContainer(docker.RemoveContainerOptions{
			ID:    container.ID,
			Force: true,
		})
	}()

	binds := []string{
		vol.Mountpoint + ":/var/backups:ro",
	}

	err = dockerpty.Start(c.Client, container, &docker.HostConfig{
		Binds: binds,
	})
	checkErr(err, "Failed to start container for volume "+vol.Name+": %v", -1)
	return
}

func (c *conplicity) pullImage() (err error) {
	if _, err = c.InspectImage(c.Image); err != nil {
		// TODO: output pull to logs
		log.Infof("Pulling image %v", c.Image)
		err = c.PullImage(docker.PullImageOptions{
			Repository: c.Image,
		}, docker.AuthConfiguration{})
	}

	return err
}

func checkErr(err error, msg string, exit int) {
	if err != nil {
		log.Errorf(msg, err)

		if exit != -1 {
			os.Exit(exit)
		}
	}
}
<commit_msg>Ignore volumes marked as io.conplicity.<vol>.ignore=true<commit_after>package main

import (
	"os"
	"unicode/utf8"

	log "github.com/Sirupsen/logrus"
	"github.com/caarlos0/env"
	"github.com/fgrehm/go-dockerpty"
	"github.com/fsouza/go-dockerclient"
)

const labelPrefix string = "io.conplicity"

type environment struct {
	Image              string `env:"DUPLICITY_DOCKER_IMAGE" envDefault:"camptocamp/duplicity:latest"`
	DuplicityTargetURL string `env:"DUPLICITY_TARGET_URL"`
	AWSAccessKeyID     string `env:"AWS_ACCESS_KEY_ID"`
	AWSSecretAccessKey string `env:"AWS_SECRET_ACCESS_KEY"`
	SwiftUsername      string `env:"SWIFT_USERNAME"`
	SwiftPassword      string `env:"SWIFT_PASSWORD"`
	SwiftAuthURL       string `env:"SWIFT_AUTHURL"`
	SwiftTenantName    string `env:"SWIFT_TENANTNAME"`
	SwiftRegionName    string `env:"SWIFT_REGIONNAME"`
}

type conplicity struct {
	*docker.Client
	*environment
	Hostname string
}

func main() {
	log.Infof("Starting backup...")

	var err error

	c := &conplicity{}

	c.getEnv()

	c.Hostname, err = os.Hostname()
	checkErr(err, "Failed to get hostname: %v", 1)

	endpoint := "unix:///var/run/docker.sock"

	c.Client, err = docker.NewClient(endpoint)
	checkErr(err, "Failed to create Docker client: %v", 1)

	vols, err := c.ListVolumes(docker.ListVolumesOptions{})
	checkErr(err, "Failed to list Docker volumes: %v", 1)

	err = c.pullImage()
	checkErr(err, "Failed to pull image: %v", 1)

	for _, vol := range vols {
    voll, _ := c.InspectVolume(vol.Name)
		checkErr(err, "Failed to inspect volume "+vol.Name+": %v", -1)
		err = c.backupVolume(*voll)
		checkErr(err, "Failed to process volume "+vol.Name+": %v", -1)
	}

	log.Infof("End backup...")
}

func (c *conplicity) getEnv() (err error) {
	c.environment = &environment{}
	env.Parse(c.environment)

	return
}

func (c *conplicity) backupVolume(vol docker.Volume) (err error) {
	if utf8.RuneCountInString(vol.Name) == 64 {
		log.Infof("Ignoring unnamed volume " + vol.Name)
		return
	}

  volLabelPrefix := labelPrefix + "." + vol.Name

  if vol.Labels[volLabelPrefix + ".ignore"] == "true" {
    log.Infof("Ignoring blacklisted volume " + vol.Name)
    return
  }

	// TODO: detect if it's a Database volume (PostgreSQL, MySQL, OpenLDAP...) and launch DUPLICITY_PRECOMMAND instead of backuping the volume
	log.Infof("ID: " + vol.Name)
	log.Infof("Driver: " + vol.Driver)
	log.Infof("Mountpoint: " + vol.Mountpoint)
	log.Infof("Creating duplicity container...")
	container, err := c.CreateContainer(
		docker.CreateContainerOptions{
			Config: &docker.Config{
				Cmd: []string{
					"--full-if-older-than", "15D",
					"--s3-use-new-style",
					"--no-encryption",
					"--allow-source-mismatch",
					"/var/backups",
					c.DuplicityTargetURL + "/" + c.Hostname + "/" + vol.Name,
				},
				Env: []string{
					"AWS_ACCESS_KEY_ID=" + c.AWSAccessKeyID,
					"AWS_SECRET_ACCESS_KEY=" + c.AWSSecretAccessKey,
					"SWIFT_USERNAME=" + c.SwiftUsername,
					"SWIFT_PASSWORD=" + c.SwiftPassword,
					"SWIFT_AUTHURL=" + c.SwiftAuthURL,
					"SWIFT_TENANTNAME=" + c.SwiftTenantName,
					"SWIFT_REGIONNAME=" + c.SwiftRegionName,
					"SWIFT_AUTHVERSION=2",
				},
				Image:        c.Image,
				OpenStdin:    true,
				StdinOnce:    true,
				AttachStdin:  true,
				AttachStdout: true,
				AttachStderr: true,
				Tty:          true,
			},
		},
	)

	checkErr(err, "Failed to create container for volume "+vol.Name+": %v", 1)

	defer func() {
		c.RemoveContainer(docker.RemoveContainerOptions{
			ID:    container.ID,
			Force: true,
		})
	}()

	binds := []string{
		vol.Mountpoint + ":/var/backups:ro",
	}

	err = dockerpty.Start(c.Client, container, &docker.HostConfig{
		Binds: binds,
	})
	checkErr(err, "Failed to start container for volume "+vol.Name+": %v", -1)
	return
}

func (c *conplicity) pullImage() (err error) {
	if _, err = c.InspectImage(c.Image); err != nil {
		// TODO: output pull to logs
		log.Infof("Pulling image %v", c.Image)
		err = c.PullImage(docker.PullImageOptions{
			Repository: c.Image,
		}, docker.AuthConfiguration{})
	}

	return err
}

func checkErr(err error, msg string, exit int) {
	if err != nil {
		log.Errorf(msg, err)

		if exit != -1 {
			os.Exit(exit)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"syscall"
)

func main() {
	if len(os.Args) != 2 {
		fmt.Printf("usage: %s /path", os.Args[0])
		os.Exit(1)
	}

	file, err := os.Open(os.Args[1])
	if err != nil {
		panic(err)
	}

	kq, err := syscall.Kqueue()
	if err != nil {
		fmt.Println(err)
	}

	ev1 := syscall.Kevent_t{
		Ident:  uint64(file.Fd()),
		Filter: syscall.EVFILT_VNODE,
		Flags:  syscall.EV_ADD | syscall.EV_ENABLE | syscall.EV_ONESHOT,
		Fflags: syscall.NOTE_DELETE | syscall.NOTE_WRITE | syscall.NOTE_EXTEND | syscall.NOTE_ATTRIB | syscall.NOTE_LINK | syscall.NOTE_RENAME | syscall.NOTE_REVOKE,
		Data:   0,
		Udata:  nil,
	}

	// create kevent
	events := make([]syscall.Kevent_t, 1)
	n, err := syscall.Kevent(kq, []syscall.Kevent_t{ev1}, events, nil)
	if err != nil {
		log.Println("Error creating kevent")
	}
	// check if there was an event
	for {
		if n > 0 {
			return
		}
	}

	fmt.Println("fin")
}
<commit_msg>	modified:   main.go<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"syscall"
)

func WatchDir(dir string, ch chan<- string) {
	file, err := os.Open(dir)
	if err != nil {
		log.Printf("err = %+v\n", err)
	}

	kq, err := syscall.Kqueue()
	if err != nil {
		log.Printf("err = %+v\n", err)
	}

	ev1 := syscall.Kevent_t{
		Ident:  uint64(file.Fd()),
		Filter: syscall.EVFILT_VNODE,
		Flags:  syscall.EV_ADD | syscall.EV_ENABLE | syscall.EV_ONESHOT,
		Fflags: syscall.NOTE_DELETE | syscall.NOTE_WRITE | syscall.NOTE_EXTEND | syscall.NOTE_ATTRIB | syscall.NOTE_LINK | syscall.NOTE_RENAME | syscall.NOTE_REVOKE,
		Data:   0,
		Udata:  nil,
	}

	// create kevent
	events := make([]syscall.Kevent_t, 1)
	n, err := syscall.Kevent(kq, []syscall.Kevent_t{ev1}, events, nil)
	if err != nil {
		log.Println("Error creating kevent")
	}
	// check if there was an event
	for {
		if n > 0 {
			ch <- dir
			return
		}
	}
}

func Scandir(dir string) []string {
	yml := []string{}
	d, err := os.Open(dir)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	defer d.Close()

	files, err := d.Readdir(-1)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	for _, file := range files {
		if file.Mode().IsRegular() {
			if filepath.Ext(file.Name()) == ".yml" {
				yml = append(yml, file.Name())
			}
		}
	}
	return yml
}

func main() {
	if len(os.Args) != 2 {
		fmt.Printf("usage: %s /path", os.Args[0])
		os.Exit(1)
	}

	dir := os.Args[1]

	watchDir := make(chan string, 1)
	watchFile := make(chan string, 1)

	yml := Scandir(dir)
	for _, y := range yml {
		fmt.Printf("Watching  %s\n", y)
		go WatchDir(filepath.Join(dir, y), watchFile)
	}

	WatchDir(dir, watchDir)

	for {
		select {
		case dir := <-watchDir:
			fmt.Printf("dir = %s\n", dir)
			println("find *.yml")
			yml2 := Scandir(dir)
			// replace this with a map On2
			for _, y := range yml2 {
				var skip bool
				for _, oy := range yml {
					if oy == y {
						skip = true
						continue
					}
				}
				if !skip {
					fmt.Printf("Watching  %s\n", y)
					go WatchDir(filepath.Join(dir, y), watchFile)
				}
			}
			go WatchDir(dir, watchDir)
		case file := <-watchFile:
			fmt.Printf("file changed = %s\n", file)
			go WatchDir(file, watchFile)
		}
	}
}
<|endoftext|>