<commit_before>// Package activitystreams provides all the basic ActivityStreams
// implementation needed for Write.as.
package activitystreams

import (
	"time"
)

const (
	Namespace = "https://www.w3.org/ns/activitystreams"
	toPublic  = "https://www.w3.org/ns/activitystreams#Public"
)

// Activity describes actions that have either already occurred, are in the
// process of occurring, or may occur in the future.
type Activity struct {
	BaseObject
	Actor     string    `json:"actor"`
	Published time.Time `json:"published,omitempty"`
	To        []string  `json:"to,omitempty"`
	CC        []string  `json:"cc,omitempty"`
	Object    *Object   `json:"object"`
}

// NewCreateActivity builds a basic Create activity that includes the given
// Object and the Object's AttributedTo property as the Actor.
func NewCreateActivity(o *Object) *Activity {
	a := Activity{
		BaseObject: BaseObject{
			Context: []interface{}{
				Namespace,
			},
			ID:   o.ID,
			Type: "Create",
		},
		Actor:     o.AttributedTo,
		Object:    o,
		Published: o.Published,
	}
	return &a
}

// NewDeleteActivity builds a basic Delete activity that includes the given
// Object and the Object's AttributedTo property as the Actor.
func NewDeleteActivity(o *Object) *Activity {
	a := Activity{
		BaseObject: BaseObject{
			Context: []interface{}{
				Namespace,
			},
			ID:   o.ID,
			Type: "Delete",
		},
		Actor:  o.AttributedTo,
		Object: o,
	}
	return &a
}

// Object is the primary base type for the Activity Streams vocabulary.
type Object struct {
	BaseObject
	Published    time.Time         `json:"published"`
	Summary      *string           `json:"summary,omitempty"`
	InReplyTo    *string           `json:"inReplyTo"`
	URL          string            `json:"url"`
	AttributedTo string            `json:"attributedTo"`
	To           []string          `json:"to"`
	CC           []string          `json:"cc,omitempty"`
	Name         string            `json:"name,omitempty"`
	Content      string            `json:"content"`
	ContentMap   map[string]string `json:"contentMap,omitempty"`
}

// NewNoteObject creates a basic Note object that includes the public
// namespace in IRIs it's addressed to.
func NewNoteObject() *Object {
	o := Object{
		BaseObject: BaseObject{
			Type: "Note",
		},
		To: []string{
			toPublic,
		},
	}
	return &o
}

// NewArticleObject creates a basic Article object that includes the public
// namespace in IRIs it's addressed to.
func NewArticleObject() *Object {
	o := Object{
		BaseObject: BaseObject{
			Type: "Article",
		},
		To: []string{
			toPublic,
		},
	}
	return &o
}
<commit_msg>Add NewUpdateActivity func<commit_after>// Package activitystreams provides all the basic ActivityStreams
// implementation needed for Write.as.
package activitystreams

import (
	"time"
)

const (
	Namespace = "https://www.w3.org/ns/activitystreams"
	toPublic  = "https://www.w3.org/ns/activitystreams#Public"
)

// Activity describes actions that have either already occurred, are in the
// process of occurring, or may occur in the future.
type Activity struct {
	BaseObject
	Actor     string    `json:"actor"`
	Published time.Time `json:"published,omitempty"`
	To        []string  `json:"to,omitempty"`
	CC        []string  `json:"cc,omitempty"`
	Object    *Object   `json:"object"`
}

// NewCreateActivity builds a basic Create activity that includes the given
// Object and the Object's AttributedTo property as the Actor.
func NewCreateActivity(o *Object) *Activity {
	a := Activity{
		BaseObject: BaseObject{
			Context: []interface{}{
				Namespace,
			},
			ID:   o.ID,
			Type: "Create",
		},
		Actor:     o.AttributedTo,
		Object:    o,
		Published: o.Published,
	}
	return &a
}

// NewUpdateActivity builds a basic Update activity that includes the given
// Object and the Object's AttributedTo property as the Actor.
func NewUpdateActivity(o *Object) *Activity {
	a := Activity{
		BaseObject: BaseObject{
			Context: []interface{}{
				Namespace,
			},
			ID:   o.ID,
			Type: "Update",
		},
		Actor:     o.AttributedTo,
		Object:    o,
		Published: o.Published,
	}
	return &a
}

// NewDeleteActivity builds a basic Delete activity that includes the given
// Object and the Object's AttributedTo property as the Actor.
func NewDeleteActivity(o *Object) *Activity {
	a := Activity{
		BaseObject: BaseObject{
			Context: []interface{}{
				Namespace,
			},
			ID:   o.ID,
			Type: "Delete",
		},
		Actor:  o.AttributedTo,
		Object: o,
	}
	return &a
}

// Object is the primary base type for the Activity Streams vocabulary.
type Object struct {
	BaseObject
	Published    time.Time         `json:"published"`
	Summary      *string           `json:"summary,omitempty"`
	InReplyTo    *string           `json:"inReplyTo"`
	URL          string            `json:"url"`
	AttributedTo string            `json:"attributedTo"`
	To           []string          `json:"to"`
	CC           []string          `json:"cc,omitempty"`
	Name         string            `json:"name,omitempty"`
	Content      string            `json:"content"`
	ContentMap   map[string]string `json:"contentMap,omitempty"`
}

// NewNoteObject creates a basic Note object that includes the public
// namespace in IRIs it's addressed to.
func NewNoteObject() *Object {
	o := Object{
		BaseObject: BaseObject{
			Type: "Note",
		},
		To: []string{
			toPublic,
		},
	}
	return &o
}

// NewArticleObject creates a basic Article object that includes the public
// namespace in IRIs it's addressed to.
func NewArticleObject() *Object {
	o := Object{
		BaseObject: BaseObject{
			Type: "Article",
		},
		To: []string{
			toPublic,
		},
	}
	return &o
}
<|endoftext|>


<commit_before>//
//   date  : 2014-06-04
//   author: xjdrew
//

package tunnel

import (
	"bufio"
	"encoding/binary"
	"errors"
	"fmt"
	"net"
	"sync"
	"time"
)

type TunnelPayload struct {
	Linkid uint16
	Data   []byte
}

type TunnelHeader struct {
	Linkid uint16
	Sz     uint16
}

type Tunnel struct {
	wlock  *sync.Mutex  // write lock
	writer *RC4Writer   // writer
	rlock  *sync.Mutex  // read lock
	reader *RC4Reader   // reader
	conn   *net.TCPConn // low level conn
	desc   string       // description
}

func (t *Tunnel) Close() {
	t.conn.Close()
}

func (t *Tunnel) Write(payload TunnelPayload) (err error) {
	defer mpool.Put(payload.Data)

	var header TunnelHeader
	header.Linkid = payload.Linkid
	header.Sz = uint16(len(payload.Data))

	t.wlock.Lock()
	defer t.wlock.Unlock()
	err = binary.Write(t.writer, binary.LittleEndian, &header)
	if err != nil {
		return
	}
	_, err = t.writer.Write(payload.Data)
	return err
}

func (t *Tunnel) Read() (payload *TunnelPayload, err error) {
	t.rlock.Lock()
	defer t.rlock.Unlock()

	var header TunnelHeader
	err = binary.Read(t.reader, binary.LittleEndian, &header)
	if err != nil {
		return
	}

	if header.Sz > options.PacketSize {
		err = errors.New("malformed packet, too long")
		return
	}

	payload = &TunnelPayload{}
	payload.Linkid = header.Linkid
	data := mpool.Get()[0:header.Sz]
	c := 0
	for c < int(header.Sz) {
		var n int
		n, err = t.reader.Read(data[c:])
		if err != nil {
			mpool.Put(data)
			return
		}
		c += n
	}
	payload.Data = data
	return
}

func (self *Tunnel) String() string {
	return fmt.Sprintf("%s", self.desc)
}

func newTunnel(conn *net.TCPConn, rc4key []byte) *Tunnel {
	desc := fmt.Sprintf("tunnel[%s <-> %s]", conn.LocalAddr(), conn.RemoteAddr())
	if err := conn.SetReadBuffer(options.TunnelReadBuffer); err != nil {
		Error("%s set read buffer failed:%s", desc, err.Error())
	}
	if err := conn.SetWriteBuffer(options.TunnelWriteBuffer); err != nil {
		Error("%s set read buffer failed:%s", desc, err.Error())
	}
	conn.SetKeepAlive(true)
	conn.SetKeepAlivePeriod(time.Second * 60)
	return &Tunnel{
		wlock:  new(sync.Mutex),
		writer: NewRC4Writer(conn, rc4key),
		rlock:  new(sync.Mutex),
		reader: NewRC4Reader(bufio.NewReaderSize(conn, 8192), rc4key),
		conn:   conn,
		desc:   desc,
	}
}
<commit_msg>use bufio.Writer<commit_after>//
//   date  : 2014-06-04
//   author: xjdrew
//

package tunnel

import (
	"bufio"
	"encoding/binary"
	"errors"
	"fmt"
	"net"
	"sync"
	"time"
)

type TunnelPayload struct {
	Linkid uint16
	Data   []byte
}

type TunnelHeader struct {
	Linkid uint16
	Sz     uint16
}

type Tunnel struct {
	wlock  *sync.Mutex   // write lock
	writer *bufio.Writer // writer
	rlock  *sync.Mutex   // read lock
	reader *bufio.Reader // reader
	conn   *net.TCPConn  // low level conn
	desc   string        // description
}

func (t *Tunnel) Close() {
	t.conn.Close()
}

func (t *Tunnel) Write(payload TunnelPayload) (err error) {
	defer mpool.Put(payload.Data)

	var header TunnelHeader
	header.Linkid = payload.Linkid
	header.Sz = uint16(len(payload.Data))

	t.wlock.Lock()
	defer t.wlock.Unlock()
	if err = binary.Write(t.writer, binary.LittleEndian, &header); err != nil {
		return
	}
	if _, err = t.writer.Write(payload.Data); err != nil {
		return
	}
	if err = t.writer.Flush(); err != nil {
		return
	}
	return
}

func (t *Tunnel) Read() (payload *TunnelPayload, err error) {
	t.rlock.Lock()
	defer t.rlock.Unlock()

	var header TunnelHeader
	err = binary.Read(t.reader, binary.LittleEndian, &header)
	if err != nil {
		return
	}

	if header.Sz > options.PacketSize {
		err = errors.New("malformed packet, too long")
		return
	}

	payload = &TunnelPayload{}
	payload.Linkid = header.Linkid
	data := mpool.Get()[0:header.Sz]
	c := 0
	for c < int(header.Sz) {
		var n int
		n, err = t.reader.Read(data[c:])
		if err != nil {
			mpool.Put(data)
			return
		}
		c += n
	}
	payload.Data = data
	return
}

func (self *Tunnel) String() string {
	return fmt.Sprintf("%s", self.desc)
}

func newTunnel(conn *net.TCPConn, rc4key []byte) *Tunnel {
	desc := fmt.Sprintf("tunnel[%s <-> %s]", conn.LocalAddr(), conn.RemoteAddr())
	if err := conn.SetReadBuffer(options.TunnelReadBuffer); err != nil {
		Error("%s set read buffer failed:%s", desc, err.Error())
	}
	if err := conn.SetWriteBuffer(options.TunnelWriteBuffer); err != nil {
		Error("%s set read buffer failed:%s", desc, err.Error())
	}
	conn.SetKeepAlive(true)
	conn.SetKeepAlivePeriod(time.Second * 60)
	bufsize := int(options.PacketSize) * 2
	return &Tunnel{
		wlock:  new(sync.Mutex),
		writer: bufio.NewWriterSize(NewRC4Writer(conn, rc4key), bufsize),
		rlock:  new(sync.Mutex),
		reader: bufio.NewReaderSize(NewRC4Reader(conn, rc4key), bufsize),
		conn:   conn,
		desc:   desc,
	}
}
<|endoftext|>


<commit_before>package cloudflare

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"time"

	"github.com/pkg/errors"
)

// TunnelRoute is the full record for a route.
type TunnelRoute struct {
	Network    string     `json:"network"`
	TunnelID   string     `json:"tunnel_id"`
	TunnelName string     `json:"tunnel_name"`
	Comment    string     `json:"comment"`
	CreatedAt  *time.Time `json:"created_at"`
	DeletedAt  *time.Time `json:"deleted_at"`
}

type TunnelRoutesListParams struct {
	AccountID       string
	TunnelID        string     `url:"tunnel_id,omitempty"`
	Comment         string     `url:"comment,omitempty"`
	IsDeleted       *bool      `url:"is_deleted,omitempty"`
	NetworkSubset   string     `url:"network_subset,omitempty"`
	NetworkSuperset string     `url:"network_superset,omitempty"`
	ExistedAt       *time.Time `url:"existed_at,omitempty"`
	PaginationOptions
}

type TunnelRoutesCreateParams struct {
	AccountID string `json:"-"`
	Network   string `json:"-"`
	TunnelID  string `json:"tunnel_id"`
	Comment   string `json:"comment,omitempty"`
}

type TunnelRoutesUpdateParams struct {
	AccountID string `json:"-"`
	Network   string `json:"network"`
	TunnelID  string `json:"tunnel_id"`
	Comment   string `json:"comment,omitempty"`
}

type TunnelRoutesForIPParams struct {
	AccountID string `json:"-"`
	Network   string `json:"-"`
}

type TunnelRoutesDeleteParams struct {
	AccountID string `json:"-"`
	Network   string `json:"-"`
}

// tunnelRouteListResponse is the API response for listing tunnel routes.
type tunnelRouteListResponse struct {
	Response
	Result []TunnelRoute `json:"result"`
}

type tunnelRouteResponse struct {
	Response
	Result TunnelRoute `json:"result"`
}

// ListTunnelRoutes lists all defined routes for tunnels in the account.
//
// See: https://api.cloudflare.com/#tunnel-route-list-tunnel-routes
func (api *API) ListTunnelRoutes(ctx context.Context, params TunnelRoutesListParams) ([]TunnelRoute, error) {
	if params.AccountID == "" {
		return []TunnelRoute{}, ErrMissingAccountID
	}

	uri := buildURI(fmt.Sprintf("/%s/%s/teamnet/routes", AccountRouteRoot, params.AccountID), params)
	res, err := api.makeRequestContext(ctx, http.MethodGet, uri, params)
	if err != nil {
		return []TunnelRoute{}, err
	}

	var resp tunnelRouteListResponse
	err = json.Unmarshal(res, &resp)
	if err != nil {
		return []TunnelRoute{}, errors.Wrap(err, errUnmarshalError)
	}

	return resp.Result, nil
}

// GetTunnelRouteForIP finds the Tunnel Route that encompasses the given IP.
//
// See: https://api.cloudflare.com/#tunnel-route-get-tunnel-route-by-ip
func (api *API) GetTunnelRouteForIP(ctx context.Context, params TunnelRoutesForIPParams) (TunnelRoute, error) {
	uri := fmt.Sprintf("/%s/%s/teamnet/routes/ip/%s", AccountRouteRoot, params.AccountID, params.Network)

	responseBody, err := api.makeRequestContext(ctx, http.MethodGet, uri, nil)
	if err != nil {
		return TunnelRoute{}, err
	}

	var routeResponse tunnelRouteResponse
	err = json.Unmarshal(responseBody, &routeResponse)
	if err != nil {
		return TunnelRoute{}, errors.Wrap(err, errUnmarshalError)
	}

	return routeResponse.Result, nil
}

// CreateTunnelRoute add a new route to the account routing table for the given
// tunnel.
//
// See: https://api.cloudflare.com/#tunnel-route-create-route
func (api *API) CreateTunnelRoute(ctx context.Context, params TunnelRoutesCreateParams) (TunnelRoute, error) {
	if params.AccountID == "" {
		return TunnelRoute{}, ErrMissingAccountID
	}

	uri := fmt.Sprintf("/%s/%s/teamnet/routes/network/%s", AccountRouteRoot, params.AccountID, url.PathEscape(params.Network))

	responseBody, err := api.makeRequestContext(ctx, http.MethodPost, uri, params)
	if err != nil {
		return TunnelRoute{}, err
	}

	var routeResponse tunnelRouteResponse
	err = json.Unmarshal(responseBody, &routeResponse)
	if err != nil {
		return TunnelRoute{}, errors.Wrap(err, errUnmarshalError)
	}

	return routeResponse.Result, nil
}

// DeleteTunnelRoute delete an existing route from the account routing table.
//
// See: https://api.cloudflare.com/#tunnel-route-delete-route
func (api *API) DeleteTunnelRoute(ctx context.Context, params TunnelRoutesDeleteParams) error {
	if params.AccountID == "" {
		return ErrMissingAccountID
	}

	uri := fmt.Sprintf("/%s/%s/teamnet/routes/network/%s", AccountRouteRoot, params.AccountID, url.PathEscape(params.Network))

	responseBody, err := api.makeRequestContext(ctx, http.MethodDelete, uri, nil)
	if err != nil {
		return err
	}

	var routeResponse tunnelRouteResponse
	err = json.Unmarshal(responseBody, &routeResponse)
	if err != nil {
		return errors.Wrap(err, errUnmarshalError)
	}

	return nil
}

// UpdateTunnelRoute updates an existing route in the account routing table for
// the given tunnel.
//
// See: https://api.cloudflare.com/#tunnel-route-update-route
func (api *API) UpdateTunnelRoute(ctx context.Context, params TunnelRoutesUpdateParams) (TunnelRoute, error) {
	if params.AccountID == "" {
		return TunnelRoute{}, ErrMissingAccountID
	}

	uri := fmt.Sprintf("/%s/%s/teamnet/routes/network/%s", AccountRouteRoot, params.AccountID, url.PathEscape(params.Network))

	responseBody, err := api.makeRequestContext(ctx, http.MethodPatch, uri, params)
	if err != nil {
		return TunnelRoute{}, err
	}

	var routeResponse tunnelRouteResponse
	err = json.Unmarshal(responseBody, &routeResponse)
	if err != nil {
		return TunnelRoute{}, errors.Wrap(err, errUnmarshalError)
	}

	return routeResponse.Result, nil
}
<commit_msg>tunnel_routes: stricter validation for parameters (#917)<commit_after>package cloudflare

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/pkg/errors"
)

var (
	ErrMissingNetwork      = errors.New("missing required network parameter")
	ErrInvalidNetworkValue = errors.New("invalid IP parameter. Cannot use CIDR ranges for this endpoint.")
)

// TunnelRoute is the full record for a route.
type TunnelRoute struct {
	Network    string     `json:"network"`
	TunnelID   string     `json:"tunnel_id"`
	TunnelName string     `json:"tunnel_name"`
	Comment    string     `json:"comment"`
	CreatedAt  *time.Time `json:"created_at"`
	DeletedAt  *time.Time `json:"deleted_at"`
}

type TunnelRoutesListParams struct {
	AccountID       string     `url:"-"`
	TunnelID        string     `url:"tunnel_id,omitempty"`
	Comment         string     `url:"comment,omitempty"`
	IsDeleted       *bool      `url:"is_deleted,omitempty"`
	NetworkSubset   string     `url:"network_subset,omitempty"`
	NetworkSuperset string     `url:"network_superset,omitempty"`
	ExistedAt       *time.Time `url:"existed_at,omitempty"`
	PaginationOptions
}

type TunnelRoutesCreateParams struct {
	AccountID string `json:"-"`
	Network   string `json:"-"`
	TunnelID  string `json:"tunnel_id"`
	Comment   string `json:"comment,omitempty"`
}

type TunnelRoutesUpdateParams struct {
	AccountID string `json:"-"`
	Network   string `json:"network"`
	TunnelID  string `json:"tunnel_id"`
	Comment   string `json:"comment,omitempty"`
}

type TunnelRoutesForIPParams struct {
	AccountID string `json:"-"`
	Network   string `json:"-"`
}

type TunnelRoutesDeleteParams struct {
	AccountID string `json:"-"`
	Network   string `json:"-"`
}

// tunnelRouteListResponse is the API response for listing tunnel routes.
type tunnelRouteListResponse struct {
	Response
	Result []TunnelRoute `json:"result"`
}

type tunnelRouteResponse struct {
	Response
	Result TunnelRoute `json:"result"`
}

// ListTunnelRoutes lists all defined routes for tunnels in the account.
//
// See: https://api.cloudflare.com/#tunnel-route-list-tunnel-routes
func (api *API) ListTunnelRoutes(ctx context.Context, params TunnelRoutesListParams) ([]TunnelRoute, error) {
	if params.AccountID == "" {
		return []TunnelRoute{}, ErrMissingAccountID
	}

	uri := buildURI(fmt.Sprintf("/%s/%s/teamnet/routes", AccountRouteRoot, params.AccountID), params)
	res, err := api.makeRequestContext(ctx, http.MethodGet, uri, nil)
	if err != nil {
		return []TunnelRoute{}, err
	}

	var resp tunnelRouteListResponse
	err = json.Unmarshal(res, &resp)
	if err != nil {
		return []TunnelRoute{}, errors.Wrap(err, errUnmarshalError)
	}

	return resp.Result, nil
}

// GetTunnelRouteForIP finds the Tunnel Route that encompasses the given IP.
//
// See: https://api.cloudflare.com/#tunnel-route-get-tunnel-route-by-ip
func (api *API) GetTunnelRouteForIP(ctx context.Context, params TunnelRoutesForIPParams) (TunnelRoute, error) {
	if params.AccountID == "" {
		return TunnelRoute{}, ErrMissingAccountID
	}

	if params.Network == "" {
		return TunnelRoute{}, ErrMissingNetwork
	}

	if strings.Contains(params.Network, "/") {
		return TunnelRoute{}, ErrInvalidNetworkValue
	}

	uri := fmt.Sprintf("/%s/%s/teamnet/routes/ip/%s", AccountRouteRoot, params.AccountID, params.Network)

	responseBody, err := api.makeRequestContext(ctx, http.MethodGet, uri, nil)
	if err != nil {
		return TunnelRoute{}, err
	}

	var routeResponse tunnelRouteResponse
	err = json.Unmarshal(responseBody, &routeResponse)
	if err != nil {
		return TunnelRoute{}, errors.Wrap(err, errUnmarshalError)
	}

	return routeResponse.Result, nil
}

// CreateTunnelRoute add a new route to the account routing table for the given
// tunnel.
//
// See: https://api.cloudflare.com/#tunnel-route-create-route
func (api *API) CreateTunnelRoute(ctx context.Context, params TunnelRoutesCreateParams) (TunnelRoute, error) {
	if params.AccountID == "" {
		return TunnelRoute{}, ErrMissingAccountID
	}

	if params.Network == "" {
		return TunnelRoute{}, ErrMissingNetwork
	}

	uri := fmt.Sprintf("/%s/%s/teamnet/routes/network/%s", AccountRouteRoot, params.AccountID, url.PathEscape(params.Network))

	responseBody, err := api.makeRequestContext(ctx, http.MethodPost, uri, params)
	if err != nil {
		return TunnelRoute{}, err
	}

	var routeResponse tunnelRouteResponse
	err = json.Unmarshal(responseBody, &routeResponse)
	if err != nil {
		return TunnelRoute{}, errors.Wrap(err, errUnmarshalError)
	}

	return routeResponse.Result, nil
}

// DeleteTunnelRoute delete an existing route from the account routing table.
//
// See: https://api.cloudflare.com/#tunnel-route-delete-route
func (api *API) DeleteTunnelRoute(ctx context.Context, params TunnelRoutesDeleteParams) error {
	if params.AccountID == "" {
		return ErrMissingAccountID
	}

	if params.Network == "" {
		return ErrMissingNetwork
	}

	uri := fmt.Sprintf("/%s/%s/teamnet/routes/network/%s", AccountRouteRoot, params.AccountID, url.PathEscape(params.Network))

	responseBody, err := api.makeRequestContext(ctx, http.MethodDelete, uri, nil)
	if err != nil {
		return err
	}

	var routeResponse tunnelRouteResponse
	err = json.Unmarshal(responseBody, &routeResponse)
	if err != nil {
		return errors.Wrap(err, errUnmarshalError)
	}

	return nil
}

// UpdateTunnelRoute updates an existing route in the account routing table for
// the given tunnel.
//
// See: https://api.cloudflare.com/#tunnel-route-update-route
func (api *API) UpdateTunnelRoute(ctx context.Context, params TunnelRoutesUpdateParams) (TunnelRoute, error) {
	if params.AccountID == "" {
		return TunnelRoute{}, ErrMissingAccountID
	}

	uri := fmt.Sprintf("/%s/%s/teamnet/routes/network/%s", AccountRouteRoot, params.AccountID, url.PathEscape(params.Network))

	responseBody, err := api.makeRequestContext(ctx, http.MethodPatch, uri, params)
	if err != nil {
		return TunnelRoute{}, err
	}

	var routeResponse tunnelRouteResponse
	err = json.Unmarshal(responseBody, &routeResponse)
	if err != nil {
		return TunnelRoute{}, errors.Wrap(err, errUnmarshalError)
	}

	return routeResponse.Result, nil
}
<|endoftext|>


<commit_before>// Package radixutil provides various methods for working with the radix.v2
// package, implenting various oft-used behaviors
package radixutil

import (
	"github.com/mediocregopher/radix.v2/cluster"
	"github.com/mediocregopher/radix.v2/pool"
	"github.com/mediocregopher/radix.v2/redis"
	"github.com/mediocregopher/radix.v2/util"
)

// DialMaybeCluster will do discovery on the redis instance at the given address
// to determine if it is part of a cluster or not. If so a radix Cluster is
// returned, using the given poolSize, if not a normal radix Pool is returned
// just for the given address.
func DialMaybeCluster(network, addr string, poolSize int) (util.Cmder, error) {
	dConn, err := redis.Dial(network, addr)
	if err != nil {
		return nil, err
	}
	defer dConn.Close()

	if dr := dConn.Cmd("CLUSTER", "SLOTS"); dr.IsType(redis.IOErr) {
		return nil, dr.Err
	} else if dr.IsType(redis.AppErr) {
		return pool.New(network, addr, poolSize)
	} else {
		return cluster.NewWithOpts(cluster.Opts{
			Addr:     addr,
			PoolSize: poolSize,
		})
	}
}

// WithGroupedKeys is used primarily when dealing with a Cluster. It will take
// all the given keys, and group them by which instance in the cluster they
// belong to. It then calls the given function with a client for that instance
// and the set of keys for it. If the given util.Cmder is not a Cluster then it
// must be a Pool, and the given function will only be called once, with all the
// given keys.
//
// This function is useful primarily if you have a large set of keys you want to
// MGET, MSET, DEL, etc.... but you're doing so in a clustered environment.
//
// Note that this operation is not atomic in anyway. If the cluster topology
// changes at any point during using this function you will most likely see an
// error during each invocation of the callback. It's also possible, in this
// case, that the callback won't even be called on every single key given. Make
// sure your application is ok with these conditions.
//
// An error is only returned during the retrieval of a client from the Pool or
// Cluster if an error is seen. Errors which occur during calling of the
// callback itself must be caught and handled outside this function.
func WithGroupedKeys(cmder util.Cmder, fn func(*redis.Client, []string), keys ...string) error {
	if p, ok := cmder.(*pool.Pool); ok {
		c, err := p.Get()
		if err != nil {
			return err
		}
		fn(c, keys)
		p.Put(c)
		return nil
	}

	cl := cmder.(*cluster.Cluster)
	m := map[string][]string{}
	for _, k := range keys {
		addr := cl.GetAddrForKey(k)
		m[addr] = append(m[addr], k)
	}

	cc, err := cl.GetEvery()
	if err != nil {
		return err
	}

	for addr, kk := range m {
		c, ok := cc[addr]
		if !ok {
			// topology changed halfway through, not much we can do
			continue
		}
		fn(c, kk)
	}
	for _, c := range cc {
		cl.Put(c)
	}

	return nil
}
<commit_msg>get rid of WithGroupedKeys method, it doesn't work<commit_after>// Package radixutil provides various methods for working with the radix.v2
// package, implenting various oft-used behaviors
package radixutil

import (
	"github.com/mediocregopher/radix.v2/cluster"
	"github.com/mediocregopher/radix.v2/pool"
	"github.com/mediocregopher/radix.v2/redis"
	"github.com/mediocregopher/radix.v2/util"
)

// DialMaybeCluster will do discovery on the redis instance at the given address
// to determine if it is part of a cluster or not. If so a radix Cluster is
// returned, using the given poolSize, if not a normal radix Pool is returned
// just for the given address.
func DialMaybeCluster(network, addr string, poolSize int) (util.Cmder, error) {
	dConn, err := redis.Dial(network, addr)
	if err != nil {
		return nil, err
	}
	defer dConn.Close()

	if dr := dConn.Cmd("CLUSTER", "SLOTS"); dr.IsType(redis.IOErr) {
		return nil, dr.Err
	} else if dr.IsType(redis.AppErr) {
		return pool.New(network, addr, poolSize)
	} else {
		return cluster.NewWithOpts(cluster.Opts{
			Addr:     addr,
			PoolSize: poolSize,
		})
	}
}
<|endoftext|>


<commit_before>package slackclient

import (
	"golang.org/x/net/context"

	"github.com/dimfeld/promulgator/commandrouter"
	"github.com/dimfeld/promulgator/model"
)

func processIncomingSlack(ctx context.Context, commandrouter *commandrouter.Router,
	command string, user string, inChannel string, toBot bool, responseChan chan string) {

	msg := model.ChatMessage{
		FromUser: user,
		Channel:  inChannel,
		Text:     command,
		ToBot:    toBot,
	}

	handled, err := commandrouter.Route(ctx, &msg, responseChan)
	if err != nil {
		logIn.Errorf("commandrouter: %s", err.Error())
		// TODO Make this message configurable?
		select {
		case responseChan <- "Internal error, please see bot logs":
		default:
		}
	} else if !handled {
		responseChan <- "I didn't recognize that command"
	}
}
<commit_msg>Trim whitespace around incoming message<commit_after>package slackclient

import (
	"strings"

	"golang.org/x/net/context"

	"github.com/dimfeld/promulgator/commandrouter"
	"github.com/dimfeld/promulgator/model"
)

func processIncomingSlack(ctx context.Context, commandrouter *commandrouter.Router,
	command string, user string, inChannel string, toBot bool, responseChan chan string) {

	msg := model.ChatMessage{
		FromUser: user,
		Channel:  inChannel,
		Text:     strings.TrimSpace(command),
		ToBot:    toBot,
	}

	handled, err := commandrouter.Route(ctx, &msg, responseChan)
	if err != nil {
		logIn.Errorf("commandrouter: %s", err.Error())
		// TODO Make this message configurable?
		select {
		case responseChan <- "Internal error, please see bot logs":
		default:
		}
	} else if !handled {
		responseChan <- "I didn't recognize that command"
	}
}
<|endoftext|>


<commit_before>package tmux

// go test -v github.com/monopole/mdrip/tmux --alsologtostderr

import (
	"strings"
	"testing"
)

const (
	badName            = "nonsensicalFakeHopeNotInstalledPgmName"
	skipAlreadyRunning = "skipping since tmux already running"
	skipNoTmux         = "skipping since tmux not found"
)

func TestBadName(t *testing.T) {
	if IsProgramInstalled(badName) {
		t.Errorf("Should fail using a nonsensical name like \"%s\".", badName)
	}
}

func TestAssureNotUp(t *testing.T) {
	if !IsProgramInstalled(Path) {
		t.Skip(skipNoTmux)
	}
	x := NewTmux(Path)
	if x.IsUp() {
		t.Error("tmux must not be up during test runs")
	}
}

func TestStartAndStopTmuxSession(t *testing.T) {
	if !IsProgramInstalled(Path) {
		t.Skip(skipNoTmux)
	}
	x := NewTmux(Path)
	if x.IsUp() {
		t.Skip(skipAlreadyRunning)
	}
	var out string
	err := x.start()
	if err != nil {
		t.Errorf("unable to start session: %s", err)
	}
	if !x.IsUp() {
		t.Errorf("tmux should appear as running: %s", err)
	}
	out, err = x.listSessions()
	if err != nil {
		t.Errorf("unable to list session: %s", err)
	}
	if !strings.Contains(out, SessionName+":") {
		t.Errorf("Expected %s:, got %s", SessionName, out)
	}
	err = x.stop()
	if err != nil {
		t.Errorf("unable to stop session: %s", err)
	}
}
<commit_msg>deletesillytest<commit_after>package tmux

import (
	"strings"
	"testing"
)

const (
	badName            = "nonsensicalFakeHopeNotInstalledPgmName"
	skipAlreadyRunning = "skipping since tmux already running"
	skipNoTmux         = "skipping since tmux not found"
)

func TestBadName(t *testing.T) {
	if IsProgramInstalled(badName) {
		t.Errorf("Should fail using a nonsensical name like \"%s\".", badName)
	}
}

func TestStartAndStopTmuxSession(t *testing.T) {
	if !IsProgramInstalled(Path) {
		t.Skip(skipNoTmux)
	}
	x := NewTmux(Path)
	if x.IsUp() {
		t.Skip(skipAlreadyRunning)
	}
	var out string
	err := x.start()
	if err != nil {
		t.Errorf("unable to start session: %s", err)
	}
	if !x.IsUp() {
		t.Errorf("tmux should appear as running: %s", err)
	}
	out, err = x.listSessions()
	if err != nil {
		t.Errorf("unable to list session: %s", err)
	}
	if !strings.Contains(out, SessionName+":") {
		t.Errorf("Expected %s:, got %s", SessionName, out)
	}
	err = x.stop()
	if err != nil {
		t.Errorf("unable to stop session: %s", err)
	}
}
<|endoftext|>


<commit_before>package sync

import (
	"github.com/watermint/toolbox/domain/common/model/mo_int"
	mo_path2 "github.com/watermint/toolbox/domain/common/model/mo_path"
	"github.com/watermint/toolbox/domain/dropbox/api/dbx_conn"
	"github.com/watermint/toolbox/domain/dropbox/model/mo_path"
	"github.com/watermint/toolbox/essentials/log/esl"
	"github.com/watermint/toolbox/infra/control/app_control"
	"github.com/watermint/toolbox/infra/recipe/rc_exec"
	"github.com/watermint/toolbox/infra/recipe/rc_recipe"
	"github.com/watermint/toolbox/ingredient/file"
	"github.com/watermint/toolbox/quality/demo/qdm_file"
	"github.com/watermint/toolbox/quality/infra/qt_errors"
	"github.com/watermint/toolbox/quality/recipe/qtr_endtoend"
)

type Up struct {
	rc_recipe.RemarkIrreversible
	Peer        dbx_conn.ConnUserFile
	LocalPath   mo_path2.ExistingFileSystemPath
	DropboxPath mo_path.DropboxPath
	ChunkSizeKb mo_int.RangeInt
	Upload      *file.Upload
	FailOnError bool
}

func (z *Up) Preset() {
	z.ChunkSizeKb.SetRange(1, 150*1024, 4*1024)
}

func (z *Up) Exec(c app_control.Control) error {
	l := c.Log()
	err := rc_exec.Exec(c, z.Upload, func(r rc_recipe.Recipe) {
		ru := r.(*file.Upload)
		ru.EstimateOnly = false
		ru.LocalPath = z.LocalPath
		ru.DropboxPath = z.DropboxPath
		ru.Overwrite = true
		ru.CreateFolder = true
		ru.Context = z.Peer.Context()
		ru.ChunkSizeKb = z.ChunkSizeKb.Value()
	})
	if z.FailOnError && err != nil {
		l.Debug("Return error", esl.Error(err))
		return err
	}
	return nil
}

func (z *Up) Test(c app_control.Control) error {
	// replay test
	{
		sc, err := qdm_file.NewScenario(false)
		if err != nil {
			return err
		}
		err = rc_exec.ExecReplay(c, &Up{}, "recipe-file-sync-up.json.gz", func(r rc_recipe.Recipe) {
			m := r.(*Up)
			m.LocalPath = mo_path2.NewExistingFileSystemPath(sc.LocalPath)
			m.DropboxPath = qtr_endtoend.NewTestDropboxFolderPath("file-sync-up")
		})
		if err != nil {
			return err
		}
	}

	err := rc_exec.ExecMock(c, &Up{}, func(r rc_recipe.Recipe) {
		m := r.(*Up)
		m.LocalPath = qtr_endtoend.NewTestExistingFileSystemFolderPath(c, "up")
		m.DropboxPath = qtr_endtoend.NewTestDropboxFolderPath("up")
	})
	if err, _ = qt_errors.ErrorsForTest(c.Log(), err); err != nil {
		return err
	}

	return qt_errors.ErrorScenarioTest
}
<commit_msg>fix #387 : change default chunk size to 64MiB from 4MiB based upon benchmarks<commit_after>package sync

import (
	"github.com/watermint/toolbox/domain/common/model/mo_int"
	mo_path2 "github.com/watermint/toolbox/domain/common/model/mo_path"
	"github.com/watermint/toolbox/domain/dropbox/api/dbx_conn"
	"github.com/watermint/toolbox/domain/dropbox/model/mo_path"
	"github.com/watermint/toolbox/essentials/log/esl"
	"github.com/watermint/toolbox/infra/control/app_control"
	"github.com/watermint/toolbox/infra/recipe/rc_exec"
	"github.com/watermint/toolbox/infra/recipe/rc_recipe"
	"github.com/watermint/toolbox/ingredient/file"
	"github.com/watermint/toolbox/quality/demo/qdm_file"
	"github.com/watermint/toolbox/quality/infra/qt_errors"
	"github.com/watermint/toolbox/quality/recipe/qtr_endtoend"
)

type Up struct {
	rc_recipe.RemarkIrreversible
	Peer        dbx_conn.ConnUserFile
	LocalPath   mo_path2.ExistingFileSystemPath
	DropboxPath mo_path.DropboxPath
	ChunkSizeKb mo_int.RangeInt
	Upload      *file.Upload
	FailOnError bool
}

func (z *Up) Preset() {
	z.ChunkSizeKb.SetRange(1, 150*1024, 64*1024)
}

func (z *Up) Exec(c app_control.Control) error {
	l := c.Log()
	err := rc_exec.Exec(c, z.Upload, func(r rc_recipe.Recipe) {
		ru := r.(*file.Upload)
		ru.EstimateOnly = false
		ru.LocalPath = z.LocalPath
		ru.DropboxPath = z.DropboxPath
		ru.Overwrite = true
		ru.CreateFolder = true
		ru.Context = z.Peer.Context()
		ru.ChunkSizeKb = z.ChunkSizeKb.Value()
	})
	if z.FailOnError && err != nil {
		l.Debug("Return error", esl.Error(err))
		return err
	}
	return nil
}

func (z *Up) Test(c app_control.Control) error {
	// replay test
	{
		sc, err := qdm_file.NewScenario(false)
		if err != nil {
			return err
		}
		err = rc_exec.ExecReplay(c, &Up{}, "recipe-file-sync-up.json.gz", func(r rc_recipe.Recipe) {
			m := r.(*Up)
			m.LocalPath = mo_path2.NewExistingFileSystemPath(sc.LocalPath)
			m.DropboxPath = qtr_endtoend.NewTestDropboxFolderPath("file-sync-up")
		})
		if err != nil {
			return err
		}
	}

	err := rc_exec.ExecMock(c, &Up{}, func(r rc_recipe.Recipe) {
		m := r.(*Up)
		m.LocalPath = qtr_endtoend.NewTestExistingFileSystemFolderPath(c, "up")
		m.DropboxPath = qtr_endtoend.NewTestDropboxFolderPath("up")
	})
	if err, _ = qt_errors.ErrorsForTest(c.Log(), err); err != nil {
		return err
	}

	return qt_errors.ErrorScenarioTest
}
<|endoftext|>


<commit_before>package ruby

import (
	"bytes"
	"text/template"

	"sync"

	"sourcegraph.com/sourcegraph/srcgraph/config"
	"sourcegraph.com/sourcegraph/srcgraph/dep2"
	"sourcegraph.com/sourcegraph/srcgraph/toolchain"
)

func init() {
	toolchain.Register("ruby", DefaultRubyVersion)
	config.Register("ruby", &Config{})
}

type Ruby struct {
	Version        string
	StdlibCloneURL string

	// resolveCache maps gem name to resolved dep target.
	resolveCache   map[string]*dep2.ResolvedTarget
	resolveCacheMu sync.Mutex
}

var DefaultRubyVersion = &Ruby{
	Version:        "2.0.0-p481",
	StdlibCloneURL: "git://github.com/ruby/ruby.git",
}

func (v *Ruby) baseDockerfile() ([]byte, error) {
	var buf bytes.Buffer
	err := template.Must(template.New("").Parse(baseDockerfile)).Execute(&buf, struct {
		Ruby   *Ruby
		GOPATH string
	}{
		Ruby: v,
	})
	if err != nil {
		return nil, err
	}

	return buf.Bytes(), nil
}

const baseDockerfile = `FROM ubuntu:14.04
RUN apt-get update -qq
RUN apt-get install -qqy curl

# Install Ruby {{.Ruby.Version}}
RUN curl -L https://get.rvm.io | bash -s stable
ENV PATH /usr/local/rvm/bin:$PATH
RUN rvm requirements
RUN rvm install {{.Ruby.Version}}
RUN rvm {{.Ruby.Version}} do gem install bundler --no-ri --no-rdoc
`

type Config struct {
	// OmitStdlib is whether to NOT load the analyzed graph of the Ruby standard
	// library when analyzing this repository. It should be true when analyzing
	// the Ruby stdlib itself.
	OmitStdlib bool
}

func (v *Ruby) rubyConfig(c *config.Repository) *Config {
	rubyConfig, _ := c.Global["ruby"].(*Config)
	if rubyConfig == nil {
		rubyConfig = new(Config)
	}
	return rubyConfig
}
<commit_msg>install git for ruby grapher<commit_after>package ruby

import (
	"bytes"
	"text/template"

	"sync"

	"sourcegraph.com/sourcegraph/srcgraph/config"
	"sourcegraph.com/sourcegraph/srcgraph/dep2"
	"sourcegraph.com/sourcegraph/srcgraph/toolchain"
)

func init() {
	toolchain.Register("ruby", DefaultRubyVersion)
	config.Register("ruby", &Config{})
}

type Ruby struct {
	Version        string
	StdlibCloneURL string

	// resolveCache maps gem name to resolved dep target.
	resolveCache   map[string]*dep2.ResolvedTarget
	resolveCacheMu sync.Mutex
}

var DefaultRubyVersion = &Ruby{
	Version:        "2.0.0-p481",
	StdlibCloneURL: "git://github.com/ruby/ruby.git",
}

func (v *Ruby) baseDockerfile() ([]byte, error) {
	var buf bytes.Buffer
	err := template.Must(template.New("").Parse(baseDockerfile)).Execute(&buf, struct {
		Ruby   *Ruby
		GOPATH string
	}{
		Ruby: v,
	})
	if err != nil {
		return nil, err
	}

	return buf.Bytes(), nil
}

const baseDockerfile = `FROM ubuntu:14.04
RUN apt-get update -qq
RUN apt-get install -qqy curl

# Install Ruby {{.Ruby.Version}}
RUN curl -L https://get.rvm.io | bash -s stable
ENV PATH /usr/local/rvm/bin:$PATH
RUN rvm requirements
RUN rvm install {{.Ruby.Version}}
RUN rvm {{.Ruby.Version}} do gem install bundler --no-ri --no-rdoc

# Lots of gemspecs run git to list files, so it's necessary.
RUN apt-get install -qqy git
`

type Config struct {
	// OmitStdlib is whether to NOT load the analyzed graph of the Ruby standard
	// library when analyzing this repository. It should be true when analyzing
	// the Ruby stdlib itself.
	OmitStdlib bool
}

func (v *Ruby) rubyConfig(c *config.Repository) *Config {
	rubyConfig, _ := c.Global["ruby"].(*Config)
	if rubyConfig == nil {
		rubyConfig = new(Config)
	}
	return rubyConfig
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"github.com/Sirupsen/logrus"
	"github.com/garyburd/redigo/redis"
	"time"
)

type InterfaceNotification struct {
	Type      string
	Message   string
	OrgID     string
	Payload   interface{}
	Timestamp time.Time
}

type RedisNotificationHandler struct {
	CacheStore *RedisClusterStorageManager
}

const (
	UIChanName string = "dashboard.ui.messages"
)

func (u *RedisNotificationHandler) Start() {
	go u.StartUIPubSubConn()

	// go func() {
	// 	for {
	// 		n := InterfaceNotification{
	// 			Type:      "debug",
	// 			Message:   "Test Debug",
	// 			OrgID:     "53ac07777cbb8c2d53000002",
	// 			Timestamp: time.Now(),
	// 		}

	// 		u.Notify(n)
	// 		time.Sleep(time.Second * 5)
	// 	}

	// }()
}

func (u *RedisNotificationHandler) Notify(n InterfaceNotification) error {
	json_err, encErr := json.Marshal(n)
	if encErr != nil {
		return encErr
	}

	if u.CacheStore != nil {
		u.CacheStore.Publish(UIChanName, string(json_err))
	}

	return nil
}

func (u *RedisNotificationHandler) StartUIPubSubConn() {
	u.CacheStore = &RedisClusterStorageManager{KeyPrefix: "gateway-notifications:", HashKeys: false}
	u.CacheStore.Connect()
	// On message, synchronize
	for {
		err := u.CacheStore.StartPubSubHandler(UIChanName, u.HandleIncommingRedisMessage)
		if err != nil {
			log.WithFields(logrus.Fields{
				"prefix": "log-notifications",
				"err":    err,
			}).Error("Connection to Redis failed, reconnect in 10s")

			time.Sleep(10 * time.Second)
			log.WithFields(logrus.Fields{
				"prefix": "log-notifications",
			}).Warning("Reconnecting")

			u.CacheStore.Connect()
			u.CacheStore.StartPubSubHandler(UIChanName, u.HandleIncommingRedisMessage)
		}

	}
}

func (u *RedisNotificationHandler) HandleIncommingRedisMessage(message redis.Message) {
	log.Debug("Inbound message received")
}
<commit_msg>Removed comment code<commit_after>package main

import (
	"encoding/json"
	"github.com/Sirupsen/logrus"
	"github.com/garyburd/redigo/redis"
	"time"
)

type InterfaceNotification struct {
	Type      string
	Message   string
	OrgID     string
	Payload   interface{}
	Timestamp time.Time
}

type RedisNotificationHandler struct {
	CacheStore *RedisClusterStorageManager
}

const (
	UIChanName string = "dashboard.ui.messages"
)

func (u *RedisNotificationHandler) Start() {
	go u.StartUIPubSubConn()
}

func (u *RedisNotificationHandler) Notify(n InterfaceNotification) error {
	json_err, encErr := json.Marshal(n)
	if encErr != nil {
		return encErr
	}

	if u.CacheStore != nil {
		u.CacheStore.Publish(UIChanName, string(json_err))
	}

	return nil
}

func (u *RedisNotificationHandler) StartUIPubSubConn() {
	u.CacheStore = &RedisClusterStorageManager{KeyPrefix: "gateway-notifications:", HashKeys: false}
	u.CacheStore.Connect()
	// On message, synchronize
	for {
		err := u.CacheStore.StartPubSubHandler(UIChanName, u.HandleIncommingRedisMessage)
		if err != nil {
			log.WithFields(logrus.Fields{
				"prefix": "log-notifications",
				"err":    err,
			}).Error("Connection to Redis failed, reconnect in 10s")

			time.Sleep(10 * time.Second)
			log.WithFields(logrus.Fields{
				"prefix": "log-notifications",
			}).Warning("Reconnecting")

			u.CacheStore.Connect()
			u.CacheStore.StartPubSubHandler(UIChanName, u.HandleIncommingRedisMessage)
		}

	}
}

func (u *RedisNotificationHandler) HandleIncommingRedisMessage(message redis.Message) {
	log.Debug("Inbound message received")
}
<|endoftext|>


<commit_before>package smtpd

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime"
	"mime/multipart"
	"net/mail"
	"net/textproto"
	"strings"
)

// Message is a nicely packaged representation of the
// recieved message
type Message struct {
	To      []*mail.Address
	From    *mail.Address
	Headers map[string]string
	Subject string
	Body    []*Part
	RawBody []byte

	// meta info
	Logger *log.Logger
}

// Part represents a single part of the message
type Part struct {
	part     *multipart.Part
	Body     []byte
	Children []*Part
}

func (m *Message) ID() string {
	return "not-implemented"
}

// Plain returns the text/plain content of the message, if any
func (m *Message) Plain() ([]byte, error) {
	return m.FindBody("text/plain")
}

// HTML returns the text/html content of the message, if any
func (m *Message) HTML() ([]byte, error) {
	return m.FindBody("text/html")
}

func findTypeInParts(contentType string, parts []*Part) *Part {
	for _, p := range parts {
		mediaType, _, err := mime.ParseMediaType(p.part.Header.Get("Content-Type"))
		if err == nil && mediaType == contentType {
			return p
		}
	}
	return nil
}

// FindByType finds the first part of the message with the specified Content-Type
func (m *Message) FindBody(contentType string) ([]byte, error) {

	// XXX: this is awful and can be done much better
	var alternative *Part
	if mixed := findTypeInParts("multipart/mixed", m.Body); mixed != nil {
		alternative = findTypeInParts("multipart/alternative", []*Part{mixed})
	} else {
		alternative = findTypeInParts("multipart/alternative", m.Body)
	}

	if alternative == nil {
		return nil, fmt.Errorf("No %v content found", contentType)
	}

	part := findTypeInParts(contentType, alternative.Children)
	if part == nil {
		return nil, fmt.Errorf("No %v content found", contentType)
	}

	return part.Body, nil
}

func parseContent(header textproto.MIMEHeader, content io.Reader) ([]*Part, error) {

	mediaType, params, err := mime.ParseMediaType(header.Get("Content-Type"))
	if err != nil && err.Error() == "mime: no media type" {
		mediaType = "text/plain"
	} else if err != nil {
		return nil, fmt.Errorf("Media Type error: %v", err)
	}

	var parts []*Part

	if strings.HasPrefix(mediaType, "multipart/") {

		mr := multipart.NewReader(content, params["boundary"])
		for {
			p, err := mr.NextPart()
			if err == io.EOF {
				break
			} else if err != nil {
				return nil, fmt.Errorf("MIME error: %v", err)
			}

			// XXX: LimitReader?
			slurp, err := ioutil.ReadAll(p)
			if err != nil {
				log.Fatal(err)
			}
			multi := &Part{part: p, Body: slurp}

			subParts, err := parseContent(p.Header, bytes.NewBuffer(slurp))
			if err != nil {
				return nil, err
			}

			multi.Children = subParts
			parts = append(parts, multi)

		}
	} else {
		// XXX: LimitReader?
		slurp, err := ioutil.ReadAll(content)
		if err != nil {
			log.Fatal(err)
		}

		parts = append(parts, &Part{
			part: &multipart.Part{
				Header: header,
			},
			Body: slurp,
		})
	}

	return parts, nil
}

// parseBody unwraps the body io.Reader into a set of *Part structs
func parseBody(m *mail.Message) ([]byte, []*Part, error) {

	// XXX: LimitReader?
	mbody, err := ioutil.ReadAll(m.Body)
	if err != nil {
		return []byte{}, []*Part{}, err
	}
	buf := bytes.NewBuffer(mbody)

	parts, err := parseContent(textproto.MIMEHeader(m.Header), buf)
	if err != nil {
		return nil, nil, err
	}

	return mbody, parts, nil
}

// NewMessage creates a Message from a data blob
func NewMessage(data []byte, logger *log.Logger) (*Message, error) {
	m, err := mail.ReadMessage(bytes.NewBuffer(data))
	if err != nil {
		return nil, err
	}

	to, err := m.Header.AddressList("to")
	if err != nil {
		return nil, err
	}

	from, err := m.Header.AddressList("from")
	if err != nil {
		return nil, err
	}

	header := make(map[string]string)

	for k, v := range m.Header {
		if len(v) == 1 {
			header[k] = v[0]
		}
	}

	raw, parts, err := parseBody(m)
	if err != nil {
		return nil, err
	}

	return &Message{
		To:      to,
		From:    from[0],
		Headers: header,
		Subject: m.Header.Get("subject"),
		Body:    parts,
		RawBody: raw,
		Logger:  logger,
	}, nil

}
<commit_msg>no fatal loggin<commit_after>package smtpd

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime"
	"mime/multipart"
	"net/mail"
	"net/textproto"
	"strings"
)

// Message is a nicely packaged representation of the
// recieved message
type Message struct {
	To      []*mail.Address
	From    *mail.Address
	Headers map[string]string
	Subject string
	Body    []*Part
	RawBody []byte

	// meta info
	Logger *log.Logger
}

// Part represents a single part of the message
type Part struct {
	part     *multipart.Part
	Body     []byte
	Children []*Part
}

func (m *Message) ID() string {
	return "not-implemented"
}

// Plain returns the text/plain content of the message, if any
func (m *Message) Plain() ([]byte, error) {
	return m.FindBody("text/plain")
}

// HTML returns the text/html content of the message, if any
func (m *Message) HTML() ([]byte, error) {
	return m.FindBody("text/html")
}

func findTypeInParts(contentType string, parts []*Part) *Part {
	for _, p := range parts {
		mediaType, _, err := mime.ParseMediaType(p.part.Header.Get("Content-Type"))
		if err == nil && mediaType == contentType {
			return p
		}
	}
	return nil
}

// FindByType finds the first part of the message with the specified Content-Type
func (m *Message) FindBody(contentType string) ([]byte, error) {

	// XXX: this is awful and can be done much better
	var alternative *Part
	if mixed := findTypeInParts("multipart/mixed", m.Body); mixed != nil {
		alternative = findTypeInParts("multipart/alternative", []*Part{mixed})
	} else {
		alternative = findTypeInParts("multipart/alternative", m.Body)
	}

	if alternative == nil {
		return nil, fmt.Errorf("No %v content found", contentType)
	}

	part := findTypeInParts(contentType, alternative.Children)
	if part == nil {
		return nil, fmt.Errorf("No %v content found", contentType)
	}

	return part.Body, nil
}

func parseContent(header textproto.MIMEHeader, content io.Reader) ([]*Part, error) {

	mediaType, params, err := mime.ParseMediaType(header.Get("Content-Type"))
	if err != nil && err.Error() == "mime: no media type" {
		mediaType = "text/plain"
	} else if err != nil {
		return nil, fmt.Errorf("Media Type error: %v", err)
	}

	var parts []*Part

	if strings.HasPrefix(mediaType, "multipart/") {

		mr := multipart.NewReader(content, params["boundary"])
		for {
			p, err := mr.NextPart()
			if err == io.EOF {
				break
			} else if err != nil {
				return nil, fmt.Errorf("MIME error: %v", err)
			}

			// XXX: LimitReader?
			slurp, err := ioutil.ReadAll(p)
			if err != nil {
				return nil, err
			}
			multi := &Part{part: p, Body: slurp}

			subParts, err := parseContent(p.Header, bytes.NewBuffer(slurp))
			if err != nil {
				return nil, err
			}

			multi.Children = subParts
			parts = append(parts, multi)

		}
	} else {
		// XXX: LimitReader?
		slurp, err := ioutil.ReadAll(content)
		if err != nil {
			return nil, err
		}

		parts = append(parts, &Part{
			part: &multipart.Part{
				Header: header,
			},
			Body: slurp,
		})
	}

	return parts, nil
}

// parseBody unwraps the body io.Reader into a set of *Part structs
func parseBody(m *mail.Message) ([]byte, []*Part, error) {

	// XXX: LimitReader?
	mbody, err := ioutil.ReadAll(m.Body)
	if err != nil {
		return []byte{}, []*Part{}, err
	}
	buf := bytes.NewBuffer(mbody)

	parts, err := parseContent(textproto.MIMEHeader(m.Header), buf)
	if err != nil {
		return nil, nil, err
	}

	return mbody, parts, nil
}

// NewMessage creates a Message from a data blob
func NewMessage(data []byte, logger *log.Logger) (*Message, error) {
	m, err := mail.ReadMessage(bytes.NewBuffer(data))
	if err != nil {
		return nil, err
	}

	to, err := m.Header.AddressList("to")
	if err != nil {
		return nil, err
	}

	from, err := m.Header.AddressList("from")
	if err != nil {
		return nil, err
	}

	header := make(map[string]string)

	for k, v := range m.Header {
		if len(v) == 1 {
			header[k] = v[0]
		}
	}

	raw, parts, err := parseBody(m)
	if err != nil {
		return nil, err
	}

	return &Message{
		To:      to,
		From:    from[0],
		Headers: header,
		Subject: m.Header.Get("subject"),
		Body:    parts,
		RawBody: raw,
		Logger:  logger,
	}, nil

}
<|endoftext|>


<commit_before>package sample

import "github.com/Shippable/sample_go/newmath"

var result float64

func Setup() {
	result = newmath.Sqrt(4)
}

func GetResult() float64 {
	return result
}
<commit_msg>refer to self<commit_after>package sample

import "github.com/slothbit/sample_go/newmath"

var result float64

func Setup() {
	result = newmath.Sqrt(4)
}

func GetResult() float64 {
	return result
}
<|endoftext|>


<commit_before>// The MIT License (MIT)
//
// Copyright (c) 2014, 2015 Jamie Alquiza
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
package main

import (
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"os/signal"
	"sort"
	"strconv"
	"strings"
	"syscall"
	"time"

	kafka "github.com/Shopify/sarama"
)

var (
	// Configs.
	brokers        []string
	topic          string
	msgSize        int
	msgRate        int64
	batchSize      int
	compressionOpt string
	compression    kafka.CompressionCodec
	clients        int
	producers      int
	noop           bool

	// Character selection for random messages.
	chars = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890!@#$^&*(){}][:<>.")

	// Counters / misc.
	signals        = make(chan os.Signal)
	killClients    = make(chan bool, 24)
	sentCntr       = make(chan int64, 1)
	latency        []float64
	latencies      = make(chan float64, 1)
	resetLatencies = make(chan bool, 1)
)

func init() {
	flag.StringVar(&topic, "topic", "sangrenel", "Topic to publish to")
	flag.IntVar(&msgSize, "size", 300, "Message size in bytes")
	flag.Int64Var(&msgRate, "rate", 100000000, "Apply a global message rate limit")
	flag.IntVar(&batchSize, "batch", 0, "Max messages per batch. Defaults to unlimited (0).")
	flag.StringVar(&compressionOpt, "compression", "none", "Message compression: none, gzip, snappy")
	flag.BoolVar(&noop, "noop", false, "Test message generation performance, do not transmit messages")
	flag.IntVar(&clients, "clients", 1, "Number of Kafka client workers")
	flag.IntVar(&producers, "producers", 5, "Number of producer instances per client")
	brokerString := flag.String("brokers", "localhost:9092", "Comma delimited list of Kafka brokers")
	flag.Parse()

	brokers = strings.Split(*brokerString, ",")

	switch compressionOpt {
	case "gzip":
		compression = kafka.CompressionGZIP
	case "snappy":
		compression = kafka.CompressionSnappy
	case "none":
		compression = kafka.CompressionNone
	default:
		fmt.Printf("Invalid compression option: %s\n", compressionOpt)
		os.Exit(1)
	}

	sentCntr <- 0
}

// clientProducer generates random messages and writes to Kafka.
// Workers track and limit message rates using incrSent() and fetchSent().
// Default 5 instances of clientProducer are created under each Kafka client.
func clientProducer(c kafka.Client) {
	producer, err := kafka.NewSyncProducerFromClient(c)
	if err != nil {
		log.Println(err.Error())
	}
	defer producer.Close()

	source := rand.NewSource(time.Now().UnixNano())
	generator := rand.New(source)
	msgData := make([]byte, msgSize)

	// Use a local accumulator then periodically update global counter.
	// Global counter can become a bottleneck with too many threads.
	tick := time.Tick(3 * time.Millisecond)
	var n int64

	for {
		// Message rate limit works by having all clientProducer loops incrementing
		// a global counter and tracking the aggregate per-second progress.
		// If the configured rate is met, the worker will sleep
		// for the remainder of the 1 second window.
		rateEnd := time.Now().Add(time.Second)
		countStart := fetchSent()
		var start time.Time
		for fetchSent()-countStart < msgRate {
			randMsg(msgData, *generator)
			msg := &kafka.ProducerMessage{Topic: topic, Value: kafka.ByteEncoder(msgData)}

			start = time.Now()
			_, _, err = producer.SendMessage(msg)
			if err != nil {
				log.Println(err)
			} else {
				// Increment global sent count and fire off
				// time since start value into the latency channel.
				n++
				select {
				case <-tick:
					incrSent(n)
					n = 0
				default:
					break
				}
				latencies <- time.Since(start).Seconds() * 1000
			}
		}
		// If the global per-second rate limit was met,
		// the inner loop breaks and the outer loop sleeps for the second remainder.
		time.Sleep(rateEnd.Sub(time.Now()) + time.Since(start))
	}
}

// clientDummyProducer is a dummy function that kafkaClient calls if noop is True.
// It is used in place of starting actual Kafka client connections to test message creation performance.
func clientDummyProducer() {
	source := rand.NewSource(time.Now().UnixNano())
	generator := rand.New(source)
	msg := make([]byte, msgSize)

	tick := time.Tick(10 * time.Millisecond)
	var n int64

	for {
		randMsg(msg, *generator)
		n++
		select {
		case <-tick:
			incrSent(n)
			n = 0
		default:
			break
		}
	}
}

// kafkaClient initializes a connection to a Kafka cluster and
// initializes one or more clientProducer() (producer instances).
func kafkaClient(n int) {
	switch noop {
	// If not noop, actually fire up Kafka connections and send messages.
	case false:
		cId := "client_" + strconv.Itoa(n)

		conf := kafka.NewConfig()
		if compression != kafka.CompressionNone {
			conf.Producer.Compression = compression
		}
		conf.Producer.Flush.MaxMessages = batchSize

		conf.Producer.MaxMessageBytes = msgSize + 50

		client, err := kafka.NewClient(brokers, conf)
		if err != nil {
			log.Println(err)
			os.Exit(1)
		} else {
			log.Printf("%s connected\n", cId)
		}
		for i := 0; i < producers; i++ {
			go clientProducer(client)
		}
	// If noop, we're not creating connections at all.
	// Just generate messages and burn CPU.
	default:
		for i := 0; i < producers; i++ {
			go clientDummyProducer()
		}
	}
	<-killClients
}

// Returns a random message generated from the chars byte slice.
// Message length of m bytes as defined by msgSize.
func randMsg(m []byte, generator rand.Rand) {
	for i := range m {
		m[i] = chars[generator.Intn(len(chars))]
	}
}

// Global counter functions.
func incrSent(n int64) {
	i := <-sentCntr
	sentCntr <- i + n
}
func fetchSent() int64 {
	i := <-sentCntr
	sentCntr <- i
	return i
}

// latencyAggregator receives latency values captured by all producer goroutines.
// May want to do something smart about this to limit time to sort
// huge slices in high-throughput configurations.
func latencyAggregator() {
	for {
		select {
		case i := <-latencies:
			latency = append(latency, i)
		case <-resetLatencies:
			latency = latency[:0]
		}
	}
}

// Calculates aggregate raw message output in human / network units.
func calcOutput(n int64) (float64, string) {
	m := (float64(n) / 5) * float64(msgSize)
	var o string
	switch {
	case m >= 131072:
		o = strconv.FormatFloat(m/131072, 'f', 0, 64) + "Mb/sec"
	case m < 131072:
		o = strconv.FormatFloat(m/1024, 'f', 0, 64) + "KB/sec"
	}
	return m, o
}

// Fetches & resets current latencies set held by 'latencyAggregator()'.
// Sorts then averages the top 10% worst latencies.
func calcLatency() float64 {
	var avg float64
	// With 'noop', we don't have latencies to operate on.
	switch noop {
	case true:
		break
	default:
		// Fetch values.
		lat := latency
		// Issue the current values to be cleared.
		resetLatencies <- true
		// Sort and sum values.
		sort.Float64s(lat)
		var sum float64
		// Get top 10%, sum values.
		topn := int(float64(len(lat)) * 0.90)
		for i := topn; i < len(lat); i++ {
			sum += lat[i]
		}
		// Calc average.
		avg = sum / float64(len(lat)-topn)
	}
	return avg
}

func main() {
	// Listens for signals.
	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
	// Fire up misc. tasks.
	go latencyAggregator()
	if graphiteIp != "" {
		go graphiteWriter()
	}

	// Print Sangrenel startup info.
	fmt.Println("\n::: Sangrenel :::")
	fmt.Printf("\nStarting %d client workers, %d producers per worker\n", clients, producers)
	fmt.Printf("Message size %d bytes, %d message limit per batch\n", msgSize, batchSize)
	switch compressionOpt {
	case "none":
		fmt.Println("Compression: none\n")
	case "gzip":
		fmt.Println("Compression: GZIP\n")
	case "snappy":
		fmt.Println("Compression: Snappy\n")
	}

	// Start client workers.
	for i := 0; i < clients; i++ {
		go kafkaClient(i + 1)
	}

	// Start Sangrenel periodic info output.
	tick := time.Tick(5 * time.Second)

	var currCnt, lastCnt int64
	for {
		select {
		case <-tick:
			// Set last and current to last read sent count.
			lastCnt = currCnt

			// Get actual current sent count, then delta from last count.
			// Delta is divided by update interval (5s) for per-second rate over a window.
			currCnt = fetchSent()
			deltaCnt := currCnt - lastCnt

			outputBytes, outputString := calcOutput(deltaCnt)

			// Update the metrics map which is also passed to the Graphite writer.
			metrics["rate"] = float64(deltaCnt / 5)
			metrics["10p"] = calcLatency() // Well, this technically appends a small latency to the 5s interval.
			metrics["output"] = outputBytes
			now := time.Now()
			ts := float64(now.Unix())
			metrics["timestamp"] = ts
			if graphiteIp != "" {
				metricsOutgoing <- metrics
			}

			log.Printf("Generating %s @ %.0f messages/sec | topic: %s | %.2fms top 10%% latency\n",
				outputString,
				metrics["rate"],
				topic,
				metrics["10p"])

		// Waits for signals. Currently just brutally kills Sangrenel.
		case <-signals:
			fmt.Println("Killing Connections")
			for i := 0; i < clients; i++ {
				killClients <- true
			}
			close(killClients)
			time.Sleep(2 * time.Second)
			os.Exit(0)
		}
	}
}
<commit_msg>switched metrics to tachymeter<commit_after>// The MIT License (MIT)
//
// Copyright (c) 2014, 2015 Jamie Alquiza
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
package main

import (
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"
	"time"

	kafka "github.com/Shopify/sarama"
	"github.com/jamiealquiza/tachymeter"
)

var (
	// Configs.
	brokers        []string
	topic          string
	msgSize        int
	msgRate        int64
	batchSize      int
	compressionOpt string
	compression    kafka.CompressionCodec
	clients        int
	producers      int
	noop           bool

	// Character selection for random messages.
	chars = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890!@#$^&*(){}][:<>.")

	// Counters / misc.
	signals     = make(chan os.Signal)
	killClients = make(chan bool, 24)
	sentCntr    = make(chan int64, 1)
)

func init() {
	flag.StringVar(&topic, "topic", "sangrenel", "Topic to publish to")
	flag.IntVar(&msgSize, "size", 300, "Message size in bytes")
	flag.Int64Var(&msgRate, "rate", 100000000, "Apply a global message rate limit")
	flag.IntVar(&batchSize, "batch", 0, "Max messages per batch. Defaults to unlimited (0).")
	flag.StringVar(&compressionOpt, "compression", "none", "Message compression: none, gzip, snappy")
	flag.BoolVar(&noop, "noop", false, "Test message generation performance, do not transmit messages")
	flag.IntVar(&clients, "clients", 1, "Number of Kafka client workers")
	flag.IntVar(&producers, "producers", 5, "Number of producer instances per client")
	brokerString := flag.String("brokers", "localhost:9092", "Comma delimited list of Kafka brokers")
	flag.Parse()

	brokers = strings.Split(*brokerString, ",")

	switch compressionOpt {
	case "gzip":
		compression = kafka.CompressionGZIP
	case "snappy":
		compression = kafka.CompressionSnappy
	case "none":
		compression = kafka.CompressionNone
	default:
		fmt.Printf("Invalid compression option: %s\n", compressionOpt)
		os.Exit(1)
	}

	sentCntr <- 0
}

// clientProducer generates random messages and writes to Kafka.
// Workers track and limit message rates using incrSent() and fetchSent().
// Default 5 instances of clientProducer are created under each Kafka client.
func clientProducer(c kafka.Client, t *tachymeter.Tachymeter) {
	producer, err := kafka.NewSyncProducerFromClient(c)
	if err != nil {
		log.Println(err.Error())
	}
	defer producer.Close()

	source := rand.NewSource(time.Now().UnixNano())
	generator := rand.New(source)
	msgData := make([]byte, msgSize)

	// Use a local accumulator then periodically update global counter.
	// Global counter can become a bottleneck with too many threads.
	//tick := time.Tick(2 * time.Millisecond)
	var n int64
	var times [10]time.Duration

	for {
		// Message rate limit works by having all clientProducer loops incrementing
		// a global counter and tracking the aggregate per-second progress.
		// If the configured rate is met, the worker will sleep
		// for the remainder of the 1 second window.
		rateEnd := time.Now().Add(time.Second)
		countStart := fetchSent()
		var start time.Time
		for fetchSent()-countStart < msgRate {
			randMsg(msgData, *generator)
			msg := &kafka.ProducerMessage{Topic: topic, Value: kafka.ByteEncoder(msgData)}

			start = time.Now()
			_, _, err = producer.SendMessage(msg)
			if err != nil {
				log.Println(err)
			} else {
				// Increment global counter and
				// tachymeter every 10 messages.
				n++
				times[n-1] = time.Since(start)
				if n == 10 {
					incrSent(10)
					t.AddCount(10)
					for _, ts := range times {
						t.AddTime(ts)
					}
					n = 0
				}
			}
		}
		// If the global per-second rate limit was met,
		// the inner loop breaks and the outer loop sleeps for the second remainder.
		time.Sleep(rateEnd.Sub(time.Now()) + time.Since(start))
	}
}

// clientDummyProducer is a dummy function that kafkaClient calls if noop is True.
// It is used in place of starting actual Kafka client connections to test message creation performance.
func clientDummyProducer() {
	source := rand.NewSource(time.Now().UnixNano())
	generator := rand.New(source)
	msg := make([]byte, msgSize)

	tick := time.Tick(10 * time.Millisecond)
	var n int64

	for {
		randMsg(msg, *generator)
		n++
		select {
		case <-tick:
			incrSent(n)
			n = 0
		default:
			break
		}
	}
}

// kafkaClient initializes a connection to a Kafka cluster and
// initializes one or more clientProducer() (producer instances).
func kafkaClient(n int, t *tachymeter.Tachymeter) {
	switch noop {
	// If not noop, actually fire up Kafka connections and send messages.
	case false:
		cId := "client_" + strconv.Itoa(n)

		conf := kafka.NewConfig()
		if compression != kafka.CompressionNone {
			conf.Producer.Compression = compression
		}
		conf.Producer.Flush.MaxMessages = batchSize

		conf.Producer.MaxMessageBytes = msgSize + 50

		client, err := kafka.NewClient(brokers, conf)
		if err != nil {
			log.Println(err)
			os.Exit(1)
		} else {
			log.Printf("%s connected\n", cId)
		}
		for i := 0; i < producers; i++ {
			go clientProducer(client, t)
		}
	// If noop, we're not creating connections at all.
	// Just generate messages and burn CPU.
	default:
		for i := 0; i < producers; i++ {
			go clientDummyProducer()
		}
	}
	<-killClients
}

// Returns a random message generated from the chars byte slice.
// Message length of m bytes as defined by msgSize.
func randMsg(m []byte, generator rand.Rand) {
	for i := range m {
		m[i] = chars[generator.Intn(len(chars))]
	}
}

// Global counter functions.
func incrSent(n int64) {
	i := <-sentCntr
	sentCntr <- i + n
}
func fetchSent() int64 {
	i := <-sentCntr
	sentCntr <- i
	return i
}

// Calculates aggregate raw message output in human / network units.
func calcOutput(n int64) (float64, string) {
	m := (float64(n) / 5) * float64(msgSize)
	var o string
	switch {
	case m >= 131072:
		o = strconv.FormatFloat(m/131072, 'f', 0, 64) + "Mb/sec"
	case m < 131072:
		o = strconv.FormatFloat(m/1024, 'f', 0, 64) + "KB/sec"
	}
	return m, o
}

func main() {
	// Listens for signals.
	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)

	if graphiteIp != "" {
		go graphiteWriter()
	}

	// Print Sangrenel startup info.
	fmt.Println("\n::: Sangrenel :::")
	fmt.Printf("\nStarting %d client workers, %d producers per worker\n", clients, producers)
	fmt.Printf("Message size %d bytes, %d message limit per batch\n", msgSize, batchSize)
	switch compressionOpt {
	case "none":
		fmt.Println("Compression: none")
	case "gzip":
		fmt.Println("Compression: GZIP")
	case "snappy":
		fmt.Println("Compression: Snappy")
	}

	t := tachymeter.New(&tachymeter.Config{Size: 1000000, Safe: true})

	// Start client workers.
	for i := 0; i < clients; i++ {
		go kafkaClient(i+1, t)
	}

	// Start Sangrenel periodic info output.
	tick := time.Tick(5 * time.Second)

	var currCnt, lastCnt int64
	start := time.Now()
	for {
		select {
		case <-tick:
			// Set tachymeter wall time.
			t.SetWallTime(time.Since(start))

			// Set last and current to last read sent count.
			lastCnt = currCnt

			// Get actual current sent count, then delta from last count.
			// Delta is divided by update interval (5s) for per-second rate over a window.
			currCnt = fetchSent()
			deltaCnt := currCnt - lastCnt

			stats := t.Calc()
			t.Reset()

			outputBytes, outputString := calcOutput(deltaCnt)

			// Update the metrics map for the Graphite writer.
			metrics["rate"] = stats.Rate.Second
			metrics["output"] = outputBytes
			metrics["5p"] = (float64(stats.Time.Long5p.Nanoseconds()) / 1000) / 1000
			// Add ts for Graphite.
			now := time.Now()
			ts := float64(now.Unix())
			metrics["timestamp"] = ts

			if graphiteIp != "" {
				metricsOutgoing <- metrics
			}

			log.Printf("\n\nGenerating %s @ %.0f messages/sec | topic: %s | %.2fms top 10%% latency\n",
				outputString,
				metrics["rate"],
				topic,
				metrics["5p"])

			stats.Dump()

			// Reset interval time.
			start = time.Now()

		// Waits for signals. Currently just brutally kills Sangrenel.
		case <-signals:
			fmt.Println("Killing Connections")
			for i := 0; i < clients; i++ {
				killClients <- true
			}
			close(killClients)
			time.Sleep(2 * time.Second)
			os.Exit(0)
		}
	}
}
<|endoftext|>


<commit_before>package fakemain

import (
	"time"

	"github.com/alexcarol/bicing-oracle/station-state/datasource"
	"github.com/alexcarol/bicing-oracle/station-state/parser"
	"github.com/alexcarol/bicing-oracle/station-state/repository"
)

func main() {
	ticker := time.NewTicker(2 * time.Second)
	quit := make(chan struct{})

	storage := repository.NewStorage()

	go func() {
		for {
			select {
			case <-ticker.C:
				data, err := parser.ParseXML(datasource.FixtureData())
				if err != nil {
					panic("Error parsing xml")
				}
				storage.PersistCollection(data)
			case <-quit:
				ticker.Stop()
				return
			}
		}
	}()

	<-quit
}
<commit_msg>Deleted unused file<commit_after><|endoftext|>


<commit_before>package mallory

import (
	"errors"
	"github.com/justmao945/mallory/ssh"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/user"
)

//
type EngineSSH struct {
	Env *Env
	URL *url.URL
	Cli *ssh.Client
	Cfg *ssh.ClientConfig
	Dir *EngineDirect
}

// Create and initialize
func CreateEngineSSH(e *Env) (self *EngineSSH, err error) {
	self = &EngineSSH{
		Env: e,
		Cfg: &ssh.ClientConfig{},
	}
	// e.g.  ssh://user:passwd@192.168.1.1:1122
	self.URL, err = url.Parse(e.Remote)
	if err != nil {
		return
	}

	if self.URL.User != nil {
		self.Cfg.User = self.URL.User.Username()
	} else {
		u, err := user.Current()
		if err != nil {
			return self, err
		}
		// u.Name is the full name, should not be used
		self.Cfg.User = u.Username
	}

	// 1) try RSA keyring first
	for {
		id_rsa := os.ExpandEnv("$HOME/.ssh/id_rsa")
		pem, err := ioutil.ReadFile(id_rsa)
		if err != nil {
			break
		}
		signer, err := ssh.ParsePrivateKey(pem)
		if err != nil {
			break
		}
		self.Cfg.Auth = append(self.Cfg.Auth, ssh.PublicKeys(signer))
		// stop !!
		break
	}
	// 2) try password
	for {
		if self.URL.User == nil {
			break
		}
		if pass, ok := self.URL.User.Password(); ok {
			self.Cfg.Auth = append(self.Cfg.Auth, ssh.Password(pass))
		}
		// stop here!!
		break
	}

	if len(self.Cfg.Auth) == 0 {
		//TODO: keyboard intercative
		err = errors.New("Invalid auth method, please add password or generate ssh keys")
		return
	}

	self.Cli, err = ssh.Dial("tcp", self.URL.Host, self.Cfg)
	if err != nil {
		return
	}

	dial := func(network, addr string) (net.Conn, error) {
		return self.Cli.Dial(network, addr)
	}

	self.Dir = &EngineDirect{
		Tr: &http.Transport{Dial: dial},
	}
	return
}

func (self *EngineSSH) Serve(s *Session) {
	self.Dir.Serve(s)
}

func (self *EngineSSH) Connect(s *Session) {
	self.Dir.Connect(s)
}
<commit_msg>auto reconnect when disconnected<commit_after>package mallory

import (
	"errors"
	"github.com/justmao945/mallory/ssh"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/user"
)

//
type EngineSSH struct {
	Env *Env
	URL *url.URL
	Cli *ssh.Client
	Cfg *ssh.ClientConfig
	Dir *EngineDirect
}

// Create and initialize
func CreateEngineSSH(e *Env) (self *EngineSSH, err error) {
	self = &EngineSSH{
		Env: e,
		Cfg: &ssh.ClientConfig{},
	}
	// e.g.  ssh://user:passwd@192.168.1.1:1122
	self.URL, err = url.Parse(e.Remote)
	if err != nil {
		return
	}

	if self.URL.User != nil {
		self.Cfg.User = self.URL.User.Username()
	} else {
		u, err := user.Current()
		if err != nil {
			return self, err
		}
		// u.Name is the full name, should not be used
		self.Cfg.User = u.Username
	}

	// 1) try RSA keyring first
	for {
		id_rsa := os.ExpandEnv("$HOME/.ssh/id_rsa")
		pem, err := ioutil.ReadFile(id_rsa)
		if err != nil {
			break
		}
		signer, err := ssh.ParsePrivateKey(pem)
		if err != nil {
			break
		}
		self.Cfg.Auth = append(self.Cfg.Auth, ssh.PublicKeys(signer))
		// stop !!
		break
	}
	// 2) try password
	for {
		if self.URL.User == nil {
			break
		}
		if pass, ok := self.URL.User.Password(); ok {
			self.Cfg.Auth = append(self.Cfg.Auth, ssh.Password(pass))
		}
		// stop here!!
		break
	}

	if len(self.Cfg.Auth) == 0 {
		//TODO: keyboard intercative
		err = errors.New("Invalid auth method, please add password or generate ssh keys")
		return
	}

	self.Cli, err = ssh.Dial("tcp", self.URL.Host, self.Cfg)
	if err != nil {
		return
	}

	dial := func(network, addr string) (c net.Conn, err error) {
		for i := 0; i < 3; i++ {
			c, err = self.Cli.Dial(network, addr)
			// We want to reconnect the network when disconnected.
			// FIXME: unexported net.errClosing
			if err != nil && err.Error() == "use of closed network connection" {
				continue
			}
			// do not reconnect when no error or other errors
			break
		}
		return c, err
	}

	self.Dir = &EngineDirect{
		Tr: &http.Transport{Dial: dial},
	}
	return
}

func (self *EngineSSH) Serve(s *Session) {
	self.Dir.Serve(s)
}

func (self *EngineSSH) Connect(s *Session) {
	self.Dir.Connect(s)
}
<|endoftext|>


<commit_before>package readline

import (
	"io"
	"os"
	"sync"
)

var (
	Stdin  io.ReadCloser  = os.Stdin
	Stdout io.WriteCloser = os.Stdout
	Stderr io.WriteCloser = os.Stderr
)

var (
	std     *Instance
	stdOnce sync.Once
)

// global instance will not submit history automatic
func getInstance() *Instance {
	stdOnce.Do(func() {
		std, _ = NewEx(&Config{
			DisableAutoSaveHistory: true,
		})
	})
	return std
}

// let readline load history from filepath
// and try to persist history into disk
// set fp to "" to prevent readline persisting history to disk
// so the `AddHistory` will return nil error forever.
func SetHistoryPath(fp string) {
	ins := getInstance()
	cfg := ins.Config.Clone()
	cfg.HistoryFile = fp
	ins.SetConfig(cfg)
}

// set auto completer to global instance
func SetAutoComplete(completer AutoCompleter) {
	ins := getInstance()
	cfg := ins.Config.Clone()
	cfg.AutoComplete = completer
	ins.SetConfig(cfg)
}

// add history to global instance manually
// raise error only if `SetHistoryPath` is set with a non-empty path
func AddHistory(content string) error {
	ins := getInstance()
	return ins.SaveHistory(content)
}

// readline with global configs
func Line(prompt string) (string, error) {
	ins := getInstance()
	ins.SetPrompt(prompt)
	return ins.Readline()
}
<commit_msg>add readline.Password()<commit_after>package readline

import (
	"io"
	"os"
	"sync"
)

var (
	Stdin  io.ReadCloser  = os.Stdin
	Stdout io.WriteCloser = os.Stdout
	Stderr io.WriteCloser = os.Stderr
)

var (
	std     *Instance
	stdOnce sync.Once
)

// global instance will not submit history automatic
func getInstance() *Instance {
	stdOnce.Do(func() {
		std, _ = NewEx(&Config{
			DisableAutoSaveHistory: true,
		})
	})
	return std
}

// let readline load history from filepath
// and try to persist history into disk
// set fp to "" to prevent readline persisting history to disk
// so the `AddHistory` will return nil error forever.
func SetHistoryPath(fp string) {
	ins := getInstance()
	cfg := ins.Config.Clone()
	cfg.HistoryFile = fp
	ins.SetConfig(cfg)
}

// set auto completer to global instance
func SetAutoComplete(completer AutoCompleter) {
	ins := getInstance()
	cfg := ins.Config.Clone()
	cfg.AutoComplete = completer
	ins.SetConfig(cfg)
}

// add history to global instance manually
// raise error only if `SetHistoryPath` is set with a non-empty path
func AddHistory(content string) error {
	ins := getInstance()
	return ins.SaveHistory(content)
}

func Password(prompt string) ([]byte, error) {
	ins := getInstance()
	return ins.ReadPassword(prompt)
}

// readline with global configs
func Line(prompt string) (string, error) {
	ins := getInstance()
	ins.SetPrompt(prompt)
	return ins.Readline()
}
<|endoftext|>


<commit_before>package multihash

import (
	"errors"
)

// ErrSumNotSupported is returned when the Sum function code is not implemented
var ErrSumNotSupported = errors.New("Function not implemented. Complain to lib maintainer.")

var ErrLenTooLarge = errors.New("requested length was too large for digest")

// Sum obtains the cryptographic sum of a given buffer. The length parameter
// indicates the length of the resulting digest and passing a negative value
// use default length values for the selected hash function.
func Sum(data []byte, code uint64, length int) (Multihash, error) {
	// Get the algorithm.
	hasher, err := GetHasher(code)
	if err != nil {
		return nil, err
	}

	// Feed data in.
	hasher.Write(data)

	// Compute hash.
	//  Use a fixed size array here: should keep things on the stack.
	var space [64]byte
	sum := hasher.Sum(space[0:0])

	// Deal with any truncation.
	if length < 0 {
		length = hasher.Size()
	}
	if len(sum) < length {
		return nil, ErrLenTooLarge
	}
	if length >= 0 {
		sum = sum[:length]
	}

	// Put the multihash metainfo bytes at the front of the buffer.
	// FIXME: this does many avoidable allocations, but it's the shape of the Encode method arguments that forces this.
	return Encode(sum, code)
}
<commit_msg>Remove misplaced optimism about escape analysis.<commit_after>package multihash

import (
	"errors"
)

// ErrSumNotSupported is returned when the Sum function code is not implemented
var ErrSumNotSupported = errors.New("Function not implemented. Complain to lib maintainer.")

var ErrLenTooLarge = errors.New("requested length was too large for digest")

// Sum obtains the cryptographic sum of a given buffer. The length parameter
// indicates the length of the resulting digest and passing a negative value
// use default length values for the selected hash function.
func Sum(data []byte, code uint64, length int) (Multihash, error) {
	// Get the algorithm.
	hasher, err := GetHasher(code)
	if err != nil {
		return nil, err
	}

	// Feed data in.
	hasher.Write(data)

	// Compute final hash.
	//  A new slice is allocated.  FUTURE: see other comment below about allocation, and review together with this line to try to improve.
	sum := hasher.Sum(nil)

	// Deal with any truncation.
	if length < 0 {
		length = hasher.Size()
	}
	if len(sum) < length {
		return nil, ErrLenTooLarge
	}
	if length >= 0 {
		sum = sum[:length]
	}

	// Put the multihash metainfo bytes at the front of the buffer.
	//  FUTURE: try to improve allocations here.  Encode does several which are probably avoidable, but it's the shape of the Encode method arguments that forces this.
	return Encode(sum, code)
}
<|endoftext|>


<commit_before>package request

import (
	"crypto/md5"
	"errors"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"math/rand"
	"mime/multipart"
	"net"
	"net/http"
	"net/url"
	"os"
	"path"
	"strconv"
	"strings"

	"github.com/MG-RAST/Shock/shock-server/auth"
	"github.com/MG-RAST/Shock/shock-server/conf"
	e "github.com/MG-RAST/Shock/shock-server/errors"
	"github.com/MG-RAST/Shock/shock-server/logger"
	"github.com/MG-RAST/Shock/shock-server/node/archive"
	"github.com/MG-RAST/Shock/shock-server/node/file"
	"github.com/MG-RAST/Shock/shock-server/responder"
	"github.com/MG-RAST/Shock/shock-server/user"
	"github.com/MG-RAST/Shock/shock-server/util"
	"github.com/MG-RAST/golib/httpclient"
	"github.com/MG-RAST/golib/stretchr/goweb/context"
	"github.com/jum/tinyftp"
)

type checkSumCom struct {
	buf      []byte
	n        int
	checksum string
}

func Log(req *http.Request) {
	host, _, _ := net.SplitHostPort(req.RemoteAddr)
	// failed attempt to get the host in ipv4
	//addrs, _ := net.LookupIP(host)
	//fmt.Println(addrs)
	suffix := ""
	if _, auth := req.Header["Authorization"]; auth {
		suffix += " AUTH"
	}

	if l, has := req.Header["Content-Length"]; has {
		suffix += " Content-Length: " + l[0]
	}
	url := ""
	if req.URL.RawQuery != "" {
		url = fmt.Sprintf("%s %s?%s", req.Method, req.URL.Path, req.URL.RawQuery)
	} else {
		url = fmt.Sprintf("%s %s", req.Method, req.URL.Path)
	}
	logger.Infof("%s \"%s%s\"", host, url, suffix)
}

// Authenticate _
func Authenticate(req *http.Request) (u *user.User, err error) {
	if _, ok := req.Header["Authorization"]; !ok {
		err = errors.New(e.NoAuth)
		return
	}
	header := req.Header.Get("Authorization")
	u, err = auth.Authenticate(header)
	return
}

// AuthError _
func AuthError(err error, ctx context.Context) error {

	if conf.DEBUG_AUTH {
		return responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
	}

	if err.Error() == e.InvalidAuth {
		return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid authorization header or content")
	}
	err_msg := "Error at Auth: " + err.Error()
	logger.Error(err_msg)
	return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
}

// helper function to create a node from an http data post
func DataUpload(r *http.Request) (params map[string]string, files file.FormFiles, err error) {
	params = make(map[string]string)
	files = make(file.FormFiles)
	tmpPath := fmt.Sprintf("%s/temp/%d%d", conf.PATH_DATA, rand.Int(), rand.Int())

	files["upload"] = file.FormFile{Name: "filename", Path: tmpPath, Checksum: make(map[string]string)}
	if tmpFile, err := os.Create(tmpPath); err == nil {
		defer tmpFile.Close()
		md5h := md5.New()
		dst := io.MultiWriter(tmpFile, md5h)
		if _, err = io.Copy(dst, r.Body); err != nil {
			return nil, nil, err
		}
		files["upload"].Checksum["md5"] = fmt.Sprintf("%x", md5h.Sum(nil))
	} else {
		return nil, nil, err
	}

	return
}

// ParseMultipartForm helper function for create & update
func ParseMultipartForm(r *http.Request) (params map[string]string, files file.FormFiles, err error) {

	params = make(map[string]string)
	files = make(file.FormFiles)
	reader, err := r.MultipartReader()
	if err != nil {
		err = fmt.Errorf("(ParseMultipartForm) MultipartReader returned: %s", err.Error())
		return
	}

	tmpPath := ""
	for { // read until EOF
		logger.Error("ParseMultipartForm forloop")
		var part *multipart.Part
		part, err = reader.NextPart()
		if err != nil {
			//fmt.Printf("(ParseMultipartForm) NextPart returned: %s\n", err.Error())
			//logger.Error("NextPart returned " + err.Error())
			if err == io.EOF {
				err = nil
			} else {
				params = nil
				err = fmt.Errorf("(ParseMultipartForm) reader.NextPart returned: %s", err.Error())
			}

			return
		}

		// params don't have a FileName()
		// files must have FormName() of either "upload", "gzip", "bzip2", "attributes", "subset_indices", or an integer
		if part.FileName() == "" {
			if !util.IsValidParamName(part.FormName()) {
				return nil, files, errors.New("invalid param: " + part.FormName())
			}
			buffer := make([]byte, 32*1024)
			var n int
			n, err = part.Read(buffer)
			if err != nil {
				err = nil
				return
			}
			if n == 0 {
				return
			}
			formValue := fmt.Sprintf("%s", buffer[0:n])
			if part.FormName() == "upload_url" {
				tempDir := path.Join(conf.PATH_DATA, "temp")
				var tmpFile *os.File
				tmpFile, err = ioutil.TempFile(tempDir, "")
				if err != nil {
					err = fmt.Errorf("(ParseMultipartForm) ioutil.TempFile returned: %s", err.Error())
					return
				}
				defer tmpFile.Close()
				tmpPath = tmpFile.Name()
				//tmpPath = fmt.Sprintf("%s/temp/%d%d", conf.PATH_DATA, rand.Int(), rand.Int())
				files[part.FormName()] = file.FormFile{Name: "", Path: tmpPath, Checksum: make(map[string]string)}
				// download from url
				//var tmpFile *os.File
				// tmpFile, err = os.Create(tmpPath)
				// if err != nil {
				// 	err = fmt.Errorf("(ParseMultipartForm) os.Create returned: %s", err.Error())
				// 	return
				// }

				var tmpform = files[part.FormName()]
				md5h := md5.New()
				dst := io.MultiWriter(tmpFile, md5h)
				var fileName string
				var body io.ReadCloser
				fileName, body, err = fetchFileStream(formValue)
				if err != nil {
					return nil, files, errors.New("unable to stream url: " + err.Error())
				}
				defer body.Close()
				if _, err = io.Copy(dst, body); err != nil {
					err = fmt.Errorf("(ParseMultipartForm) io.Copy returned: %s", err.Error())
					return nil, files, err
				}
				tmpform.Name = fileName
				tmpform.Checksum["md5"] = fmt.Sprintf("%x", md5h.Sum(nil))
				files[part.FormName()] = tmpform

			} else {
				// regular form field
				params[part.FormName()] = formValue
			}
		} else {
			// determine file type
			isSubsetFile := false
			if part.FormName() == "subset_indices" {
				isSubsetFile = true
			}
			isPartsFile := false
			if _, er := strconv.Atoi(part.FormName()); er == nil {
				isPartsFile = true
			}
			if !isPartsFile && !util.IsValidFileName(part.FormName()) {
				return nil, files, errors.New("invalid file param: " + part.FormName())
			}
			// download it
			tempDir := path.Join(conf.PATH_DATA, "temp")
			var tmpFile *os.File
			tmpFile, err = ioutil.TempFile(tempDir, "")
			if err != nil {
				err = fmt.Errorf("(ParseMultipartForm) ioutil.TempFile returned: %s", err.Error())
				return
			}
			defer tmpFile.Close()

			tmpPath = tmpFile.Name()
			//tmpPath = fmt.Sprintf("%s/temp/%d%d", conf.PATH_DATA, rand.Int(), rand.Int())
			files[part.FormName()] = file.FormFile{Name: part.FileName(), Path: tmpPath, Checksum: make(map[string]string)}
			//var tmpFile *os.File
			// tmpFile, err = os.Create(tmpPath)
			// if err != nil {
			// 	err = fmt.Errorf("(ParseMultipartForm) os.Create returned: %s", err.Error())
			// 	return
			// }

			if util.IsValidUploadFile(part.FormName()) || isPartsFile || isSubsetFile {
				// handle upload or parts files
				var tmpform = files[part.FormName()]
				md5h := md5.New()
				dst := io.MultiWriter(tmpFile, md5h)
				ucReader, ucErr := archive.UncompressReader(part.FormName(), part)
				if ucErr != nil {
					ucErr = fmt.Errorf("(ParseMultipartForm) archive.UncompressReader returned: %s", ucErr.Error())
					return nil, files, ucErr
				}
				if _, err = io.Copy(dst, ucReader); err != nil {
					err = fmt.Errorf("(ParseMultipartForm) io.Copy returned: %s", err.Error())
					return nil, files, err
				}
				if archive.IsValidUncompress(part.FormName()) {
					tmpform.Name = util.StripSuffix(part.FileName())
				}
				tmpform.Checksum["md5"] = fmt.Sprintf("%x", md5h.Sum(nil))
				files[part.FormName()] = tmpform
			} else {
				// handle file where md5 not needed
				if _, err = io.Copy(tmpFile, part); err != nil {
					err = fmt.Errorf("(ParseMultipartForm) io.Copy returned: %s", err.Error())
					return nil, files, err
				}
			}

		}

	}
	return
}

func fetchFileStream(urlStr string) (f string, r io.ReadCloser, err error) {
	u, _ := url.Parse(urlStr)
	if (u.Scheme == "") || (u.Host == "") || (u.Path == "") {
		return "", nil, errors.New("Not a valid url: " + urlStr)
	}
	pathParts := strings.Split(strings.TrimRight(u.Path, "/"), "/")
	fileName := pathParts[len(pathParts)-1]
	cleanPath := strings.Join(pathParts, "/")

	if (u.Scheme == "http") || (u.Scheme == "https") {
		res, err := httpclient.Get(u.String(), httpclient.Header{}, nil, nil)
		if err != nil {
			return "", nil, errors.New("httpclient returned: " + err.Error())
		}
		if res.StatusCode != 200 { //err in fetching data
			resbody, _ := ioutil.ReadAll(res.Body)
			return "", nil, errors.New(fmt.Sprintf("url=%s, res=%s", u.String(), resbody))
		}
		return fileName, res.Body, err
	} else if u.Scheme == "ftp" {
		// set port if missing
		ftpHost := u.Host
		hostParts := strings.Split(u.Host, ":")
		if len(hostParts) == 1 {
			ftpHost = u.Host + ":21"
		}
		c, _, _, err := tinyftp.Dial("tcp", ftpHost)
		if err != nil {
			return "", nil, errors.New("ftpclient returned: " + err.Error())
		}
		defer c.Close()
		if _, _, err = c.Login("", ""); err != nil {
			return "", nil, errors.New("ftpclient returned: " + err.Error())
		}
		dconn, _, _, err := c.RetrieveFrom(cleanPath)
		if err != nil {
			return "", nil, errors.New("ftpclient returned: " + err.Error())
		}
		return fileName, dconn, err
	}
	return "", nil, errors.New("unsupported protocol scheme: " + u.Scheme)
}

func writeChecksum(f func() hash.Hash, c chan checkSumCom) {
	go func() {
		h := f()
		for {
			select {
			case b := <-c:
				if b.n == 0 {
					c <- checkSumCom{checksum: fmt.Sprintf("%x", h.Sum(nil))}
					return
				} else {
					h.Write(b.buf[0:b.n])
				}
			}
		}
	}()
}
<commit_msg>remove debug output<commit_after>package request

import (
	"crypto/md5"
	"errors"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"math/rand"
	"mime/multipart"
	"net"
	"net/http"
	"net/url"
	"os"
	"path"
	"strconv"
	"strings"

	"github.com/MG-RAST/Shock/shock-server/auth"
	"github.com/MG-RAST/Shock/shock-server/conf"
	e "github.com/MG-RAST/Shock/shock-server/errors"
	"github.com/MG-RAST/Shock/shock-server/logger"
	"github.com/MG-RAST/Shock/shock-server/node/archive"
	"github.com/MG-RAST/Shock/shock-server/node/file"
	"github.com/MG-RAST/Shock/shock-server/responder"
	"github.com/MG-RAST/Shock/shock-server/user"
	"github.com/MG-RAST/Shock/shock-server/util"
	"github.com/MG-RAST/golib/httpclient"
	"github.com/MG-RAST/golib/stretchr/goweb/context"
	"github.com/jum/tinyftp"
)

type checkSumCom struct {
	buf      []byte
	n        int
	checksum string
}

func Log(req *http.Request) {
	host, _, _ := net.SplitHostPort(req.RemoteAddr)
	// failed attempt to get the host in ipv4
	//addrs, _ := net.LookupIP(host)
	//fmt.Println(addrs)
	suffix := ""
	if _, auth := req.Header["Authorization"]; auth {
		suffix += " AUTH"
	}

	if l, has := req.Header["Content-Length"]; has {
		suffix += " Content-Length: " + l[0]
	}
	url := ""
	if req.URL.RawQuery != "" {
		url = fmt.Sprintf("%s %s?%s", req.Method, req.URL.Path, req.URL.RawQuery)
	} else {
		url = fmt.Sprintf("%s %s", req.Method, req.URL.Path)
	}
	logger.Infof("%s \"%s%s\"", host, url, suffix)
}

// Authenticate _
func Authenticate(req *http.Request) (u *user.User, err error) {
	if _, ok := req.Header["Authorization"]; !ok {
		err = errors.New(e.NoAuth)
		return
	}
	header := req.Header.Get("Authorization")
	u, err = auth.Authenticate(header)
	return
}

// AuthError _
func AuthError(err error, ctx context.Context) error {

	if conf.DEBUG_AUTH {
		return responder.RespondWithError(ctx, http.StatusBadRequest, err.Error())
	}

	if err.Error() == e.InvalidAuth {
		return responder.RespondWithError(ctx, http.StatusBadRequest, "Invalid authorization header or content")
	}
	err_msg := "Error at Auth: " + err.Error()
	logger.Error(err_msg)
	return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
}

// helper function to create a node from an http data post
func DataUpload(r *http.Request) (params map[string]string, files file.FormFiles, err error) {
	params = make(map[string]string)
	files = make(file.FormFiles)
	tmpPath := fmt.Sprintf("%s/temp/%d%d", conf.PATH_DATA, rand.Int(), rand.Int())

	files["upload"] = file.FormFile{Name: "filename", Path: tmpPath, Checksum: make(map[string]string)}
	if tmpFile, err := os.Create(tmpPath); err == nil {
		defer tmpFile.Close()
		md5h := md5.New()
		dst := io.MultiWriter(tmpFile, md5h)
		if _, err = io.Copy(dst, r.Body); err != nil {
			return nil, nil, err
		}
		files["upload"].Checksum["md5"] = fmt.Sprintf("%x", md5h.Sum(nil))
	} else {
		return nil, nil, err
	}

	return
}

// ParseMultipartForm helper function for create & update
func ParseMultipartForm(r *http.Request) (params map[string]string, files file.FormFiles, err error) {

	params = make(map[string]string)
	files = make(file.FormFiles)
	reader, err := r.MultipartReader()
	if err != nil {
		err = fmt.Errorf("(ParseMultipartForm) MultipartReader returned: %s", err.Error())
		return
	}

	tmpPath := ""
	for { // read until EOF
		//logger.Error("ParseMultipartForm forloop")
		var part *multipart.Part
		part, err = reader.NextPart()
		if err != nil {
			//fmt.Printf("(ParseMultipartForm) NextPart returned: %s\n", err.Error())
			//logger.Error("NextPart returned " + err.Error())
			if err == io.EOF {
				err = nil
			} else {
				params = nil
				err = fmt.Errorf("(ParseMultipartForm) reader.NextPart returned: %s", err.Error())
			}

			return
		}

		// params don't have a FileName()
		// files must have FormName() of either "upload", "gzip", "bzip2", "attributes", "subset_indices", or an integer
		if part.FileName() == "" {
			if !util.IsValidParamName(part.FormName()) {
				return nil, files, errors.New("invalid param: " + part.FormName())
			}
			buffer := make([]byte, 32*1024)
			var n int
			n, err = part.Read(buffer)
			if err != nil {
				err = nil
				return
			}
			if n == 0 {
				return
			}
			formValue := fmt.Sprintf("%s", buffer[0:n])
			if part.FormName() == "upload_url" {
				tempDir := path.Join(conf.PATH_DATA, "temp")
				var tmpFile *os.File
				tmpFile, err = ioutil.TempFile(tempDir, "")
				if err != nil {
					err = fmt.Errorf("(ParseMultipartForm) ioutil.TempFile returned: %s", err.Error())
					return
				}
				defer tmpFile.Close()
				tmpPath = tmpFile.Name()
				//tmpPath = fmt.Sprintf("%s/temp/%d%d", conf.PATH_DATA, rand.Int(), rand.Int())
				files[part.FormName()] = file.FormFile{Name: "", Path: tmpPath, Checksum: make(map[string]string)}
				// download from url
				//var tmpFile *os.File
				// tmpFile, err = os.Create(tmpPath)
				// if err != nil {
				// 	err = fmt.Errorf("(ParseMultipartForm) os.Create returned: %s", err.Error())
				// 	return
				// }

				var tmpform = files[part.FormName()]
				md5h := md5.New()
				dst := io.MultiWriter(tmpFile, md5h)
				var fileName string
				var body io.ReadCloser
				fileName, body, err = fetchFileStream(formValue)
				if err != nil {
					return nil, files, errors.New("unable to stream url: " + err.Error())
				}
				defer body.Close()
				if _, err = io.Copy(dst, body); err != nil {
					err = fmt.Errorf("(ParseMultipartForm) io.Copy returned: %s", err.Error())
					return nil, files, err
				}
				tmpform.Name = fileName
				tmpform.Checksum["md5"] = fmt.Sprintf("%x", md5h.Sum(nil))
				files[part.FormName()] = tmpform

			} else {
				// regular form field
				params[part.FormName()] = formValue
			}
		} else {
			// determine file type
			isSubsetFile := false
			if part.FormName() == "subset_indices" {
				isSubsetFile = true
			}
			isPartsFile := false
			if _, er := strconv.Atoi(part.FormName()); er == nil {
				isPartsFile = true
			}
			if !isPartsFile && !util.IsValidFileName(part.FormName()) {
				return nil, files, errors.New("invalid file param: " + part.FormName())
			}
			// download it
			tempDir := path.Join(conf.PATH_DATA, "temp")
			var tmpFile *os.File
			tmpFile, err = ioutil.TempFile(tempDir, "")
			if err != nil {
				err = fmt.Errorf("(ParseMultipartForm) ioutil.TempFile returned: %s", err.Error())
				return
			}
			defer tmpFile.Close()

			tmpPath = tmpFile.Name()
			//tmpPath = fmt.Sprintf("%s/temp/%d%d", conf.PATH_DATA, rand.Int(), rand.Int())
			files[part.FormName()] = file.FormFile{Name: part.FileName(), Path: tmpPath, Checksum: make(map[string]string)}
			//var tmpFile *os.File
			// tmpFile, err = os.Create(tmpPath)
			// if err != nil {
			// 	err = fmt.Errorf("(ParseMultipartForm) os.Create returned: %s", err.Error())
			// 	return
			// }

			if util.IsValidUploadFile(part.FormName()) || isPartsFile || isSubsetFile {
				// handle upload or parts files
				var tmpform = files[part.FormName()]
				md5h := md5.New()
				dst := io.MultiWriter(tmpFile, md5h)
				ucReader, ucErr := archive.UncompressReader(part.FormName(), part)
				if ucErr != nil {
					ucErr = fmt.Errorf("(ParseMultipartForm) archive.UncompressReader returned: %s", ucErr.Error())
					return nil, files, ucErr
				}
				if _, err = io.Copy(dst, ucReader); err != nil {
					err = fmt.Errorf("(ParseMultipartForm) io.Copy returned: %s", err.Error())
					return nil, files, err
				}
				if archive.IsValidUncompress(part.FormName()) {
					tmpform.Name = util.StripSuffix(part.FileName())
				}
				tmpform.Checksum["md5"] = fmt.Sprintf("%x", md5h.Sum(nil))
				files[part.FormName()] = tmpform
			} else {
				// handle file where md5 not needed
				if _, err = io.Copy(tmpFile, part); err != nil {
					err = fmt.Errorf("(ParseMultipartForm) io.Copy returned: %s", err.Error())
					return nil, files, err
				}
			}

		}

	}
	return
}

func fetchFileStream(urlStr string) (f string, r io.ReadCloser, err error) {
	u, _ := url.Parse(urlStr)
	if (u.Scheme == "") || (u.Host == "") || (u.Path == "") {
		return "", nil, errors.New("Not a valid url: " + urlStr)
	}
	pathParts := strings.Split(strings.TrimRight(u.Path, "/"), "/")
	fileName := pathParts[len(pathParts)-1]
	cleanPath := strings.Join(pathParts, "/")

	if (u.Scheme == "http") || (u.Scheme == "https") {
		res, err := httpclient.Get(u.String(), httpclient.Header{}, nil, nil)
		if err != nil {
			return "", nil, errors.New("httpclient returned: " + err.Error())
		}
		if res.StatusCode != 200 { //err in fetching data
			resbody, _ := ioutil.ReadAll(res.Body)
			return "", nil, errors.New(fmt.Sprintf("url=%s, res=%s", u.String(), resbody))
		}
		return fileName, res.Body, err
	} else if u.Scheme == "ftp" {
		// set port if missing
		ftpHost := u.Host
		hostParts := strings.Split(u.Host, ":")
		if len(hostParts) == 1 {
			ftpHost = u.Host + ":21"
		}
		c, _, _, err := tinyftp.Dial("tcp", ftpHost)
		if err != nil {
			return "", nil, errors.New("ftpclient returned: " + err.Error())
		}
		defer c.Close()
		if _, _, err = c.Login("", ""); err != nil {
			return "", nil, errors.New("ftpclient returned: " + err.Error())
		}
		dconn, _, _, err := c.RetrieveFrom(cleanPath)
		if err != nil {
			return "", nil, errors.New("ftpclient returned: " + err.Error())
		}
		return fileName, dconn, err
	}
	return "", nil, errors.New("unsupported protocol scheme: " + u.Scheme)
}

func writeChecksum(f func() hash.Hash, c chan checkSumCom) {
	go func() {
		h := f()
		for {
			select {
			case b := <-c:
				if b.n == 0 {
					c <- checkSumCom{checksum: fmt.Sprintf("%x", h.Sum(nil))}
					return
				} else {
					h.Write(b.buf[0:b.n])
				}
			}
		}
	}()
}
<|endoftext|>