<commit_before>package telebot

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"strconv"

	"github.com/pkg/errors"
)

func (b *Bot) debug(err error) {
	err = errors.WithStack(err)
	log.Printf("%+v\n", err)
}

func (b *Bot) deferDebug() {
	if r := recover(); r != nil {
		if err, ok := r.(error); ok {
			b.debug(err)
		} else if str, ok := r.(string); ok {
			b.debug(errors.Errorf("%s", str))
		}
	}
}

func (b *Bot) runHandler(h HandlerFunc, c Context) {
	f := func() {
		defer b.deferDebug()
		if err := h(c); err != nil {
			if b.OnError != nil {
				b.OnError(err, c)
			} else {
				log.Println(err)
			}
		}
	}
	if b.synchronous {
		f()
	} else {
		go f()
	}
}

func applyMiddleware(h HandlerFunc, middleware ...MiddlewareFunc) HandlerFunc {
	for i := len(middleware) - 1; i >= 0; i-- {
		h = middleware[i](h)
	}
	return h
}

// wrapError returns new wrapped telebot-related error.
func wrapError(err error) error {
	return errors.Wrap(err, "telebot")
}

// extractOk checks given result for error. If result is ok returns nil.
// In other cases it extracts API error. If error is not presented
// in errors.go, it will be prefixed with `unknown` keyword.
func extractOk(data []byte) error {
	// Parse the error message as JSON
	var tgramApiError struct {
		Ok          bool                   `json:"ok"`
		ErrorCode   int                    `json:"error_code"`
		Description string                 `json:"description"`
		Parameters  map[string]interface{} `json:"parameters"`
	}
	err := json.Unmarshal(data, &tgramApiError)
	if err != nil {
		//return errors.Wrap(err, "can't parse JSON reply, the Telegram server is mibehaving")
		// FIXME / TODO: in this case the error might be at HTTP level, or the content is not JSON (eg. image?)
		return nil
	}

	if tgramApiError.Ok {
		// No error
		return nil
	}

	err = ErrByDescription(tgramApiError.Description)
	if err != nil {
		apierr, _ := err.(*APIError)
		// Formally this is wrong, as the error is not created on the fly
		// However, given the current way of handling errors, this a working
		// workaround which doesn't break the API
		apierr.Parameters = tgramApiError.Parameters
		return apierr
	}

	switch tgramApiError.ErrorCode {
	case http.StatusTooManyRequests:
		retryAfter, ok := tgramApiError.Parameters["retry_after"]
		if !ok {
			return NewAPIError(429, tgramApiError.Description)
		}
		retryAfterInt, _ := strconv.Atoi(fmt.Sprint(retryAfter))

		err = FloodError{
			APIError:   NewAPIError(429, tgramApiError.Description),
			RetryAfter: retryAfterInt,
		}
	default:
		err = fmt.Errorf("telegram unknown: %s (%d)", tgramApiError.Description, tgramApiError.ErrorCode)
	}

	return err
}

// extractMessage extracts common Message result from given data.
// Should be called after extractOk or b.Raw() to handle possible errors.
func extractMessage(data []byte) (*Message, error) {
	var resp struct {
		Result *Message
	}
	if err := json.Unmarshal(data, &resp); err != nil {
		var resp struct {
			Result bool
		}
		if err := json.Unmarshal(data, &resp); err != nil {
			return nil, wrapError(err)
		}
		if resp.Result {
			return nil, ErrTrueResult
		}
		return nil, wrapError(err)
	}
	return resp.Result, nil
}

func extractOptions(how []interface{}) *SendOptions {
	var opts *SendOptions

	for _, prop := range how {
		switch opt := prop.(type) {
		case *SendOptions:
			opts = opt.copy()
		case *ReplyMarkup:
			if opts == nil {
				opts = &SendOptions{}
			}
			if opt != nil {
				opts.ReplyMarkup = opt.copy()
			}
		case Option:
			if opts == nil {
				opts = &SendOptions{}
			}

			switch opt {
			case NoPreview:
				opts.DisableWebPagePreview = true
			case Silent:
				opts.DisableNotification = true
			case ForceReply:
				if opts.ReplyMarkup == nil {
					opts.ReplyMarkup = &ReplyMarkup{}
				}
				opts.ReplyMarkup.ForceReply = true
			case OneTimeKeyboard:
				if opts.ReplyMarkup == nil {
					opts.ReplyMarkup = &ReplyMarkup{}
				}
				opts.ReplyMarkup.OneTimeKeyboard = true
			case RemoveKeyboard:
				if opts.ReplyMarkup == nil {
					opts.ReplyMarkup = &ReplyMarkup{}
				}
				opts.ReplyMarkup.RemoveKeyboard = true
			default:
				panic("telebot: unsupported flag-option")
			}
		case ParseMode:
			if opts == nil {
				opts = &SendOptions{}
			}
			opts.ParseMode = opt
		default:
			panic("telebot: unsupported send-option")
		}
	}

	return opts
}

func (b *Bot) embedSendOptions(params map[string]string, opt *SendOptions) {
	if b.parseMode != ModeDefault {
		params["parse_mode"] = b.parseMode
	}

	if opt == nil {
		return
	}

	if opt.ReplyTo != nil && opt.ReplyTo.ID != 0 {
		params["reply_to_message_id"] = strconv.Itoa(opt.ReplyTo.ID)
	}

	if opt.DisableWebPagePreview {
		params["disable_web_page_preview"] = "true"
	}

	if opt.DisableNotification {
		params["disable_notification"] = "true"
	}

	if opt.ParseMode != ModeDefault {
		params["parse_mode"] = opt.ParseMode
	}

	if opt.ReplyMarkup != nil {
		processButtons(opt.ReplyMarkup.InlineKeyboard)
		replyMarkup, _ := json.Marshal(opt.ReplyMarkup)
		params["reply_markup"] = string(replyMarkup)
	}
}

func processButtons(keys [][]InlineButton) {
	if keys == nil || len(keys) < 1 || len(keys[0]) < 1 {
		return
	}

	for i := range keys {
		for j := range keys[i] {
			key := &keys[i][j]
			if key.Unique != "" {
				// Format: "\f<callback_name>|<data>"
				data := key.Data
				if data == "" {
					key.Data = "\f" + key.Unique
				} else {
					key.Data = "\f" + key.Unique + "|" + data
				}
			}
		}
	}
}

func embedRights(p map[string]interface{}, rights Rights) {
	data, _ := json.Marshal(rights)
	_ = json.Unmarshal(data, &p)
}

func thumbnailToFilemap(thumb *Photo) map[string]File {
	if thumb != nil {
		return map[string]File{"thumb": thumb.File}
	}
	return nil
}

func isUserInList(user *User, list []User) bool {
	for _, user2 := range list {
		if user.ID == user2.ID {
			return true
		}
	}
	return false
}
<commit_msg>Decode big integers using Number and not float64 in error messages<commit_after>package telebot

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"strconv"
	"bytes"
	"github.com/pkg/errors"
)

func (b *Bot) debug(err error) {
	err = errors.WithStack(err)
	log.Printf("%+v\n", err)
}

func (b *Bot) deferDebug() {
	if r := recover(); r != nil {
		if err, ok := r.(error); ok {
			b.debug(err)
		} else if str, ok := r.(string); ok {
			b.debug(errors.Errorf("%s", str))
		}
	}
}

func (b *Bot) runHandler(h HandlerFunc, c Context) {
	f := func() {
		defer b.deferDebug()
		if err := h(c); err != nil {
			if b.OnError != nil {
				b.OnError(err, c)
			} else {
				log.Println(err)
			}
		}
	}
	if b.synchronous {
		f()
	} else {
		go f()
	}
}

func applyMiddleware(h HandlerFunc, middleware ...MiddlewareFunc) HandlerFunc {
	for i := len(middleware) - 1; i >= 0; i-- {
		h = middleware[i](h)
	}
	return h
}

// wrapError returns new wrapped telebot-related error.
func wrapError(err error) error {
	return errors.Wrap(err, "telebot")
}

// extractOk checks given result for error. If result is ok returns nil.
// In other cases it extracts API error. If error is not presented
// in errors.go, it will be prefixed with `unknown` keyword.
func extractOk(data []byte) error {
	// Parse the error message as JSON
	var tgramApiError struct {
		Ok          bool                   `json:"ok"`
		ErrorCode   int                    `json:"error_code"`
		Description string                 `json:"description"`
		Parameters  map[string]interface{} `json:"parameters"`
	}
	jdecoder := json.NewDecoder(bytes.NewReader(data))
	jdecoder.UseNumber()

	err := jdecoder.Decode(&tgramApiError)
	if err != nil {
		//return errors.Wrap(err, "can't parse JSON reply, the Telegram server is mibehaving")
		// FIXME / TODO: in this case the error might be at HTTP level, or the content is not JSON (eg. image?)
		return nil
	}

	if tgramApiError.Ok {
		// No error
		return nil
	}

	err = ErrByDescription(tgramApiError.Description)
	if err != nil {
		apierr, _ := err.(*APIError)
		// Formally this is wrong, as the error is not created on the fly
		// However, given the current way of handling errors, this a working
		// workaround which doesn't break the API
		apierr.Parameters = tgramApiError.Parameters
		return apierr
	}

	switch tgramApiError.ErrorCode {
	case http.StatusTooManyRequests:
		retryAfter, ok := tgramApiError.Parameters["retry_after"]
		if !ok {
			return NewAPIError(429, tgramApiError.Description)
		}
		retryAfterInt, _ := strconv.Atoi(fmt.Sprint(retryAfter))

		err = FloodError{
			APIError:   NewAPIError(429, tgramApiError.Description),
			RetryAfter: retryAfterInt,
		}
	default:
		err = fmt.Errorf("telegram unknown: %s (%d)", tgramApiError.Description, tgramApiError.ErrorCode)
	}

	return err
}

// extractMessage extracts common Message result from given data.
// Should be called after extractOk or b.Raw() to handle possible errors.
func extractMessage(data []byte) (*Message, error) {
	var resp struct {
		Result *Message
	}
	if err := json.Unmarshal(data, &resp); err != nil {
		var resp struct {
			Result bool
		}
		if err := json.Unmarshal(data, &resp); err != nil {
			return nil, wrapError(err)
		}
		if resp.Result {
			return nil, ErrTrueResult
		}
		return nil, wrapError(err)
	}
	return resp.Result, nil
}

func extractOptions(how []interface{}) *SendOptions {
	var opts *SendOptions

	for _, prop := range how {
		switch opt := prop.(type) {
		case *SendOptions:
			opts = opt.copy()
		case *ReplyMarkup:
			if opts == nil {
				opts = &SendOptions{}
			}
			if opt != nil {
				opts.ReplyMarkup = opt.copy()
			}
		case Option:
			if opts == nil {
				opts = &SendOptions{}
			}

			switch opt {
			case NoPreview:
				opts.DisableWebPagePreview = true
			case Silent:
				opts.DisableNotification = true
			case ForceReply:
				if opts.ReplyMarkup == nil {
					opts.ReplyMarkup = &ReplyMarkup{}
				}
				opts.ReplyMarkup.ForceReply = true
			case OneTimeKeyboard:
				if opts.ReplyMarkup == nil {
					opts.ReplyMarkup = &ReplyMarkup{}
				}
				opts.ReplyMarkup.OneTimeKeyboard = true
			case RemoveKeyboard:
				if opts.ReplyMarkup == nil {
					opts.ReplyMarkup = &ReplyMarkup{}
				}
				opts.ReplyMarkup.RemoveKeyboard = true
			default:
				panic("telebot: unsupported flag-option")
			}
		case ParseMode:
			if opts == nil {
				opts = &SendOptions{}
			}
			opts.ParseMode = opt
		default:
			panic("telebot: unsupported send-option")
		}
	}

	return opts
}

func (b *Bot) embedSendOptions(params map[string]string, opt *SendOptions) {
	if b.parseMode != ModeDefault {
		params["parse_mode"] = b.parseMode
	}

	if opt == nil {
		return
	}

	if opt.ReplyTo != nil && opt.ReplyTo.ID != 0 {
		params["reply_to_message_id"] = strconv.Itoa(opt.ReplyTo.ID)
	}

	if opt.DisableWebPagePreview {
		params["disable_web_page_preview"] = "true"
	}

	if opt.DisableNotification {
		params["disable_notification"] = "true"
	}

	if opt.ParseMode != ModeDefault {
		params["parse_mode"] = opt.ParseMode
	}

	if opt.ReplyMarkup != nil {
		processButtons(opt.ReplyMarkup.InlineKeyboard)
		replyMarkup, _ := json.Marshal(opt.ReplyMarkup)
		params["reply_markup"] = string(replyMarkup)
	}
}

func processButtons(keys [][]InlineButton) {
	if keys == nil || len(keys) < 1 || len(keys[0]) < 1 {
		return
	}

	for i := range keys {
		for j := range keys[i] {
			key := &keys[i][j]
			if key.Unique != "" {
				// Format: "\f<callback_name>|<data>"
				data := key.Data
				if data == "" {
					key.Data = "\f" + key.Unique
				} else {
					key.Data = "\f" + key.Unique + "|" + data
				}
			}
		}
	}
}

func embedRights(p map[string]interface{}, rights Rights) {
	data, _ := json.Marshal(rights)
	_ = json.Unmarshal(data, &p)
}

func thumbnailToFilemap(thumb *Photo) map[string]File {
	if thumb != nil {
		return map[string]File{"thumb": thumb.File}
	}
	return nil
}

func isUserInList(user *User, list []User) bool {
	for _, user2 := range list {
		if user.ID == user2.ID {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package memberlist

import (
	"bytes"
	"compress/lzw"
	"encoding/binary"
	"fmt"
	"io"
	"math"
	"math/rand"
	"strings"
	"time"

	"github.com/hashicorp/go-msgpack/codec"
)

// pushPullScale is the minimum number of nodes
// before we start scaling the push/pull timing. The scale
// effect is the log2(Nodes) - log2(pushPullScale). This means
// that the 33rd node will cause us to double the interval,
// while the 65th will triple it.
const pushPullScaleThreshold = 32

const (
	// Constant litWidth 2-8
	lzwLitWidth = 8
)

func init() {
	// Seed the random number generator
	rand.Seed(time.Now().UnixNano())
}

// Decode reverses the encode operation on a byte slice input
func decode(buf []byte, out interface{}) error {
	r := bytes.NewReader(buf)
	hd := codec.MsgpackHandle{}
	dec := codec.NewDecoder(r, &hd)
	return dec.Decode(out)
}

// Encode writes an encoded object to a new bytes buffer
func encode(msgType messageType, in interface{}) (*bytes.Buffer, error) {
	buf := bytes.NewBuffer(nil)
	buf.WriteByte(uint8(msgType))
	hd := codec.MsgpackHandle{}
	enc := codec.NewEncoder(buf, &hd)
	err := enc.Encode(in)
	return buf, err
}

// Returns a random offset between 0 and n
func randomOffset(n int) int {
	if n == 0 {
		return 0
	}
	return int(rand.Uint32() % uint32(n))
}

// suspicionTimeout computes the timeout that should be used when
// a node is suspected
func suspicionTimeout(suspicionMult, n int, interval time.Duration) time.Duration {
	nodeScale := math.Max(1.0, math.Log10(math.Max(1.0, float64(n))))
	// multiply by 1000 to keep some precision because time.Duration is an int64 type
	timeout := time.Duration(suspicionMult) * time.Duration(nodeScale*1000) * interval / 1000
	return timeout
}

// retransmitLimit computes the limit of retransmissions
func retransmitLimit(retransmitMult, n int) int {
	nodeScale := math.Ceil(math.Log10(float64(n + 1)))
	limit := retransmitMult * int(nodeScale)
	return limit
}

// shuffleNodes randomly shuffles the input nodes using the Fisher-Yates shuffle
func shuffleNodes(nodes []*nodeState) {
	n := len(nodes)
	for i := n - 1; i > 0; i-- {
		j := rand.Intn(i + 1)
		nodes[i], nodes[j] = nodes[j], nodes[i]
	}
}

// pushPushScale is used to scale the time interval at which push/pull
// syncs take place. It is used to prevent network saturation as the
// cluster size grows
func pushPullScale(interval time.Duration, n int) time.Duration {
	// Don't scale until we cross the threshold
	if n <= pushPullScaleThreshold {
		return interval
	}

	multiplier := math.Ceil(math.Log2(float64(n))-math.Log2(pushPullScaleThreshold)) + 1.0
	return time.Duration(multiplier) * interval
}

// moveDeadNodes moves all the nodes in the dead state
// to the end of the slice and returns the index of the first dead node.
func moveDeadNodes(nodes []*nodeState) int {
	numDead := 0
	n := len(nodes)
	for i := 0; i < n-numDead; i++ {
		if nodes[i].State != stateDead {
			continue
		}

		// Move this node to the end
		nodes[i], nodes[n-numDead-1] = nodes[n-numDead-1], nodes[i]
		numDead++
		i--
	}
	return n - numDead
}

// kRandomNodes is used to select up to k random nodes, excluding any nodes where
// the filter function returns true. It is possible that less than k nodes are
// returned.
func kRandomNodes(k int, nodes []*nodeState, filterFn func(*nodeState) bool) []*nodeState {
	n := len(nodes)
	kNodes := make([]*nodeState, 0, k)
OUTER:
	// Probe up to 3*n times, with large n this is not necessary
	// since k << n, but with small n we want search to be
	// exhaustive
	for i := 0; i < 3*n && len(kNodes) < k; i++ {
		// Get random node
		idx := randomOffset(n)
		node := nodes[idx]

		// Give the filter a shot at it.
		if filterFn != nil && filterFn(node) {
			continue OUTER
		}

		// Check if we have this node already
		for j := 0; j < len(kNodes); j++ {
			if node == kNodes[j] {
				continue OUTER
			}
		}

		// Append the node
		kNodes = append(kNodes, node)
	}
	return kNodes
}

// makeCompoundMessage takes a list of messages and generates
// a single compound message containing all of them
func makeCompoundMessage(msgs [][]byte) *bytes.Buffer {
	// Create a local buffer
	buf := bytes.NewBuffer(nil)

	// Write out the type
	buf.WriteByte(uint8(compoundMsg))

	// Write out the number of message
	buf.WriteByte(uint8(len(msgs)))

	// Add the message lengths
	for _, m := range msgs {
		binary.Write(buf, binary.BigEndian, uint16(len(m)))
	}

	// Append the messages
	for _, m := range msgs {
		buf.Write(m)
	}

	return buf
}

// decodeCompoundMessage splits a compound message and returns
// the slices of individual messages. Also returns the number
// of truncated messages and any potential error
func decodeCompoundMessage(buf []byte) (trunc int, parts [][]byte, err error) {
	if len(buf) < 1 {
		err = fmt.Errorf("missing compound length byte")
		return
	}
	numParts := uint8(buf[0])
	buf = buf[1:]

	// Check we have enough bytes
	if len(buf) < int(numParts*2) {
		err = fmt.Errorf("truncated len slice")
		return
	}

	// Decode the lengths
	lengths := make([]uint16, numParts)
	for i := 0; i < int(numParts); i++ {
		lengths[i] = binary.BigEndian.Uint16(buf[i*2 : i*2+2])
	}
	buf = buf[numParts*2:]

	// Split each message
	for idx, msgLen := range lengths {
		if len(buf) < int(msgLen) {
			trunc = int(numParts) - idx
			return
		}

		// Extract the slice, seek past on the buffer
		slice := buf[:msgLen]
		buf = buf[msgLen:]
		parts = append(parts, slice)
	}
	return
}

// Given a string of the form "host", "host:port",
// "ipv6::addr" or "[ipv6::address]:port",
// return true if the string includes a port.
func hasPort(s string) bool {
	last := strings.LastIndex(s, ":")
	if last == -1 {
		return false
	}
	if s[0] == '[' {
		return s[last-1] == ']'
	}
	return strings.Index(s, ":") == last
}

// compressPayload takes an opaque input buffer, compresses it
// and wraps it in a compress{} message that is encoded.
func compressPayload(inp []byte) (*bytes.Buffer, error) {
	var buf bytes.Buffer
	compressor := lzw.NewWriter(&buf, lzw.LSB, lzwLitWidth)

	_, err := compressor.Write(inp)
	if err != nil {
		return nil, err
	}

	// Ensure we flush everything out
	if err := compressor.Close(); err != nil {
		return nil, err
	}

	// Create a compressed message
	c := compress{
		Algo: lzwAlgo,
		Buf:  buf.Bytes(),
	}
	return encode(compressMsg, &c)
}

// decompressPayload is used to unpack an encoded compress{}
// message and return its payload uncompressed
func decompressPayload(msg []byte) ([]byte, error) {
	// Decode the message
	var c compress
	if err := decode(msg, &c); err != nil {
		return nil, err
	}
	return decompressBuffer(&c)
}

// decompressBuffer is used to decompress the buffer of
// a single compress message, handling multiple algorithms
func decompressBuffer(c *compress) ([]byte, error) {
	// Verify the algorithm
	if c.Algo != lzwAlgo {
		return nil, fmt.Errorf("Cannot decompress unknown algorithm %d", c.Algo)
	}

	// Create a uncompressor
	uncomp := lzw.NewReader(bytes.NewReader(c.Buf), lzw.LSB, lzwLitWidth)
	defer uncomp.Close()

	// Read all the data
	var b bytes.Buffer
	_, err := io.Copy(&b, uncomp)
	if err != nil {
		return nil, err
	}

	// Return the uncompressed bytes
	return b.Bytes(), nil
}
<commit_msg>Use `seed` to consistently seed.<commit_after>package memberlist

import (
	"bytes"
	"compress/lzw"
	"encoding/binary"
	"fmt"
	"io"
	"math"
	"math/rand"
	"strings"
	"time"

	"github.com/hashicorp/go-msgpack/codec"
	"github.com/sean-/seed"
)

// pushPullScale is the minimum number of nodes
// before we start scaling the push/pull timing. The scale
// effect is the log2(Nodes) - log2(pushPullScale). This means
// that the 33rd node will cause us to double the interval,
// while the 65th will triple it.
const pushPullScaleThreshold = 32

const (
	// Constant litWidth 2-8
	lzwLitWidth = 8
)

func init() {
	seed.Init()
}

// Decode reverses the encode operation on a byte slice input
func decode(buf []byte, out interface{}) error {
	r := bytes.NewReader(buf)
	hd := codec.MsgpackHandle{}
	dec := codec.NewDecoder(r, &hd)
	return dec.Decode(out)
}

// Encode writes an encoded object to a new bytes buffer
func encode(msgType messageType, in interface{}) (*bytes.Buffer, error) {
	buf := bytes.NewBuffer(nil)
	buf.WriteByte(uint8(msgType))
	hd := codec.MsgpackHandle{}
	enc := codec.NewEncoder(buf, &hd)
	err := enc.Encode(in)
	return buf, err
}

// Returns a random offset between 0 and n
func randomOffset(n int) int {
	if n == 0 {
		return 0
	}
	return int(rand.Uint32() % uint32(n))
}

// suspicionTimeout computes the timeout that should be used when
// a node is suspected
func suspicionTimeout(suspicionMult, n int, interval time.Duration) time.Duration {
	nodeScale := math.Max(1.0, math.Log10(math.Max(1.0, float64(n))))
	// multiply by 1000 to keep some precision because time.Duration is an int64 type
	timeout := time.Duration(suspicionMult) * time.Duration(nodeScale*1000) * interval / 1000
	return timeout
}

// retransmitLimit computes the limit of retransmissions
func retransmitLimit(retransmitMult, n int) int {
	nodeScale := math.Ceil(math.Log10(float64(n + 1)))
	limit := retransmitMult * int(nodeScale)
	return limit
}

// shuffleNodes randomly shuffles the input nodes using the Fisher-Yates shuffle
func shuffleNodes(nodes []*nodeState) {
	n := len(nodes)
	for i := n - 1; i > 0; i-- {
		j := rand.Intn(i + 1)
		nodes[i], nodes[j] = nodes[j], nodes[i]
	}
}

// pushPushScale is used to scale the time interval at which push/pull
// syncs take place. It is used to prevent network saturation as the
// cluster size grows
func pushPullScale(interval time.Duration, n int) time.Duration {
	// Don't scale until we cross the threshold
	if n <= pushPullScaleThreshold {
		return interval
	}

	multiplier := math.Ceil(math.Log2(float64(n))-math.Log2(pushPullScaleThreshold)) + 1.0
	return time.Duration(multiplier) * interval
}

// moveDeadNodes moves all the nodes in the dead state
// to the end of the slice and returns the index of the first dead node.
func moveDeadNodes(nodes []*nodeState) int {
	numDead := 0
	n := len(nodes)
	for i := 0; i < n-numDead; i++ {
		if nodes[i].State != stateDead {
			continue
		}

		// Move this node to the end
		nodes[i], nodes[n-numDead-1] = nodes[n-numDead-1], nodes[i]
		numDead++
		i--
	}
	return n - numDead
}

// kRandomNodes is used to select up to k random nodes, excluding any nodes where
// the filter function returns true. It is possible that less than k nodes are
// returned.
func kRandomNodes(k int, nodes []*nodeState, filterFn func(*nodeState) bool) []*nodeState {
	n := len(nodes)
	kNodes := make([]*nodeState, 0, k)
OUTER:
	// Probe up to 3*n times, with large n this is not necessary
	// since k << n, but with small n we want search to be
	// exhaustive
	for i := 0; i < 3*n && len(kNodes) < k; i++ {
		// Get random node
		idx := randomOffset(n)
		node := nodes[idx]

		// Give the filter a shot at it.
		if filterFn != nil && filterFn(node) {
			continue OUTER
		}

		// Check if we have this node already
		for j := 0; j < len(kNodes); j++ {
			if node == kNodes[j] {
				continue OUTER
			}
		}

		// Append the node
		kNodes = append(kNodes, node)
	}
	return kNodes
}

// makeCompoundMessage takes a list of messages and generates
// a single compound message containing all of them
func makeCompoundMessage(msgs [][]byte) *bytes.Buffer {
	// Create a local buffer
	buf := bytes.NewBuffer(nil)

	// Write out the type
	buf.WriteByte(uint8(compoundMsg))

	// Write out the number of message
	buf.WriteByte(uint8(len(msgs)))

	// Add the message lengths
	for _, m := range msgs {
		binary.Write(buf, binary.BigEndian, uint16(len(m)))
	}

	// Append the messages
	for _, m := range msgs {
		buf.Write(m)
	}

	return buf
}

// decodeCompoundMessage splits a compound message and returns
// the slices of individual messages. Also returns the number
// of truncated messages and any potential error
func decodeCompoundMessage(buf []byte) (trunc int, parts [][]byte, err error) {
	if len(buf) < 1 {
		err = fmt.Errorf("missing compound length byte")
		return
	}
	numParts := uint8(buf[0])
	buf = buf[1:]

	// Check we have enough bytes
	if len(buf) < int(numParts*2) {
		err = fmt.Errorf("truncated len slice")
		return
	}

	// Decode the lengths
	lengths := make([]uint16, numParts)
	for i := 0; i < int(numParts); i++ {
		lengths[i] = binary.BigEndian.Uint16(buf[i*2 : i*2+2])
	}
	buf = buf[numParts*2:]

	// Split each message
	for idx, msgLen := range lengths {
		if len(buf) < int(msgLen) {
			trunc = int(numParts) - idx
			return
		}

		// Extract the slice, seek past on the buffer
		slice := buf[:msgLen]
		buf = buf[msgLen:]
		parts = append(parts, slice)
	}
	return
}

// Given a string of the form "host", "host:port",
// "ipv6::addr" or "[ipv6::address]:port",
// return true if the string includes a port.
func hasPort(s string) bool {
	last := strings.LastIndex(s, ":")
	if last == -1 {
		return false
	}
	if s[0] == '[' {
		return s[last-1] == ']'
	}
	return strings.Index(s, ":") == last
}

// compressPayload takes an opaque input buffer, compresses it
// and wraps it in a compress{} message that is encoded.
func compressPayload(inp []byte) (*bytes.Buffer, error) {
	var buf bytes.Buffer
	compressor := lzw.NewWriter(&buf, lzw.LSB, lzwLitWidth)

	_, err := compressor.Write(inp)
	if err != nil {
		return nil, err
	}

	// Ensure we flush everything out
	if err := compressor.Close(); err != nil {
		return nil, err
	}

	// Create a compressed message
	c := compress{
		Algo: lzwAlgo,
		Buf:  buf.Bytes(),
	}
	return encode(compressMsg, &c)
}

// decompressPayload is used to unpack an encoded compress{}
// message and return its payload uncompressed
func decompressPayload(msg []byte) ([]byte, error) {
	// Decode the message
	var c compress
	if err := decode(msg, &c); err != nil {
		return nil, err
	}
	return decompressBuffer(&c)
}

// decompressBuffer is used to decompress the buffer of
// a single compress message, handling multiple algorithms
func decompressBuffer(c *compress) ([]byte, error) {
	// Verify the algorithm
	if c.Algo != lzwAlgo {
		return nil, fmt.Errorf("Cannot decompress unknown algorithm %d", c.Algo)
	}

	// Create a uncompressor
	uncomp := lzw.NewReader(bytes.NewReader(c.Buf), lzw.LSB, lzwLitWidth)
	defer uncomp.Close()

	// Read all the data
	var b bytes.Buffer
	_, err := io.Copy(&b, uncomp)
	if err != nil {
		return nil, err
	}

	// Return the uncompressed bytes
	return b.Bytes(), nil
}
<|endoftext|>


<commit_before>package snmpgo

import (
	"bytes"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"math"
	"math/rand"
	"net"
	"regexp"
	"strings"
	"sync"
	"time"
)

var random *rand.Rand
var randOnce sync.Once

func initRandom() {
	random = rand.New(rand.NewSource(time.Now().UnixNano()))
}

func genRequestId() int {
	randOnce.Do(initRandom)
	return int(random.Int31())
}

func genSalt32() int32 {
	randOnce.Do(initRandom)
	return random.Int31()
}

func genSalt64() int64 {
	randOnce.Do(initRandom)
	return random.Int63()
}

var mesId int = math.MaxInt32 - 1
var mesMutex sync.Mutex

func genMessageId() (id int) {
	randOnce.Do(initRandom)
	mesMutex.Lock()
	mesId++
	if mesId == math.MaxInt32 {
		mesId = int(random.Int31())
	}
	id = mesId
	mesMutex.Unlock()
	return
}

func retry(retries int, f func() error) (err error) {
	for i := 0; i <= retries; i++ {
		err = f()
		switch e := err.(type) {
		case net.Error:
			if e.Timeout() {
				continue
			}
		case *notInTimeWindowError:
			err = e.error
			continue
		}
		return
	}
	return
}

func confirmedType(t PduType) bool {
	if t == GetRequest || t == GetNextRequest || t == SetRequest ||
		t == GetBulkRequest || t == InformRequest {
		return true
	}
	return false
}

func engineIdToBytes(engineId string) ([]byte, error) {
	b, err := hex.DecodeString(engineId)
	if l := len(b); err != nil || (l < 5 || l > 32) {
		return nil, &ArgumentError{
			Value:   engineId,
			Message: "EngineId must be a hexadecimal string and length is range 5..32",
		}
	}
	return b, nil
}

var hexPrefix *regexp.Regexp = regexp.MustCompile(`^0[xX]`)

func stripHexPrefix(s string) string {
	return hexPrefix.ReplaceAllString(s, "")
}

func toHexStr(a []byte, sep string) string {
	s := make([]string, len(a))
	for i, b := range a {
		s[i] = fmt.Sprintf("%02x", b)
	}
	return strings.Join(s, sep)
}

func escape(s interface{}) string {
	r, _ := json.Marshal(s)
	return string(r)
}

func xor(a, b []byte) []byte {
	c := make([]byte, len(a))
	for i := 0; i < len(a); i++ {
		c[i] = a[i] ^ b[i]
	}
	return c
}

func padding(b []byte, size int) []byte {
	pad := size - (len(b) % size)
	if pad > 0 {
		b = append(b, bytes.Repeat([]byte{0x00}, pad)...)
	}
	return b
}
<commit_msg>Fix for data race in generating request ids<commit_after>package snmpgo

import (
	"bytes"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"math"
	"math/rand"
	"net"
	"regexp"
	"strings"
	"sync"
	"time"
)

var random *rand.Rand
var randOnce sync.Once

func initRandom() {
	random = rand.New(rand.NewSource(time.Now().UnixNano()))
}

var reqMutex sync.Mutex
func genRequestId() int {
	randOnce.Do(initRandom)
	reqMutex.Lock()
	val := int(random.Int31())
	reqMutex.Unlock()
	return val
}

func genSalt32() int32 {
	randOnce.Do(initRandom)
	return random.Int31()
}

func genSalt64() int64 {
	randOnce.Do(initRandom)
	return random.Int63()
}

var mesId int = math.MaxInt32 - 1
var mesMutex sync.Mutex

func genMessageId() (id int) {
	randOnce.Do(initRandom)
	mesMutex.Lock()
	mesId++
	if mesId == math.MaxInt32 {
		mesId = int(random.Int31())
	}
	id = mesId
	mesMutex.Unlock()
	return
}

func retry(retries int, f func() error) (err error) {
	for i := 0; i <= retries; i++ {
		err = f()
		switch e := err.(type) {
		case net.Error:
			if e.Timeout() {
				continue
			}
		case *notInTimeWindowError:
			err = e.error
			continue
		}
		return
	}
	return
}

func confirmedType(t PduType) bool {
	if t == GetRequest || t == GetNextRequest || t == SetRequest ||
		t == GetBulkRequest || t == InformRequest {
		return true
	}
	return false
}

func engineIdToBytes(engineId string) ([]byte, error) {
	b, err := hex.DecodeString(engineId)
	if l := len(b); err != nil || (l < 5 || l > 32) {
		return nil, &ArgumentError{
			Value:   engineId,
			Message: "EngineId must be a hexadecimal string and length is range 5..32",
		}
	}
	return b, nil
}

var hexPrefix *regexp.Regexp = regexp.MustCompile(`^0[xX]`)

func stripHexPrefix(s string) string {
	return hexPrefix.ReplaceAllString(s, "")
}

func toHexStr(a []byte, sep string) string {
	s := make([]string, len(a))
	for i, b := range a {
		s[i] = fmt.Sprintf("%02x", b)
	}
	return strings.Join(s, sep)
}

func escape(s interface{}) string {
	r, _ := json.Marshal(s)
	return string(r)
}

func xor(a, b []byte) []byte {
	c := make([]byte, len(a))
	for i := 0; i < len(a); i++ {
		c[i] = a[i] ^ b[i]
	}
	return c
}

func padding(b []byte, size int) []byte {
	pad := size - (len(b) % size)
	if pad > 0 {
		b = append(b, bytes.Repeat([]byte{0x00}, pad)...)
	}
	return b
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"regexp"
	"strings"
)

// A map for Puppet Versions JSON
type PuppetVersions interface{}

// A map for Production Versions JSON
type QaVersions interface{}

// The final map to be passed to template
type Compared map[string]map[string]map[string]string

// Our bare page
type Page struct {
	Title string
	Body  []byte
	Pv    PuppetVersions
	//	Rv    RunningServices
}

// What do *you* think this does?
func colorize(versions []string) (color string, err error) {
	if len(versions) > 0 {
		for i, version := range versions {
			if version[i] == version[i+1] {
				color = "green"
				return color, nil
			} else {
				color = "red"
				return color, nil
			}
		}
	} else {
		return "green", nil
	}
	return "versions not an array?", err
}

func compare(puppet_v map[string]interface{}, qa_v map[string]map[string]string) (Compared, error) {
	c := make(map[string]map[string]map[string]string)

	// Setup regex for QA match
	match_qa, err := regexp.Compile(`_qa`)
	if err != nil {
		log.Println("Couldn't compile regex")
		return nil, err
	}
	match_prod, err := regexp.Compile(`_production`)
	if err != nil {
		log.Println("Couldn't compile regex")
		return nil, err
	}
	log.Println("COMPARE PV: ", puppet_v)
	log.Println("COMPARE QA: ", qa_v)

	// Get environments from PuppetVersions, populate top level map
	c["qa"] = make(map[string]map[string]string)
	c["production"] = make(map[string]map[string]string)

	for p_name, pv := range puppet_v {
		pv_string := pv.(string)
		log.Println("NAME: ", p_name, "version ", pv)
		if match_qa.MatchString(p_name) {
			log.Println("QA MATCH: ", p_name, " ", pv)
			// Add the name and puppet version to QA map
			c["qa"][p_name] = make(map[string]string)
			c["qa"][p_name]["pv"] = pv_string

			// Init new array, add versions for this service
			colorize_arry := []string{}
			colorize_arry = append(colorize_arry, pv_string)

			for _, endpoints := range qa_v {
				for ep, version := range endpoints {
					c["qa"][p_name][ep] = version
					colorize_arry = append(colorize_arry, version)
					color, _ := colorize(colorize_arry)
					c["qa"][p_name]["color"] = color
				}
			}
		}
		if match_prod.MatchString(p_name) {
			log.Println("Production MATCH: ", p_name, " ", pv)
			c["production"][p_name] = make(map[string]string)
		}
	}

	return c, nil
}

func puppetversions(url string) (PuppetVersions, error) {
	// Get response and handle any errors on return
	resp, err := http.Get(url)
	defer resp.Body.Close()
	if err != nil {
		log.Println(err)
		return nil, err
	}

	// Read JSON from request
	jsonDataFromHttp, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	// Unmarshel the JSON to our struct
	var v PuppetVersions
	err = json.Unmarshal(jsonDataFromHttp, &v)
	if err != nil {
		return nil, err
	}

	return v, nil
}

func getServices(url string) (interface{}, error) {
	resp, err := http.Get(url)
	defer resp.Body.Close()
	if err != nil {
		log.Println(err)
		return nil, err
	}

	jsonDataFromHttp, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var available_services interface{}

	err = json.Unmarshal(jsonDataFromHttp, &available_services)
	if err != nil {
		return nil, err
	}

	return available_services, nil
}

func queryServiceVersion(endpoint string) (version string, err error) {
	log.Println("Querying SERVICE address: ", endpoint)
	query_arry := []string{"http://", endpoint, "/info"}
	query := strings.Join(query_arry, "")
	log.Println("Query string: ", query)
	// Query the URI
	resp, err := http.Get(query)
	defer resp.Body.Close()
	if err != nil {
		log.Println("ERROR querying ", query, " ", err)
		return "Failed to get server response for endpoint", err
	}
	// Get data and unmarshel the JSON to our map
	jsonDataFromHttp, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Println("ERROR unmarsheling data for ", jsonDataFromHttp)
		return "Failed to read JSON", err
	}
	var info_response interface{}
	err = json.Unmarshal(jsonDataFromHttp, &info_response)
	if err != nil {
		return "Failed to get info response ", err
	}
	// Parse out the version from the response
	log.Println("INFO for ", endpoint, ":\n", info_response)

	info_map := info_response.(map[string]interface{})
	for _, values := range info_map {
		log.Println("String: ", values)
		sub_info_map := values.(map[string]interface{})
		for key, info := range sub_info_map {
			string_info := info.(string)
			log.Println("Sub info: ", string_info)
			if key == "version" {
				log.Println("Version: ", string_info)
				return string_info, nil
			}
		}
	}
	return version, nil
}

func getVersions(services interface{}) (runningversions map[string]map[string]string, err error) {

	rv := make(map[string]map[string]string)

	s := services.(map[string]interface{})
	for k, v := range s {
		log.Println("Ranging over ", k)
		switch values := v.(type) {
		case map[string]interface{}:
			for name, endpoints := range values {
				log.Println("Found service: ", name)
				rv[name] = make(map[string]string)
				switch eps := endpoints.(type) {
				case []interface{}:
					for _, ep := range eps {
						log.Println("Endpoint: ", ep)
						switch ep_string := ep.(type) {
						case string:
							query_arry := strings.Fields(ep_string)
							if len(query_arry) == 2 {
								log.Println("IP 1: ", query_arry[0])
								log.Println("IP 2: ", query_arry[1])
								info_ep := query_arry[1]
								version, _ := queryServiceVersion(info_ep)
								rv[name][info_ep] = version
							} else {
								log.Println("IP 1: ", query_arry[0])
								info_ep := query_arry[0]
								rv[name][info_ep] = "blah"
							}
						}
					}
				}
			}
		}
	}
	runningversions = rv
	return runningversions, nil
}

func loadPage(title string) (*Page, error) {
	// Get the versions from the puppet master
	log.Println("Getting Puppet Versions - Make sure VPN is on!")
	pv, err := puppetversions("http://puppet.ec2.srcclr.com:1015/versions")
	if err != nil {
		log.Println("Failed to get Puppet Versions from http://puppet.ec2.srcclr.com:1015/versions\n")
		log.Println(err)
	}
	log.Println("Puppet Versions: ", pv)

	// Get running services, prs
	log.Println("Getting available services...")
	qa_rs, err := getServices("http://is.qa.ec2.srcclr.com:3000/services")
	if err != nil {
		log.Println("Failed getting production versions")
	}

	log.Println("RUNNING SERVICES QA: ", qa_rs)

	qa_v, err := getVersions(qa_rs)
	if err != nil {
		log.Println("Failed getting versions for ", qa_rs)
	}

	log.Println("Running Versions: ", qa_v)

	pv_map := pv.(map[string]interface{})
	compared, _ := compare(pv_map, qa_v)

	for k, v := range compared {
		log.Println(k, " ", v, "\n")
	}

	filename := title + ".html"
	body, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}

	return &Page{
			Title: title,
			Body:  body,
			Pv:    pv,
			//	Rv:    rv,
		},
		nil
}

func viewHandler(w http.ResponseWriter, r *http.Request) {
	log.Println("Loading page view...")
	title := "versionctl"
	p, err := loadPage(title)
	if err != nil {
		p = &Page{Title: title}
	}

	// Parse the template, execute and write it to stdout for good measure
	log.Println("Parsing go template...")
	t, _ := template.ParseFiles("versionctl.html")
	t.Execute(w, p)
	//log.Println("Serving:\n", string(p.Title), string(p.Body))
}

func main() {
	log.Println("Starting vctl...")
	http.HandleFunc("/", viewHandler)
	http.ListenAndServe(":9000", nil)
}
<commit_msg>still have too many eps<commit_after>package main

import (
	"encoding/json"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"regexp"
	"strings"
)

// A map for Puppet Versions JSON
type PuppetVersions interface{}

// A map for Production Versions JSON
type QaVersions interface{}

// The final map to be passed to template
type Compared map[string]map[string]map[string]string

// Our bare page
type Page struct {
	Title string
	Body  []byte
	Pv    PuppetVersions
	//	Rv    RunningServices
}

// What do *you* think this does?
func colorize(versions []string) (color string, err error) {
	if len(versions) > 1 {
		for i := 0; i < len(versions); i++ {
			if versions[i] == "Failed" || versions[i+1] == "Failed" {
				log.Println("Refusing to compare failed info repsonse: ", versions[i], " and ", versions[i+1])
				return "green", nil
			}
			log.Println("Comparing: ", versions[i], " ", versions[i+1])
			if versions[i] == versions[i+1] {
				log.Println("COLORS MATCH!")
				color = "green"
				return color, nil
			} else {
				log.Println("NO COLOR MATCH")
				color = "red"
				return color, nil
			}
		}
	} else {
		return "green", nil
	}
	return "versions not an array?", err
}

func compare(puppet_v map[string]interface{}, qa_v map[string]map[string]string) (Compared, error) {
	c := make(map[string]map[string]map[string]string)
	// Setup regex for QA match
	match_qa, err := regexp.Compile(`_qa`)
	if err != nil {
		log.Println("Couldn't compile regex")
		return nil, err
	}
	match_prod, err := regexp.Compile(`_production`)
	if err != nil {
		log.Println("Couldn't compile regex")
		return nil, err
	}
	// Get environments from PuppetVersions, populate top level map
	c["qa"] = make(map[string]map[string]string)
	c["production"] = make(map[string]map[string]string)

	for p_name, pv := range puppet_v {
		pv_string := pv.(string)
		if match_qa.MatchString(p_name) {
			log.Println("Qa MATCH: ", p_name, " ", pv)
			// Add the name and puppet version to QA map
			c["qa"][p_name] = make(map[string]string)
			c["qa"][p_name]["pv"] = pv_string

			// Init new array, add versions for this service
			colorize_arry := []string{}

			colorize_arry = append(colorize_arry, pv_string)

			for _, endpoints := range qa_v {
				for ep, version := range endpoints {
					c["qa"][p_name][ep] = version
					colorize_arry = append(colorize_arry, version)
					color, _ := colorize(colorize_arry)
					c["qa"][p_name]["color"] = color
				}
			}
		}
		if match_prod.MatchString(p_name) {
			log.Println("Production MATCH: ", p_name, " ", pv)
			c["production"][p_name] = make(map[string]string)
		}
	}

	return c, nil
}

func puppetversions(url string) (PuppetVersions, error) {
	// Get response and handle any errors on return
	resp, err := http.Get(url)
	defer resp.Body.Close()
	if err != nil {
		log.Println(err)
		return nil, err
	}

	// Read JSON from request
	jsonDataFromHttp, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	// Unmarshel the JSON to our struct
	var v PuppetVersions
	err = json.Unmarshal(jsonDataFromHttp, &v)
	if err != nil {
		return nil, err
	}

	return v, nil
}

func getServices(url string) (interface{}, error) {
	resp, err := http.Get(url)
	defer resp.Body.Close()
	if err != nil {
		log.Println(err)
		return nil, err
	}

	jsonDataFromHttp, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var available_services interface{}

	err = json.Unmarshal(jsonDataFromHttp, &available_services)
	if err != nil {
		return nil, err
	}

	return available_services, nil
}

func queryServiceVersion(endpoint string) (version string, err error) {
	log.Println("Querying SERVICE address: ", endpoint)
	query_arry := []string{"http://", endpoint, "/info"}
	query := strings.Join(query_arry, "")
	// Query the URI
	resp, err := http.Get(query)
	defer resp.Body.Close()
	if err != nil {
		log.Println("ERROR querying ", query, " ", err)
		return "Failed to get server response for endpoint", err
	}
	// Get data and unmarshel the JSON to our map
	jsonDataFromHttp, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Println("ERROR unmarsheling data for ", jsonDataFromHttp)
		return "Failed to read JSON", err
	}
	var info_response interface{}
	err = json.Unmarshal(jsonDataFromHttp, &info_response)
	if err != nil {
		return "Failed", err
	}
	// Parse out the version from the response
	info_map := info_response.(map[string]interface{})
	log.Println("Response: ", info_response)
	for _, values := range info_map {
		sub_info_map := values.(map[string]interface{})
		for key, info := range sub_info_map {
			string_info := info.(string)
			if key == "version" {
				log.Println("Version: ", string_info)
				return string_info, nil
			}
		}
	}
	return version, nil
}

func getVersions(services interface{}) (runningversions map[string]map[string]string, err error) {

	rv := make(map[string]map[string]string)

	s := services.(map[string]interface{})
	for _, v := range s {
		switch values := v.(type) {
		case map[string]interface{}:
			for name, endpoints := range values {
				log.Println("Found service: ", name)
				rv[name] = make(map[string]string)
				switch eps := endpoints.(type) {
				case []interface{}:
					for _, ep := range eps {
						switch ep_string := ep.(type) {
						case string:
							query_arry := strings.Fields(ep_string)
							if len(query_arry) == 2 {
								info_ep := query_arry[1]
								version, _ := queryServiceVersion(info_ep)
								rv[name][info_ep] = version
							} else {
								info_ep := query_arry[0]
								rv[name][info_ep] = "blah"
							}
						}
					}
				}
			}
		}
	}
	runningversions = rv
	return runningversions, nil
}

func loadPage(title string) (*Page, error) {
	// Get the versions from the puppet master
	log.Println("Getting Puppet Versions - Make sure VPN is on!")
	pv, err := puppetversions("http://puppet.ec2.srcclr.com:1015/versions")
	if err != nil {
		log.Println("Failed to get Puppet Versions from http://puppet.ec2.srcclr.com:1015/versions\n")
		log.Println(err)
	}
	log.Println("Puppet Versions: ", pv)

	// Get running services, prs
	log.Println("Getting available services...")
	qa_rs, err := getServices("http://is.qa.ec2.srcclr.com:3000/services")
	if err != nil {
		log.Println("Failed getting production versions")
	}

	log.Println("RUNNING SERVICES QA: ", qa_rs)

	qa_v, err := getVersions(qa_rs)
	if err != nil {
		log.Println("Failed getting versions for ", qa_rs)
	}

	log.Println("Running Versions: ", qa_v)

	pv_map := pv.(map[string]interface{})
	compared, _ := compare(pv_map, qa_v)

	for k, v := range compared {
		log.Println(k, " ", v, "\n")
	}

	filename := title + ".html"
	body, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}

	return &Page{
			Title: title,
			Body:  body,
			Pv:    pv,
			//	Rv:    rv,
		},
		nil
}

func viewHandler(w http.ResponseWriter, r *http.Request) {
	log.Println("Loading page view...")
	title := "versionctl"
	p, err := loadPage(title)
	if err != nil {
		p = &Page{Title: title}
	}

	// Parse the template, execute and write it to stdout for good measure
	log.Println("Parsing go template...")
	t, _ := template.ParseFiles("versionctl.html")
	t.Execute(w, p)
	//log.Println("Serving:\n", string(p.Title), string(p.Body))
}

func main() {
	log.Println("Starting vctl...")
	http.HandleFunc("/", viewHandler)
	http.ListenAndServe(":9000", nil)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"fmt"
	"os/exec"
	"regexp"
	"strconv"
	"strings"

	"gopkg.in/telegram-bot-api.v4"
)

// RegisterVLAN adds "vlan", "in", "out" commands to bot
func RegisterVLAN(bot Bot, ifaces *Interfaces) {
	ifaces.Update()
	v := &vlan{Interfaces: ifaces}
	bot.Add("vlan", func(bot Bot, msg *tgbotapi.Message, tokens *Tokens) string {
		return v.replyToVLAN(bot, msg, tokens)
	})
	bot.Add("up", func(bot Bot, msg *tgbotapi.Message, tokens *Tokens) string {
		return v.replyToIn(bot, msg, tokens)
	})
	bot.Add("in", func(bot Bot, msg *tgbotapi.Message, tokens *Tokens) string {
		return v.replyToIn(bot, msg, tokens)
	})
	bot.Add("down", func(bot Bot, msg *tgbotapi.Message, tokens *Tokens) string {
		return v.replyToOut(bot, msg, tokens)
	})
	bot.Add("out", func(bot Bot, msg *tgbotapi.Message, tokens *Tokens) string {
		return v.replyToOut(bot, msg, tokens)
	})
}

// VLAN data
type vlan struct {
	Selected   int         // Currently selected vlan
	Interfaces *Interfaces // Enumeration of interfaces
	Device     string      // Device name for selected VLAN
	IFB        string      // IFB device name for selected vlan
}

// Impairment parameters
type params struct {
	delay, jitter     int
	loss, correlation float64
}

// ReplyToVLAN selects a particular VLAN
func (v *vlan) replyToVLAN(bot Bot, msg *tgbotapi.Message, tokens *Tokens) string {
	if tokens.Remaining() < 1 {
		return "Error: must provide the VLAN number (vlan <vlan_number>)"
	}
	vlan, err := strconv.Atoi(tokens.Next())
	if err != nil {
		return err.Error()
	}
	if vlan < 1 || vlan > 4094 {
		return "Error: VLAN number must be between 1 and 4094"
	}
	suffix := fmt.Sprintf(".%d", vlan)
	found := ""
	for name := range v.Interfaces.Current {
		if strings.HasSuffix(name, suffix) {
			found = name
			break
		}
	}
	if found == "" {
		return fmt.Sprintf("Error: VLAN %d is not found. Run \"ip\" for more info", vlan)
	}
	v.Selected = vlan
	v.Device = found
	ifb, err := v.getIFB()
	if err != nil {
		return fmt.Sprintf("Could not get IFB: %s", err.Error())
	}
	v.IFB = ifb
	return fmt.Sprintf("VLAN %d selected", vlan)
}

// ReplyToIn adds delay in the upstream direction
func (v *vlan) replyToIn(bot Bot, msg *tgbotapi.Message, tokens *Tokens) string {
	if v.IFB == "" {
		return "Current VLAN does not have IFB device assigned"
	}
	params, err := v.getParams(msg, tokens)
	if err != nil {
		return err.Error()
	}
	return v.impair(v.IFB, params)
}

// ReplyToOut adds delay in the downstream direction
func (v *vlan) replyToOut(bot Bot, msg *tgbotapi.Message, tokens *Tokens) string {
	params, err := v.getParams(msg, tokens)
	if err != nil {
		return err.Error()
	}
	return v.impair(v.Device, params)
}

// Get Delay, Jitter, PL and PL correlation from command
func (v *vlan) getParams(msg *tgbotapi.Message, tokens *Tokens) (params, error) {
	result := params{}
	if v.Selected == 0 {
		return result, errors.New("No VLAN selected. Run \"vlan\" for more info")
	}
	if tokens.Remaining() <= 0 {
		return result, errors.New("Error: must at least provide delay (ms). Format: [in|out] <delay_ms> <jitter_ms> <PL %> <correlation %>")
	}
	msDelay, err := strconv.Atoi(tokens.Next())
	if err != nil {
		return result, fmt.Errorf("delay is not an int: %s", err.Error())
	}
	if msDelay < 0 || msDelay > 4094 {
		return result, errors.New("Error: Delay must be between 0 and 4094 milliseconds")
	}
	result.delay = msDelay
	if tokens.Remaining() > 0 {
		msJitter, err := strconv.Atoi(tokens.Next())
		if err != nil {
			tokens.Back()
			return result, nil
		}
		if msJitter < 0 || msJitter > 4094 {
			return result, errors.New("Error: Jitter must be between 0 and 4094 milliseconds")
		}
		result.jitter = msJitter
	}
	if tokens.Remaining() > 0 {
		pl, err := strconv.ParseFloat(tokens.Next(), 32)
		if err != nil {
			tokens.Back()
			return result, nil
		}
		if pl < 0 || pl > 100 {
			return result, errors.New("Error: Packet loss must be between 0.0 and 100.0 percent")
		}
		result.loss = pl
	}
	if tokens.Remaining() > 0 {
		corr, err := strconv.ParseFloat(tokens.Next(), 32)
		if err != nil {
			tokens.Back()
			return result, nil
		}
		if corr < 0 || corr > 100 {
			return result, errors.New("Error: Correlation must be between 0.0 and 100.0 percent")
		}
		result.correlation = corr
	}
	return result, nil
}

// Add impairments (delay, jitter, loss...) to an interface
func (v *vlan) impair(iface string, p params) string {
	messages := make([]string, 0, 10)
	// Remove any qdisc
	cmd := exec.Command("tc", "qdisc", "del", "dev", iface, "root")
	var outDel bytes.Buffer
	cmd.Stdout = &outDel
	if err := cmd.Run(); err != nil {
		messages = append(messages, fmt.Sprintf("Warn: nothing to clear in interface %s. Proceeding (%s)", iface, err.Error()))
	} else {
		messages = append(messages, fmt.Sprintf("Cleared interface %s", iface))
	}
	messages = append(messages, outDel.String())
	// Prepare for adding jitter and packet loss
	cmdLine := fmt.Sprintf("tc qdisc add dev %s root netem", iface)
	doApply := false
	if p.delay != 0 {
		doApply = true
		cmdLine = fmt.Sprintf("%s delay %dms", cmdLine, p.delay)
		if p.jitter != 0 {
			cmdLine = fmt.Sprintf("%s %dms distribution normal", cmdLine, p.jitter)
		}
	}
	if p.loss != 0 {
		doApply = true
		cmdLine = fmt.Sprintf("%s loss %f%%", cmdLine, p.loss)
		if p.correlation != 0 {
			cmdLine = fmt.Sprintf("%s %f%%", cmdLine, p.correlation)
		}
	}
	// If delay != 0, add it
	var outAdd bytes.Buffer
	if doApply {
		messages = append(messages, fmt.Sprintf("Policy for interface %s: %dms delay (%dms jitter), %f%% PL (%f%% correlation)", iface, p.delay, p.jitter, p.loss, p.correlation))
		fields := strings.Fields(cmdLine)
		cmd = exec.Command(fields[0], fields[1:]...)
		cmd.Stdout = &outAdd
		if err := cmd.Run(); err != nil {
			messages = append(messages, fmt.Sprintf("Error at qdisc add: %s", err.Error()))
		}
		messages = append(messages, outAdd.String())

	}
	// Return the output of the qdisc commands
	return strings.Join(messages, "\n")
}

// Gets the IFB interface associated to the selected VLAN
func (v *vlan) getIFB() (string, error) {
	cmd := exec.Command("tc", "filter", "show", "dev", v.Device, "root")
	var outShow bytes.Buffer
	cmd.Stdout = &outShow
	if err := cmd.Run(); err != nil {
		return "", fmt.Errorf("Error at filter show: %s", err.Error())
	}
	data := outShow.String()
	re := regexp.MustCompile("Egress Redirect to device ifb[0-9]")
	match := re.FindString(data)
	if match == "" {
		return "", fmt.Errorf("Missing IFB device for %s in %s", v.Device, data)
	}
	ifbFields := strings.Fields(match)
	return ifbFields[len(ifbFields)-1], nil
}
<commit_msg>Added support for generic interfaces<commit_after>package main

import (
	"bytes"
	"errors"
	"fmt"
	"os/exec"
	"regexp"
	"strconv"
	"strings"

	"gopkg.in/telegram-bot-api.v4"
)

// RegisterVLAN adds "vlan", "in", "out" commands to bot
func RegisterVLAN(bot Bot, ifaces *Interfaces) {
	ifaces.Update()
	v := &vlan{Interfaces: ifaces}
	bot.Add("current", func(bot Bot, msg *tgbotapi.Message, tokens *Tokens) string {
		return v.replyToCurrent(bot, msg, tokens)
	})
	bot.Add("iface", func(bot Bot, msg *tgbotapi.Message, tokens *Tokens) string {
		return v.replyToIface(bot, msg, tokens)
	})
	bot.Add("interface", func(bot Bot, msg *tgbotapi.Message, tokens *Tokens) string {
		return v.replyToIface(bot, msg, tokens)
	})
	bot.Add("vlan", func(bot Bot, msg *tgbotapi.Message, tokens *Tokens) string {
		return v.replyToVLAN(bot, msg, tokens)
	})
	bot.Add("up", func(bot Bot, msg *tgbotapi.Message, tokens *Tokens) string {
		return v.replyToIn(bot, msg, tokens)
	})
	bot.Add("in", func(bot Bot, msg *tgbotapi.Message, tokens *Tokens) string {
		return v.replyToIn(bot, msg, tokens)
	})
	bot.Add("down", func(bot Bot, msg *tgbotapi.Message, tokens *Tokens) string {
		return v.replyToOut(bot, msg, tokens)
	})
	bot.Add("out", func(bot Bot, msg *tgbotapi.Message, tokens *Tokens) string {
		return v.replyToOut(bot, msg, tokens)
	})
}

// VLAN data
type vlan struct {
	Interfaces *Interfaces // Enumeration of interfaces
	Device     string      // Device name for selected VLAN
	IFB        string      // IFB device name for selected vlan
}

// Impairment parameters
type params struct {
	delay, jitter     int
	loss, correlation float64
}

// ReplyToIface selects a particular interface
func (v *vlan) replyToIface(bot Bot, msg *tgbotapi.Message, tokens *Tokens) string {
	if tokens.Remaining() < 1 {
		return "Error: must provide the device name (interface <name>)"
	}
	prefix := tokens.Next()
	if prefix == "" {
		return "Error: Must provide an interface name"
	}
	matches := make([]string, 0, 10)
	for name := range v.Interfaces.Current {
		if strings.HasPrefix(name, prefix) {
			// VLAN interfaces are excluded, use the "VLAN" command
			if !strings.Contains(name, ".") {
				matches = append(matches, name)
			}
		}
	}
	if len(matches) <= 0 {
		return fmt.Sprintf("Error: Interface %s is not found. Run \"ip\" for more info", prefix)
	}
	if len(matches) > 1 {
		return fmt.Sprintf("Error: Interface %s is ambiguous, matches: %s", strings.Join(matches, ", "))
	}
	return v.setDevice(matches[0])
}

// ReplyToVLAN selects a particular VLAN
func (v *vlan) replyToVLAN(bot Bot, msg *tgbotapi.Message, tokens *Tokens) string {
	if tokens.Remaining() < 1 {
		return "Error: must provide the VLAN number (vlan <vlan_number>)"
	}
	vlan, err := strconv.Atoi(tokens.Next())
	if err != nil {
		return err.Error()
	}
	if vlan < 1 || vlan > 4094 {
		return "Error: VLAN number must be between 1 and 4094"
	}
	suffix := fmt.Sprintf(".%d", vlan)
	found := ""
	for name := range v.Interfaces.Current {
		if strings.HasSuffix(name, suffix) {
			found = name
			break
		}
	}
	if found == "" {
		return fmt.Sprintf("Error: VLAN %d is not found. Run \"ip\" for more info", vlan)
	}
	return v.setDevice(found)
}

func (v *vlan) setDevice(device string) string {
	v.Device = device
	v.IFB = ""
	ifb, err := v.getIFB()
	if err != nil {
		return fmt.Sprintf("Could not get IFB: %s.\n Interface %s will only accept out or down commands.", err.Error(), device)
	}
	v.IFB = ifb
	return fmt.Sprintf("Device %s selected", device)
}

// ReplyToCurrent dumps the current interface
func (v *vlan) replyToCurrent(bot Bot, msg *tgbotapi.Message, tokens *Tokens) string {
	return fmt.Sprintf("Selected device: [%s]. Matching IFB: [%s]", v.Device, v.IFB)
}

// ReplyToIn adds delay in the upstream direction
func (v *vlan) replyToIn(bot Bot, msg *tgbotapi.Message, tokens *Tokens) string {
	if v.IFB == "" {
		return "Current VLAN does not have IFB device assigned"
	}
	params, err := v.getParams(msg, tokens)
	if err != nil {
		return err.Error()
	}
	return v.impair(v.IFB, params)
}

// ReplyToOut adds delay in the downstream direction
func (v *vlan) replyToOut(bot Bot, msg *tgbotapi.Message, tokens *Tokens) string {
	params, err := v.getParams(msg, tokens)
	if err != nil {
		return err.Error()
	}
	return v.impair(v.Device, params)
}

// Get Delay, Jitter, PL and PL correlation from command
func (v *vlan) getParams(msg *tgbotapi.Message, tokens *Tokens) (params, error) {
	result := params{}
	if v.Device == "" {
		return result, errors.New("No device selected. Run \"ip\" for more info")
	}
	if tokens.Remaining() <= 0 {
		return result, errors.New("Error: must at least provide delay (ms). Format: [in|out] <delay_ms> <jitter_ms> <PL %> <correlation %>")
	}
	msDelay, err := strconv.Atoi(tokens.Next())
	if err != nil {
		return result, fmt.Errorf("delay is not an int: %s", err.Error())
	}
	if msDelay < 0 || msDelay > 4094 {
		return result, errors.New("Error: Delay must be between 0 and 4094 milliseconds")
	}
	result.delay = msDelay
	if tokens.Remaining() > 0 {
		msJitter, err := strconv.Atoi(tokens.Next())
		if err != nil {
			tokens.Back()
			return result, nil
		}
		if msJitter < 0 || msJitter > 4094 {
			return result, errors.New("Error: Jitter must be between 0 and 4094 milliseconds")
		}
		result.jitter = msJitter
	}
	if tokens.Remaining() > 0 {
		pl, err := strconv.ParseFloat(tokens.Next(), 32)
		if err != nil {
			tokens.Back()
			return result, nil
		}
		if pl < 0 || pl > 100 {
			return result, errors.New("Error: Packet loss must be between 0.0 and 100.0 percent")
		}
		result.loss = pl
	}
	if tokens.Remaining() > 0 {
		corr, err := strconv.ParseFloat(tokens.Next(), 32)
		if err != nil {
			tokens.Back()
			return result, nil
		}
		if corr < 0 || corr > 100 {
			return result, errors.New("Error: Correlation must be between 0.0 and 100.0 percent")
		}
		result.correlation = corr
	}
	return result, nil
}

// Add impairments (delay, jitter, loss...) to an interface
func (v *vlan) impair(iface string, p params) string {
	messages := make([]string, 0, 10)
	// Remove any qdisc
	cmd := exec.Command("tc", "qdisc", "del", "dev", iface, "root")
	var outDel bytes.Buffer
	cmd.Stdout = &outDel
	if err := cmd.Run(); err != nil {
		messages = append(messages, fmt.Sprintf("Warn: nothing to clear in interface %s. Proceeding (%s)", iface, err.Error()))
	} else {
		messages = append(messages, fmt.Sprintf("Cleared interface %s", iface))
	}
	messages = append(messages, outDel.String())
	// Prepare for adding jitter and packet loss
	cmdLine := fmt.Sprintf("tc qdisc add dev %s root netem", iface)
	doApply := false
	if p.delay != 0 {
		doApply = true
		cmdLine = fmt.Sprintf("%s delay %dms", cmdLine, p.delay)
		if p.jitter != 0 {
			cmdLine = fmt.Sprintf("%s %dms distribution normal", cmdLine, p.jitter)
		}
	}
	if p.loss != 0 {
		doApply = true
		cmdLine = fmt.Sprintf("%s loss %f%%", cmdLine, p.loss)
		if p.correlation != 0 {
			cmdLine = fmt.Sprintf("%s %f%%", cmdLine, p.correlation)
		}
	}
	// If delay != 0, add it
	var outAdd bytes.Buffer
	if doApply {
		messages = append(messages, fmt.Sprintf("Policy for interface %s: %dms delay (%dms jitter), %.2f%% PL (%.2f%% correlation)", iface, p.delay, p.jitter, p.loss, p.correlation))
		fields := strings.Fields(cmdLine)
		cmd = exec.Command(fields[0], fields[1:]...)
		cmd.Stdout = &outAdd
		if err := cmd.Run(); err != nil {
			messages = append(messages, fmt.Sprintf("Error at qdisc add: %s", err.Error()))
		}
		messages = append(messages, outAdd.String())

	}
	// Return the output of the qdisc commands
	return strings.Join(messages, "\n")
}

// Gets the IFB interface associated to the selected VLAN
func (v *vlan) getIFB() (string, error) {
	cmd := exec.Command("tc", "filter", "show", "dev", v.Device, "root")
	var outShow bytes.Buffer
	cmd.Stdout = &outShow
	if err := cmd.Run(); err != nil {
		return "", fmt.Errorf("Error at filter show: %s", err.Error())
	}
	data := outShow.String()
	re := regexp.MustCompile("Egress Redirect to device ifb[0-9]")
	match := re.FindString(data)
	if match == "" {
		return "", fmt.Errorf("Missing IFB device for %s in [%s]", v.Device, data)
	}
	ifbFields := strings.Fields(match)
	return ifbFields[len(ifbFields)-1], nil
}
<|endoftext|>


<commit_before>package someutils

import (
	"errors"
	"flag"
	"fmt"
	"io"
	"math"
	"mime"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

func init() {
	Register(Util{
		"wget",
		Wget})
}

type WgetOptions struct {
}

func Wget(call []string) error {

	//options := WgetOptions{}
	flagSet := flag.NewFlagSet("wget", flag.ContinueOnError)
	helpFlag := flagSet.Bool("help", false, "Show this help")

	err := flagSet.Parse(splitSingleHyphenOpts(call[1:]))
	if err != nil {
		return err
	}
	if *helpFlag {
		println("`grep` [options] PATTERN [files...]")
		flagSet.PrintDefaults()
		return nil
	}
	args := flagSet.Args()
	if len(args) < 1 {
		flagSet.PrintDefaults()
		return errors.New("Not enough args")
	}
	if len(args) > 0 {
		links := args
		return wget(links)
	} else {
		if IsPipingStdin() {
			//check STDIN
			return wget([]string{})
		} else {
			//NOT piping.
			return errors.New("Not enough args")
		}
	}
}

func wget(links []string) error {
	for _, link := range links {
		err := wgetOne(link)
		if err != nil {
			return err
		}
	}
	return nil
}

func wgetOne(link string) error {
	if !strings.Contains(link, ":") {
		link = "http://" + link
	}
	startTime := time.Now()
	resp, err := http.Get(link)
	if err != nil {
		return err
	}
	fmt.Printf("Http response: %s\n", resp.Status)

	lenS := resp.Header.Get("Content-Length")
	len := int64(-1)
	if lenS != "" {
		len, err = strconv.ParseInt(lenS, 10, 32)
		if err != nil {
			return err
		}
	}
	typ := resp.Header.Get("Content-Type")
	fmt.Printf("Length: %v [%s]\n", len, typ)

	defer resp.Body.Close()

	filename, err := getFilename(resp)
	if err != nil {
		return err
	}
	fmt.Printf("Saving to: '%v'\n\n", filename)
	out, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer out.Close()

	buf := make([]byte, 4068)
	tot := int64(0)
	i := 0

	for {
        // read a chunk
        n, err := resp.Body.Read(buf)
        if err != nil && err != io.EOF {
			return err
		}
        if n == 0 { break }
		tot += int64(n)

        // write a chunk
        if _, err := out.Write(buf[:n]); err != nil {
            return err
        }
		i+=1
		if len > -1 {
			if len < 1 {
				fmt.Printf("\r     [ <=>                                  ] %d\t-.--KB/s eta ?s             ", tot)
			} else {
				//show percentage
				perc := (100 * tot) / len
				prog := progress(perc)
				nowTime := time.Now()
				totTime := nowTime.Sub(startTime)
				spd := float64(tot / 1000) / totTime.Seconds()
				remKb := float64(len - tot) / float64(1000)
				eta :=  remKb / spd
				fmt.Printf("\r%3d%% [%s] %d\t%0.2fKB/s eta %0.1fs             ", perc, prog, tot, spd, eta)
			}
		} else {
			//show dots
			if math.Mod(float64(i), 20) == 0 {
				fmt.Print(".")
			}
		}
    }
	perc := (100 * tot) / len
	prog := progress(perc)
	nowTime := time.Now()
	totTime := nowTime.Sub(startTime)
	spd := float64(tot / 1000) / totTime.Seconds()
	if len < 1 {
		fmt.Printf("\r     [ <=>                                  ] %d\t-.--KB/s in %0.1fs             ", tot, totTime.Seconds())
		fmt.Printf("\n (%0.2fKB/s) - '%v' saved [%v]\n", spd, filename, tot)
	} else {
		fmt.Printf("\r%3d%% [%s] %d\t%0.2fKB/s in %0.1fs             ", perc, prog, tot, spd, totTime.Seconds())
		fmt.Printf("\n '%v' saved [%v/%v]\n", filename, tot, len)
	}
	if err != nil {
		return err
	}
	err = out.Close()
	return err
}

func progress(perc int64) string {
	equalses := perc * 38 / 100
	if equalses < 0 {
		equalses = 0
	}
	spaces := 38 - equalses
	if spaces < 0 {
		spaces = 0
	}
	prog := strings.Repeat("=", int(equalses)) + ">" + strings.Repeat(" ", int(spaces))
	return prog
}

func getFilename(resp *http.Response) (string, error) {
	filename := filepath.Base(resp.Request.URL.Path)
	//invalid filenames ...
	if filename == "" || filename == "/" ||filename == "\\" || filename == "." {
		filename = "index"
	}
	if !strings.Contains(filename, ".") {
		ct := resp.Header.Get("Content-Type")
		//println(ct)
		ext := "htm"
		mediatype, _, err := mime.ParseMediaType(ct)
		if err != nil {
			fmt.Printf("mime error: %v\n", err)
		} else {
			fmt.Printf("mime type: %v (from Content-Type %v)\n", mediatype, ct)
			slash := strings.Index(mediatype, "/")
			if slash != -1 {
				_, sub := mediatype[:slash], mediatype[slash+1:]
				if sub != "" {
					ext = sub
				}
			}
		}
		filename = filename + "." + ext
	}
	_, err := os.Stat(filename)
	if err != nil {
		if os.IsNotExist(err) {
			return filename, nil
		} else {
			return "", err
		}
	} else {
		num := 1
		//just stop after 100
		for num < 100 {
			filenameNew := filename + "." + strconv.Itoa(num)
			_, err := os.Stat(filenameNew)
			if err != nil {
				if os.IsNotExist(err) {
					return filenameNew, nil
				} else {
					return "", err
				}
			}
			num += 1
		}
	}
	return filename, errors.New("Stopping after trying 100 filenames")
}<commit_msg>beginning wget 'continue' support<commit_after>package someutils

import (
	"errors"
	"flag"
	"fmt"
	"io"
	"math"
	"mime"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

func init() {
	Register(Util{
		"wget",
		Wget})
}

type WgetOptions struct {
	IsContinue *bool
	Filename *string
}

const (
	FILEMODE os.FileMode = 0660
)

func Wget(call []string) error {

	options := WgetOptions{}
	flagSet := flag.NewFlagSet("wget", flag.ContinueOnError)
	options.IsContinue = flagSet.Bool("c", false, "continue")
	options.Filename = flagSet.String("o", "", "output filename")
	helpFlag := flagSet.Bool("help", false, "Show this help")

	err := flagSet.Parse(splitSingleHyphenOpts(call[1:]))
	if err != nil {
		return err
	}
	if *helpFlag {
		println("wget [options] URL")
		flagSet.PrintDefaults()
		return nil
	}
	args := flagSet.Args()
	if len(args) < 1 {
		flagSet.PrintDefaults()
		return errors.New("Not enough args")
	}
	if len(args) > 0 {
		links := args
		return wget(links, options)
	} else {
		if IsPipingStdin() {
			//check STDIN
			return wget([]string{}, options)
		} else {
			//NOT piping.
			return errors.New("Not enough args")
		}
	}
}

func wget(links []string, options WgetOptions) error {
	for _, link := range links {
		err := wgetOne(link, options)
		if err != nil {
			return err
		}
	}
	return nil
}

func tidyFilename(filename string) string {
	//invalid filenames ...
	if filename == "" || filename == "/" ||filename == "\\" || filename == "." {
		filename = "index"
	}
	return filename
}

func wgetOne(link string, options WgetOptions) error {
	if !strings.Contains(link, ":") {
		link = "http://" + link
	}
	startTime := time.Now()
	request, err := http.NewRequest("GET", link, nil)
	//resp, err := http.Get(link)
	if err != nil {
		return err
	}

	filename := ""
	if *options.Filename != "" {
		filename = *options.Filename
	}
	client := &http.Client{}
	//continue from where we left off ...
	if *options.IsContinue {
		if filename == "" {
			filename = filepath.Base(request.URL.Path)
			filename = tidyFilename(filename)
			if !strings.Contains(filename, ".") {
				filename = filename + ".html"
			}
		}
		fi, err := os.Stat(filename)
		if err != nil {
			return err
		}
		from := fi.Size()
		headRequest, err := http.NewRequest("HEAD", link, nil)
		if err != nil {
			return err
		}
		headResp, err := client.Do(headRequest)
		if err != nil {
			return err
		}
		cl := headResp.Header.Get("Content-Length")
		if cl != "" {
		rangeHeader := fmt.Sprintf("bytes %d-%s", from, cl)
		fmt.Printf("Adding range header: %s\n", rangeHeader)
		request.Header.Add("Range", rangeHeader)
		} else {
			fmt.Println("Could not find file length using HEAD request")
		}
	}
	resp, err := client.Do(request)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	fmt.Printf("Http response: %s\n", resp.Status)

	lenS := resp.Header.Get("Content-Length")
	len := int64(-1)
	if lenS != "" {
		len, err = strconv.ParseInt(lenS, 10, 32)
		if err != nil {
			return err
		}
	}
	typ := resp.Header.Get("Content-Type")
	fmt.Printf("Length: %v [%s]\n", len, typ)

	defer resp.Body.Close()
	if filename == "" {
		filename, err = getFilename(resp)
		if err != nil {
			return err
		}
	}
	fmt.Printf("Saving to: '%v'\n\n", filename)
	out, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer out.Close()

	buf := make([]byte, 4068)
	tot := int64(0)
	i := 0

	for {
        // read a chunk
        n, err := resp.Body.Read(buf)
        if err != nil && err != io.EOF {
			return err
		}
        if n == 0 { break }
		tot += int64(n)

        // write a chunk
        if _, err := out.Write(buf[:n]); err != nil {
            return err
        }
		i+=1
		if len > -1 {
			if len < 1 {
				fmt.Printf("\r     [ <=>                                  ] %d\t-.--KB/s eta ?s             ", tot)
			} else {
				//show percentage
				perc := (100 * tot) / len
				prog := progress(perc)
				nowTime := time.Now()
				totTime := nowTime.Sub(startTime)
				spd := float64(tot / 1000) / totTime.Seconds()
				remKb := float64(len - tot) / float64(1000)
				eta :=  remKb / spd
				fmt.Printf("\r%3d%% [%s] %d\t%0.2fKB/s eta %0.1fs             ", perc, prog, tot, spd, eta)
			}
		} else {
			//show dots
			if math.Mod(float64(i), 20) == 0 {
				fmt.Print(".")
			}
		}
    }
	perc := (100 * tot) / len
	prog := progress(perc)
	nowTime := time.Now()
	totTime := nowTime.Sub(startTime)
	spd := float64(tot / 1000) / totTime.Seconds()
	if len < 1 {
		fmt.Printf("\r     [ <=>                                  ] %d\t-.--KB/s in %0.1fs             ", tot, totTime.Seconds())
		fmt.Printf("\n (%0.2fKB/s) - '%v' saved [%v]\n", spd, filename, tot)
	} else {
		fmt.Printf("\r%3d%% [%s] %d\t%0.2fKB/s in %0.1fs             ", perc, prog, tot, spd, totTime.Seconds())
		fmt.Printf("\n '%v' saved [%v/%v]\n", filename, tot, len)
	}
	if err != nil {
		return err
	}
	err = out.Close()
	return err
}

func progress(perc int64) string {
	equalses := perc * 38 / 100
	if equalses < 0 {
		equalses = 0
	}
	spaces := 38 - equalses
	if spaces < 0 {
		spaces = 0
	}
	prog := strings.Repeat("=", int(equalses)) + ">" + strings.Repeat(" ", int(spaces))
	return prog
}

func getFilename(resp *http.Response) (string, error) {
	filename := filepath.Base(resp.Request.URL.Path)
	filename = tidyFilename(filename)

	if !strings.Contains(filename, ".") {
		ct := resp.Header.Get("Content-Type")
		//println(ct)
		ext := "htm"
		mediatype, _, err := mime.ParseMediaType(ct)
		if err != nil {
			fmt.Printf("mime error: %v\n", err)
		} else {
			fmt.Printf("mime type: %v (from Content-Type %v)\n", mediatype, ct)
			slash := strings.Index(mediatype, "/")
			if slash != -1 {
				_, sub := mediatype[:slash], mediatype[slash+1:]
				if sub != "" {
					ext = sub
				}
			}
		}
		filename = filename + "." + ext
	}
	_, err := os.Stat(filename)
	if err != nil {
		if os.IsNotExist(err) {
			return filename, nil
		} else {
			return "", err
		}
	} else {
		num := 1
		//just stop after 100
		for num < 100 {
			filenameNew := filename + "." + strconv.Itoa(num)
			_, err := os.Stat(filenameNew)
			if err != nil {
				if os.IsNotExist(err) {
					return filenameNew, nil
				} else {
					return "", err
				}
			}
			num += 1
		}
		return filename, errors.New("Stopping after trying 100 filename variants")
	}
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"html/template"
	"io/ioutil"
	"net/http"
	"os"
	"regexp"
	"sort"

	"log"

	"fmt"

	"github.com/golang-commonmark/markdown"
)

var wikiDir string

type basePage struct {
	Title string
	Nav   []string
}
type wikiPage struct {
	Body     template.HTML
	Created  string
	Modified string
	basePage
}
type searchPage struct {
	basePage
	Results []QueryResults
}

func checkErr(err error) {
	if err != nil {
		panic(err)
	}
}

func (p *wikiPage) save() error {
	filename := wikiDir + p.Title
	return ioutil.WriteFile(filename, []byte(p.Body), 0600)
}

func convertMarkdown(page *wikiPage, err error) (*wikiPage, error) {
	if err != nil {
		return page, err
	}
	md := markdown.New(markdown.HTML(true))
	page.Body = template.HTML(md.RenderToString([]byte(page.Body)))
	return page, nil

}
func loadPage(p *wikiPage) (*wikiPage, error) {
	filename := wikiDir + p.Title

	file, err := os.Open(filename)
	if err != nil {
		fmt.Println(err)
		return p, err
	}
	defer file.Close()

	body, err := ioutil.ReadAll(file)
	if err != nil {
		fmt.Println(err)
		return p, err
	}
	p.Body = template.HTML(body)

	info, err := file.Stat()
	if err != nil {
		fmt.Println(err)
		return p, err
	}

	p.Modified = info.ModTime().String()
	return p, nil
}

func viewHandler(w http.ResponseWriter, r *http.Request, p *wikiPage) {
	p, err := convertMarkdown(loadPage(p))
	if err != nil {
		http.Redirect(w, r, "/edit/"+p.Title, http.StatusFound)
		return
	}
	p.Body = template.HTML(parseWikiWords([]byte(p.Body)))
	renderTemplate(w, "view", p)
}

func editHandler(w http.ResponseWriter, r *http.Request, p *wikiPage) {
	p, _ = loadPage(p)
	renderTemplate(w, "edit", p)
}

func searchHandler(fn navFunc) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		term := r.URL.Query().Get("term") // Get the search term
		if len(term) == 0 {
			http.NotFound(w, r)
			return
		}

		results := ParseQueryResults(SearchWikis(wikiDir, term))
		p := &searchPage{Results: results, basePage: basePage{Title: "Search", Nav: fn()}}

		renderTemplate(w, "search", p)
	}
}

type navFunc func() []string

func homeHandler(page string, fn navFunc) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		renderTemplate(w, page, fn())
	}

}

func saveHandler(w http.ResponseWriter, r *http.Request, p *wikiPage) {
	body := r.FormValue("body")
	p.Body = template.HTML(body)
	err := p.save()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	http.Redirect(w, r, "/view/"+p.Title, http.StatusFound)
}

var templates = template.Must(template.ParseFiles(
	"views/edit.html",
	"views/view.html",
	"views/home.html",
	"views/search.html",
	"views/index.html",
	"views/leftnav.html"))

func renderTemplate(w http.ResponseWriter, tmpl string, p interface{}) {
	err := templates.ExecuteTemplate(w, tmpl+".html", p)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

var validPath = regexp.MustCompile("^/(edit|save|view|search)/([a-zA-Z0-9\\.\\-_ ]*)$")

func makeHandler(fn func(http.ResponseWriter, *http.Request, *wikiPage), navfn navFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		wword := r.URL.Query().Get("wword") // Get the wiki word param if available
		if len(wword) == 0 {
			m := validPath.FindStringSubmatch(r.URL.Path)
			if m == nil {
				fmt.Println("Dont like " + r.URL.Path)
				http.NotFound(w, r)
				return
			}
			wword = m[2]
		}
		p := &wikiPage{basePage: basePage{Title: wword, Nav: navfn()}}
		fn(w, r, p)
	}
}

type byModTime []os.FileInfo

func (m byModTime) Len() int           { return len(m) }
func (m byModTime) Swap(i, j int)      { m[i], m[j] = m[j], m[i] }
func (m byModTime) Less(i, j int) bool { return m[i].ModTime().Before(m[j].ModTime()) }

func getNav() []string {
	return getWikiList(wikiDir)
}
func getWikiList(path string) []string {
	files, err := ioutil.ReadDir(path)
	checkErr(err)

	sort.Sort(sort.Reverse(byModTime(files)))

	var names []string
	for _, f := range files {
		names = append(names, f.Name())
	}

	return names

}

func parseWikiWords(target []byte) []byte {
	var wikiWord = regexp.MustCompile(`\{([^\}]+)\}`)

	return wikiWord.ReplaceAll(target, []byte("$1"))
}

func main() {

	config, err := LoadConfig("config.json")
	if err != nil {
		log.Fatal(err)
	}

	wikiDir = config.WikiDir

	os.Mkdir(config.WikiDir, 0755)
	http.HandleFunc("/", homeHandler("home", getNav))
	http.HandleFunc("/search/", searchHandler(getNav))
	http.HandleFunc("/view/", makeHandler(viewHandler, getNav))
	http.HandleFunc("/edit/", makeHandler(editHandler, getNav))
	http.HandleFunc("/save/", makeHandler(saveHandler, getNav))
	http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))

	http.ListenAndServe(":8080", nil)
}
<commit_msg>Change to use .md extension<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"html/template"
	"io/ioutil"
	"net/http"
	"os"
	"regexp"
	"sort"

	"log"

	"fmt"

	"strings"

	"github.com/golang-commonmark/markdown"
)

var wikiDir string

type basePage struct {
	Title string
	Nav   []string
}
type wikiPage struct {
	Body     template.HTML
	Created  string
	Modified string
	basePage
}
type searchPage struct {
	basePage
	Results []QueryResults
}

func checkErr(err error) {
	if err != nil {
		panic(err)
	}
}

func (p *wikiPage) save() error {
	filename := wikiDir + p.Title + ".md"
	return ioutil.WriteFile(filename, []byte(p.Body), 0600)
}

func convertMarkdown(page *wikiPage, err error) (*wikiPage, error) {
	if err != nil {
		return page, err
	}
	md := markdown.New(markdown.HTML(true))
	page.Body = template.HTML(md.RenderToString([]byte(page.Body)))
	return page, nil

}
func loadPage(p *wikiPage) (*wikiPage, error) {
	filename := wikiDir + p.Title + ".md"

	file, err := os.Open(filename)
	if err != nil {
		fmt.Println(err)
		return p, err
	}
	defer file.Close()

	body, err := ioutil.ReadAll(file)
	if err != nil {
		fmt.Println(err)
		return p, err
	}
	p.Body = template.HTML(body)

	info, err := file.Stat()
	if err != nil {
		fmt.Println(err)
		return p, err
	}

	p.Modified = info.ModTime().String()
	return p, nil
}

func viewHandler(w http.ResponseWriter, r *http.Request, p *wikiPage) {
	p, err := convertMarkdown(loadPage(p))
	if err != nil {
		http.Redirect(w, r, "/edit/"+p.Title, http.StatusFound)
		return
	}
	p.Body = template.HTML(parseWikiWords([]byte(p.Body)))
	renderTemplate(w, "view", p)
}

func editHandler(w http.ResponseWriter, r *http.Request, p *wikiPage) {
	p, _ = loadPage(p)
	renderTemplate(w, "edit", p)
}

func searchHandler(fn navFunc) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		term := r.URL.Query().Get("term") // Get the search term
		if len(term) == 0 {
			http.NotFound(w, r)
			return
		}

		results := ParseQueryResults(SearchWikis(wikiDir, term))
		p := &searchPage{Results: results, basePage: basePage{Title: "Search", Nav: fn()}}

		renderTemplate(w, "search", p)
	}
}

type navFunc func() []string

func homeHandler(page string, fn navFunc) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		renderTemplate(w, page, fn())
	}

}

func saveHandler(w http.ResponseWriter, r *http.Request, p *wikiPage) {
	body := r.FormValue("body")
	p.Body = template.HTML(body)
	err := p.save()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	http.Redirect(w, r, "/view/"+p.Title, http.StatusFound)
}

var templates = template.Must(template.ParseFiles(
	"views/edit.html",
	"views/view.html",
	"views/home.html",
	"views/search.html",
	"views/index.html",
	"views/leftnav.html"))

func renderTemplate(w http.ResponseWriter, tmpl string, p interface{}) {
	err := templates.ExecuteTemplate(w, tmpl+".html", p)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

var validPath = regexp.MustCompile("^/(edit|save|view|search)/([a-zA-Z0-9\\.\\-_ ]*)$")

func makeHandler(fn func(http.ResponseWriter, *http.Request, *wikiPage), navfn navFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		wword := r.URL.Query().Get("wword") // Get the wiki word param if available
		if len(wword) == 0 {
			m := validPath.FindStringSubmatch(r.URL.Path)
			if m == nil {
				fmt.Println("Dont like " + r.URL.Path)
				http.NotFound(w, r)
				return
			}
			wword = m[2]
		}
		p := &wikiPage{basePage: basePage{Title: wword, Nav: navfn()}}
		fn(w, r, p)
	}
}

type byModTime []os.FileInfo

func (m byModTime) Len() int           { return len(m) }
func (m byModTime) Swap(i, j int)      { m[i], m[j] = m[j], m[i] }
func (m byModTime) Less(i, j int) bool { return m[i].ModTime().Before(m[j].ModTime()) }

func getNav() []string {
	return getWikiList(wikiDir)
}
func getWikiList(path string) []string {
	files, err := ioutil.ReadDir(path)
	checkErr(err)

	sort.Sort(sort.Reverse(byModTime(files)))

	var names []string
	for _, f := range files {
		names = append(names, strings.TrimSuffix(f.Name(), ".md"))
	}

	return names

}

func parseWikiWords(target []byte) []byte {
	var wikiWord = regexp.MustCompile(`\{([^\}]+)\}`)

	return wikiWord.ReplaceAll(target, []byte("$1"))
}

func main() {

	config, err := LoadConfig("config.json")
	if err != nil {
		log.Fatal(err)
	}

	wikiDir = config.WikiDir

	os.Mkdir(config.WikiDir, 0755)
	http.HandleFunc("/", homeHandler("home", getNav))
	http.HandleFunc("/search/", searchHandler(getNav))
	http.HandleFunc("/view/", makeHandler(viewHandler, getNav))
	http.HandleFunc("/edit/", makeHandler(editHandler, getNav))
	http.HandleFunc("/save/", makeHandler(saveHandler, getNav))
	http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))

	http.ListenAndServe(":8080", nil)
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"syscall"

	"cloud.google.com/go/pubsub"
	"cloud.google.com/go/storage"
	"golang.org/x/net/context"
)

const updateTopicString string = "onesie-updates"
const updateSub string = "onesie-server"

func main() {
	log.Println("Starting")
	pubsubClient, err := pubsub.NewClient(context.Background(), "940380154622")
	if err != nil {
		log.Fatalf("Failed to create client: %v", err)
	}

	updateTopic := pubsubClient.Topic(updateTopicString)
	sub := pubsubClient.Subscription(updateSub)
	b, err := sub.Exists(context.Background())
	if err != nil {
		log.Fatal(err)
	}
	if !b {
		sub, err = pubsubClient.CreateSubscription(context.Background(), updateSub, pubsub.SubscriptionConfig{
			Topic: updateTopic,
		})
		if err != nil {
			log.Fatal(err)
		}
	}

	// Make sure all files are public.
	go func(context ctx) {
		// Open google storage client
		client, err := storage.NewClient(ctx)
		if err != nil {
			log.Panicf("Error connecting to Google Storage: %+v", err)
		}
		defer client.Close()
		bkt := client.Bucket("onesie")
		it := bkt.Objects(ctx, nil)
		for {
			obj, err := it.Next()
			if err == iterator.Done {
				break
			}
			if err != nil {
				return err
			}

			// Let everyone read it.
			if err := obj.ACL().Set(ctx, storage.AllUsers, storage.RoleReader); err != nil {
				log.Infof("Could not set acl for %+v", obj)
				return err
			}
		}
	}(ctx)

	var mu sync.Mutex
	received := 0
	cctx, cancel := context.WithCancel(context.Background())
	err = sub.Receive(cctx, func(ctx context.Context, msg *pubsub.Message) {
		mu.Lock()
		defer mu.Unlock()
		received++
		if received >= 4 {
			cancel()
			msg.Nack()
			return
		}

		msgStr := string(msg.Data)
		fmt.Printf("Got message: %q\n", msgStr)

		if msgStr == "deploy" {
			domain := msg.Attributes["domain"]
			path := msg.Attributes["path"]
			log.Printf("Opening for archive: %s", path)

			// Open google storage client
			client, err := storage.NewClient(ctx)
			if err != nil {
				log.Panicf("Error connecting to Google Storage: %+v", err)
			}
			defer client.Close()
			bkt := client.Bucket("onesie")
			obj := bkt.Object(path)
			r, err := obj.NewReader(ctx)
			if err != nil {
				log.Panicf("Error opening object: %+v", err)
			}
			defer r.Close()

			// Expand into archive
			archive, err := gzip.NewReader(r)
			if err != nil {
				log.Panicf("Error creating gzip reader: %+v", err)
			}
			defer archive.Close()

			// Go through file by file
			tarReader := tar.NewReader(archive)
			buf := make([]byte, 160)
			for {
				header, err := tarReader.Next()
				if err == io.EOF {
					break
				} else if err != nil {
					log.Panicf("Error reading tar: %+v", err)
				}

				path := filepath.Join(domain, header.Name)
				switch header.Typeflag {
				case tar.TypeDir:
					continue
				case tar.TypeReg:
					w := bkt.Object(path).NewWriter(ctx)
					defer w.Close()
					w.ACL = []storage.ACLRule{{Entity: storage.AllUsers, Role: storage.RoleReader}}
					if filepath.Ext(path) != "" {
						w.ObjectAttrs.ContentType = mime.TypeByExtension(filepath.Ext(path))
					}
					wrtn, err := io.CopyBuffer(w, tarReader, buf)
					if err != nil {
						log.Printf("Error writing data to GCS: %+v", err)
					}
					log.Printf("Wrote %v bytes to %s", wrtn, path)
				default:
					log.Printf("Unable to figure out type: %v (%s)", header.Typeflag, path)
				}
			}
		}

		if msgStr == "update" {
			// Merge Certs
			files, err := ioutil.ReadDir("/opt/onesie-configs/certs/")
			if err != nil {
				log.Fatalf("Error iterating through files: %+v", err)
			}

			dhparam, err := ioutil.ReadFile(fmt.Sprintf("/opt/onesie-configs/dhparam.pem"))
			if err != nil {
				log.Fatalf("Error reading dhparam: %+v", err)
			}

			for _, file := range files {
				// cat /opt/onesie-configs/certs/$i/{privkey,fullchain}.pem /opt/onesie-configs/dhparam.pem > /opt/onesie-configs/hitch/$i.pem
				log.Printf("Parsing file: %+v", file)
				privkey, err := ioutil.ReadFile(fmt.Sprintf("/opt/onesie-configs/certs/%s/privkey.pem", file.Name()))
				if err != nil {
					log.Printf("Error reading privkey: %+v", err)
					continue
				}
				fullchain, err := ioutil.ReadFile(fmt.Sprintf("/opt/onesie-configs/certs/%s/fullchain.pem", file.Name()))
				if err != nil {
					log.Printf("Error reading fullchain: %+v", err)
					continue
				}

				// Write out
				f, err := os.OpenFile(fmt.Sprintf("/opt/onesie-configs/hitch/%s.pem", file.Name()), os.O_APPEND|os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
				if err != nil {
					log.Printf("Error opening output pem: %+v", err)
					continue
				}
				defer f.Close()

				if _, err = f.Write(privkey); err != nil {
					log.Printf("Error writing privkey to output pem: %+v", err)
					continue
				}
				if _, err = f.Write(fullchain); err != nil {
					log.Printf("Error writing fullchain to output pem: %+v", err)
					continue
				}
				if _, err = f.Write(dhparam); err != nil {
					log.Printf("Error writing dhparam to output pem: %+v", err)
					continue
				}
			}

			// Get hitch PID, send sighup
			out, err := exec.Command("/bin/pidof", "hitch").Output()
			if err != nil {
				log.Printf("Error running pidof: %+v", err)
			}
			for _, pidStr := range strings.Split(string(out), " ") {
				pid, err := strconv.Atoi(strings.TrimSpace(pidStr))
				if err != nil {
					log.Printf("Error parsing string: %+v", err)
					continue
				}
				log.Printf("Sending SIGHUP to %+v", pid)
				syscall.Kill(pid, syscall.SIGHUP)
			}
		}
		msg.Ack()
	})

	if err != nil {
		log.Println(err)
	}

	log.Println("Finished.")
}
<commit_msg>Fix compile errors<commit_after>package main

import (
	"archive/tar"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"syscall"

	"cloud.google.com/go/pubsub"
	"cloud.google.com/go/storage"
	"golang.org/x/net/context"
	"google.golang.org/api/iterator"
)

const updateTopicString string = "onesie-updates"
const updateSub string = "onesie-server"

func main() {
	log.Println("Starting")
	pubsubClient, err := pubsub.NewClient(context.Background(), "940380154622")
	if err != nil {
		log.Fatalf("Failed to create client: %v", err)
	}

	updateTopic := pubsubClient.Topic(updateTopicString)
	sub := pubsubClient.Subscription(updateSub)
	b, err := sub.Exists(context.Background())
	if err != nil {
		log.Fatal(err)
	}
	if !b {
		sub, err = pubsubClient.CreateSubscription(context.Background(), updateSub, pubsub.SubscriptionConfig{
			Topic: updateTopic,
		})
		if err != nil {
			log.Fatal(err)
		}
	}

	// Make sure all files are public.
	go func(c context.Context) {
		// Open google storage client
		client, err := storage.NewClient(c)
		if err != nil {
			log.Panicf("Error connecting to Google Storage: %+v", err)
		}
		defer client.Close()
		bkt := client.Bucket("onesie")
		it := bkt.Objects(c, nil)
		for {
			objAttrs, err := it.Next()
			if err == iterator.Done {
				break
			}
			if err != nil {
				log.Fatalf("Could not iterate: %+v", err)
			}

			// Let everyone read it.
			obj := bkt.Object(objAttrs.Name)
			if err := obj.ACL().Set(c, storage.AllUsers, storage.RoleReader); err != nil {
				log.Fatalf("Could not set acl for %+v", obj)
			}
		}
	}(context.Background())

	var mu sync.Mutex
	received := 0
	cctx, cancel := context.WithCancel(context.Background())
	err = sub.Receive(cctx, func(ctx context.Context, msg *pubsub.Message) {
		mu.Lock()
		defer mu.Unlock()
		received++
		if received >= 4 {
			cancel()
			msg.Nack()
			return
		}

		msgStr := string(msg.Data)
		fmt.Printf("Got message: %q\n", msgStr)

		if msgStr == "deploy" {
			domain := msg.Attributes["domain"]
			path := msg.Attributes["path"]
			log.Printf("Opening for archive: %s", path)

			// Open google storage client
			client, err := storage.NewClient(ctx)
			if err != nil {
				log.Panicf("Error connecting to Google Storage: %+v", err)
			}
			defer client.Close()
			bkt := client.Bucket("onesie")
			obj := bkt.Object(path)
			r, err := obj.NewReader(ctx)
			if err != nil {
				log.Panicf("Error opening object: %+v", err)
			}
			defer r.Close()

			// Expand into archive
			archive, err := gzip.NewReader(r)
			if err != nil {
				log.Panicf("Error creating gzip reader: %+v", err)
			}
			defer archive.Close()

			// Go through file by file
			tarReader := tar.NewReader(archive)
			buf := make([]byte, 160)
			for {
				header, err := tarReader.Next()
				if err == io.EOF {
					break
				} else if err != nil {
					log.Panicf("Error reading tar: %+v", err)
				}

				path := filepath.Join(domain, header.Name)
				switch header.Typeflag {
				case tar.TypeDir:
					continue
				case tar.TypeReg:
					w := bkt.Object(path).NewWriter(ctx)
					defer w.Close()
					w.ACL = []storage.ACLRule{{Entity: storage.AllUsers, Role: storage.RoleReader}}
					if filepath.Ext(path) != "" {
						w.ObjectAttrs.ContentType = mime.TypeByExtension(filepath.Ext(path))
					}
					wrtn, err := io.CopyBuffer(w, tarReader, buf)
					if err != nil {
						log.Printf("Error writing data to GCS: %+v", err)
					}
					log.Printf("Wrote %v bytes to %s", wrtn, path)
				default:
					log.Printf("Unable to figure out type: %v (%s)", header.Typeflag, path)
				}
			}
		}

		if msgStr == "update" {
			// Merge Certs
			files, err := ioutil.ReadDir("/opt/onesie-configs/certs/")
			if err != nil {
				log.Fatalf("Error iterating through files: %+v", err)
			}

			dhparam, err := ioutil.ReadFile(fmt.Sprintf("/opt/onesie-configs/dhparam.pem"))
			if err != nil {
				log.Fatalf("Error reading dhparam: %+v", err)
			}

			for _, file := range files {
				// cat /opt/onesie-configs/certs/$i/{privkey,fullchain}.pem /opt/onesie-configs/dhparam.pem > /opt/onesie-configs/hitch/$i.pem
				log.Printf("Parsing file: %+v", file)
				privkey, err := ioutil.ReadFile(fmt.Sprintf("/opt/onesie-configs/certs/%s/privkey.pem", file.Name()))
				if err != nil {
					log.Printf("Error reading privkey: %+v", err)
					continue
				}
				fullchain, err := ioutil.ReadFile(fmt.Sprintf("/opt/onesie-configs/certs/%s/fullchain.pem", file.Name()))
				if err != nil {
					log.Printf("Error reading fullchain: %+v", err)
					continue
				}

				// Write out
				f, err := os.OpenFile(fmt.Sprintf("/opt/onesie-configs/hitch/%s.pem", file.Name()), os.O_APPEND|os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
				if err != nil {
					log.Printf("Error opening output pem: %+v", err)
					continue
				}
				defer f.Close()

				if _, err = f.Write(privkey); err != nil {
					log.Printf("Error writing privkey to output pem: %+v", err)
					continue
				}
				if _, err = f.Write(fullchain); err != nil {
					log.Printf("Error writing fullchain to output pem: %+v", err)
					continue
				}
				if _, err = f.Write(dhparam); err != nil {
					log.Printf("Error writing dhparam to output pem: %+v", err)
					continue
				}
			}

			// Get hitch PID, send sighup
			out, err := exec.Command("/bin/pidof", "hitch").Output()
			if err != nil {
				log.Printf("Error running pidof: %+v", err)
			}
			for _, pidStr := range strings.Split(string(out), " ") {
				pid, err := strconv.Atoi(strings.TrimSpace(pidStr))
				if err != nil {
					log.Printf("Error parsing string: %+v", err)
					continue
				}
				log.Printf("Sending SIGHUP to %+v", pid)
				syscall.Kill(pid, syscall.SIGHUP)
			}
		}
		msg.Ack()
	})

	if err != nil {
		log.Println(err)
	}

	log.Println("Finished.")
}
<|endoftext|>