<commit_before>package main

import (
	"apiproxy/apicache"
	"fmt"
	"log"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

type apiReq struct {
	apiReq  *apicache.Request
	apiResp *apicache.Response

	data   []byte
	apiErr apicache.APIError

	expires time.Time

	worker   int
	httpCode int
	err      error
	respChan chan apiReq
}

var workChan chan apiReq

var apiClient apicache.Client
var activeWorkerCount, workerCount int32

func APIReq(url string, params map[string]string) ([]byte, int, error) {
	var errorStr string

	if atomic.LoadInt32(&workerCount) <= 0 {
		panic("No workers!")
	}
	useLog := atomic.LoadInt32(&logActive)

	// Build the request
	apireq := apicache.NewRequest(url)
	logParams := ""
	var paramVal string
	for k, v := range params {
		apireq.Set(k, v)
		// Show full vcode for log level 3
		if strings.ToLower(k) == "vcode" && useLog < 3 {
			paramVal = params[k][0:8] + "..."
		} else {
			paramVal = params[k]
		}
		logParams = fmt.Sprintf("%s&%s=%s", logParams, k, paramVal)
	}
	if logParams != "" {
		logParams = "?" + logParams[1:]
	}

	workerID := "C"
	// Don't send it to a worker if we can just yank it fromm the cache
	apiResp, err := apireq.GetCached()
	if err != nil {
		respChan := make(chan apiReq)
		req := apiReq{apiReq: apireq, respChan: respChan}
		workChan <- req

		resp := <-respChan
		close(respChan)

		apiResp = resp.apiResp
		err = resp.err
		workerID = fmt.Sprintf("%d", resp.worker)
	}

	if useLog > 0 && apiResp.Error.ErrorCode != 0 {
		errorStr = fmt.Sprintf(" Error %d: %s", apiResp.Error.ErrorCode, apiResp.Error.ErrorText)
	}
	if useLog != 0 || apiResp.HTTPCode != 200 {
		log.Printf("w%s: %s%s FromCache: %v HTTP: %d Expires: %s%s", workerID, url, logParams, apiResp.FromCache, apiResp.HTTPCode, apiResp.Expires.Format("2006-01-02 15:04:05"), errorStr)
	}

	return apiResp.Data, apiResp.HTTPCode, err
}

var logActive int32

func worker(reqChan chan apiReq, workerID int) {
	time.Sleep(time.Duration(workerID) * time.Second)

	atomic.AddInt32(&workerCount, 1)
	for req := range reqChan {
		atomic.AddInt32(&activeWorkerCount, 1)

		resp, err := req.apiReq.Do()
		req.apiResp = resp
		req.err = err
		req.worker = workerID

		req.respChan <- req
		atomic.AddInt32(&activeWorkerCount, -1)
	}
	atomic.AddInt32(&workerCount, -1)
}

var startWorkersOnce = &sync.Once{}

func startWorkers() {
	startWorkersOnce.Do(realStartWorkers)
}

func realStartWorkers() {
	log.Printf("Starting %d Workers...", conf.Workers)
	workChan = make(chan apiReq)

	for i := 0; i < conf.Workers; i++ {
		go worker(workChan, i)
	}
}

func stopWorkers() {
	close(workChan)
	for atomic.LoadInt32(&activeWorkerCount) > 0 {
		time.Sleep(10 * time.Millisecond)
	}
	startWorkersOnce = &sync.Once{}
}

func PrintWorkerStats() {
	active, loaded := GetWorkerStats()
	log.Printf("%d workers idle, %d workers active.", loaded-active, active)
}

func EnableVerboseLogging() int32 {
	newLog := atomic.AddInt32(&logActive, 1)
	return newLog
}

func DisableVerboseLogging() {
	useLog := atomic.LoadInt32(&logActive)
	atomic.AddInt32(&logActive, -useLog)
}

func GetWorkerStats() (int32, int32) {
	return atomic.LoadInt32(&activeWorkerCount), atomic.LoadInt32(&workerCount)
}
<commit_msg>removed extraneous cache indicator from logs<commit_after>package main

import (
	"apiproxy/apicache"
	"fmt"
	"log"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

type apiReq struct {
	apiReq  *apicache.Request
	apiResp *apicache.Response

	data   []byte
	apiErr apicache.APIError

	expires time.Time

	worker   int
	httpCode int
	err      error
	respChan chan apiReq
}

var workChan chan apiReq

var apiClient apicache.Client
var activeWorkerCount, workerCount int32

func APIReq(url string, params map[string]string) ([]byte, int, error) {
	var errorStr string

	if atomic.LoadInt32(&workerCount) <= 0 {
		panic("No workers!")
	}
	useLog := atomic.LoadInt32(&logActive)

	// Build the request
	apireq := apicache.NewRequest(url)
	logParams := ""
	var paramVal string
	for k, v := range params {
		apireq.Set(k, v)
		// Show full vcode for log level 3
		if strings.ToLower(k) == "vcode" && useLog < 3 {
			paramVal = params[k][0:8] + "..."
		} else {
			paramVal = params[k]
		}
		logParams = fmt.Sprintf("%s&%s=%s", logParams, k, paramVal)
	}
	if logParams != "" {
		logParams = "?" + logParams[1:]
	}

	workerID := "C"
	// Don't send it to a worker if we can just yank it fromm the cache
	apiResp, err := apireq.GetCached()
	if err != nil {
		respChan := make(chan apiReq)
		req := apiReq{apiReq: apireq, respChan: respChan}
		workChan <- req

		resp := <-respChan
		close(respChan)

		apiResp = resp.apiResp
		err = resp.err
		workerID = fmt.Sprintf("%d", resp.worker)
	}

	if useLog > 0 && apiResp.Error.ErrorCode != 0 {
		errorStr = fmt.Sprintf(" Error %d: %s", apiResp.Error.ErrorCode, apiResp.Error.ErrorText)
	}
	if useLog != 0 || apiResp.HTTPCode != 200 {
		log.Printf("w%s: %s%s HTTP: %d Expires: %s%s", workerID, url, logParams, apiResp.FromCache, apiResp.HTTPCode, apiResp.Expires.Format("2006-01-02 15:04:05"), errorStr)
	}

	return apiResp.Data, apiResp.HTTPCode, err
}

var logActive int32

func worker(reqChan chan apiReq, workerID int) {
	time.Sleep(time.Duration(workerID) * time.Second)

	atomic.AddInt32(&workerCount, 1)
	for req := range reqChan {
		atomic.AddInt32(&activeWorkerCount, 1)

		resp, err := req.apiReq.Do()
		req.apiResp = resp
		req.err = err
		req.worker = workerID

		req.respChan <- req
		atomic.AddInt32(&activeWorkerCount, -1)
	}
	atomic.AddInt32(&workerCount, -1)
}

var startWorkersOnce = &sync.Once{}

func startWorkers() {
	startWorkersOnce.Do(realStartWorkers)
}

func realStartWorkers() {
	log.Printf("Starting %d Workers...", conf.Workers)
	workChan = make(chan apiReq)

	for i := 0; i < conf.Workers; i++ {
		go worker(workChan, i)
	}
}

func stopWorkers() {
	close(workChan)
	for atomic.LoadInt32(&activeWorkerCount) > 0 {
		time.Sleep(10 * time.Millisecond)
	}
	startWorkersOnce = &sync.Once{}
}

func PrintWorkerStats() {
	active, loaded := GetWorkerStats()
	log.Printf("%d workers idle, %d workers active.", loaded-active, active)
}

func EnableVerboseLogging() int32 {
	newLog := atomic.AddInt32(&logActive, 1)
	return newLog
}

func DisableVerboseLogging() {
	useLog := atomic.LoadInt32(&logActive)
	atomic.AddInt32(&logActive, -useLog)
}

func GetWorkerStats() (int32, int32) {
	return atomic.LoadInt32(&activeWorkerCount), atomic.LoadInt32(&workerCount)
}
<|endoftext|>


<commit_before>package httpfile

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"strconv"

	"github.com/anacrolix/missinggo"
)

type File struct {
	off    int64
	r      io.ReadCloser
	rOff   int64
	length int64
	url    string
}

func Open(url string) *File {
	return &File{
		url: url,
	}
}

func (me *File) prepareReader() (err error) {
	if me.r != nil && me.off != me.rOff {
		me.r.Close()
		me.r = nil
	}
	if me.r != nil {
		return nil
	}
	req, err := http.NewRequest("GET", me.url, nil)
	if err != nil {
		return
	}
	if me.off != 0 {
		req.Header.Set("Range", fmt.Sprintf("bytes=%d-", me.off))
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return
	}
	switch resp.StatusCode {
	case http.StatusPartialContent:
		cr, ok := missinggo.ParseHTTPBytesContentRange(resp.Header.Get("Content-Range"))
		if !ok || cr.First != me.off {
			err = errors.New("bad response")
			resp.Body.Close()
			return
		}
		me.length = cr.Length
	case http.StatusOK:
		if me.off != 0 {
			err = errors.New("bad response")
			resp.Body.Close()
			return
		}
		if h := resp.Header.Get("Content-Length"); h != "" {
			var cl uint64
			cl, err = strconv.ParseUint(h, 10, 64)
			if err != nil {
				resp.Body.Close()
				return
			}
			me.length = int64(cl)
		}
	default:
		err = errors.New(resp.Status)
		resp.Body.Close()
		return
	}
	me.r = resp.Body
	me.rOff = me.off
	return
}

func (me *File) Read(b []byte) (n int, err error) {
	err = me.prepareReader()
	if err != nil {
		return
	}
	n, err = me.r.Read(b)
	me.off += int64(n)
	me.rOff += int64(n)
	return
}

func instanceLength(r *http.Response) (int64, error) {
	switch r.StatusCode {
	case http.StatusOK:
		if h := r.Header.Get("Content-Length"); h != "" {
			return strconv.ParseInt(h, 10, 64)
		} else {
			return -1, nil
		}
	case http.StatusPartialContent:
		cr, ok := missinggo.ParseHTTPBytesContentRange(r.Header.Get("Content-Range"))
		if !ok {
			return -1, errors.New("bad 206 response")
		}
		return cr.Length, nil
	default:
		return -1, errors.New(r.Status)
	}
}

func (me *File) Seek(offset int64, whence int) (ret int64, err error) {
	switch whence {
	case os.SEEK_SET:
		ret = offset
	case os.SEEK_CUR:
		ret = me.off + offset
	case os.SEEK_END:
		if me.length < 0 {
			err = errors.New("length unknown")
			return
		}
		ret = me.length + offset
	default:
		err = fmt.Errorf("unhandled whence: %d", whence)
		return
	}
	me.off = ret
	return
}

func (me *File) Write(b []byte) (n int, err error) {
	req, err := http.NewRequest("PATCH", me.url, bytes.NewReader(b))
	if err != nil {
		return
	}
	req.Header.Set("Content-Range", fmt.Sprintf("bytes=%d-", me.off))
	req.ContentLength = int64(len(b))
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return
	}
	resp.Body.Close()
	if resp.StatusCode != http.StatusPartialContent {
		err = errors.New(resp.Status)
		return
	}
	n = len(b)
	me.off += int64(n)
	return
}

var (
	ErrNotFound = errors.New("not found")
)

func GetLength(url string) (ret int64, err error) {
	resp, err := http.Head(url)
	if err != nil {
		return
	}
	resp.Body.Close()
	if resp.StatusCode == http.StatusNotFound {
		err = ErrNotFound
		return
	}
	return instanceLength(resp)
}

func (me *File) Close() error {
	me.url = ""
	if me.r != nil {
		me.r.Close()
		me.r = nil
	}
	return nil
}
<commit_msg>httpfile: Doc<commit_after>package httpfile

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"strconv"

	"github.com/anacrolix/missinggo"
)

type File struct {
	off    int64
	r      io.ReadCloser
	rOff   int64
	length int64
	url    string
}

func Open(url string) *File {
	return &File{
		url: url,
	}
}

func (me *File) prepareReader() (err error) {
	if me.r != nil && me.off != me.rOff {
		me.r.Close()
		me.r = nil
	}
	if me.r != nil {
		return nil
	}
	req, err := http.NewRequest("GET", me.url, nil)
	if err != nil {
		return
	}
	if me.off != 0 {
		req.Header.Set("Range", fmt.Sprintf("bytes=%d-", me.off))
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return
	}
	switch resp.StatusCode {
	case http.StatusPartialContent:
		cr, ok := missinggo.ParseHTTPBytesContentRange(resp.Header.Get("Content-Range"))
		if !ok || cr.First != me.off {
			err = errors.New("bad response")
			resp.Body.Close()
			return
		}
		me.length = cr.Length
	case http.StatusOK:
		if me.off != 0 {
			err = errors.New("bad response")
			resp.Body.Close()
			return
		}
		if h := resp.Header.Get("Content-Length"); h != "" {
			var cl uint64
			cl, err = strconv.ParseUint(h, 10, 64)
			if err != nil {
				resp.Body.Close()
				return
			}
			me.length = int64(cl)
		}
	default:
		err = errors.New(resp.Status)
		resp.Body.Close()
		return
	}
	me.r = resp.Body
	me.rOff = me.off
	return
}

func (me *File) Read(b []byte) (n int, err error) {
	err = me.prepareReader()
	if err != nil {
		return
	}
	n, err = me.r.Read(b)
	me.off += int64(n)
	me.rOff += int64(n)
	return
}

func instanceLength(r *http.Response) (int64, error) {
	switch r.StatusCode {
	case http.StatusOK:
		if h := r.Header.Get("Content-Length"); h != "" {
			return strconv.ParseInt(h, 10, 64)
		} else {
			return -1, nil
		}
	case http.StatusPartialContent:
		cr, ok := missinggo.ParseHTTPBytesContentRange(r.Header.Get("Content-Range"))
		if !ok {
			return -1, errors.New("bad 206 response")
		}
		return cr.Length, nil
	default:
		return -1, errors.New(r.Status)
	}
}

func (me *File) Seek(offset int64, whence int) (ret int64, err error) {
	switch whence {
	case os.SEEK_SET:
		ret = offset
	case os.SEEK_CUR:
		ret = me.off + offset
	case os.SEEK_END:
		if me.length < 0 {
			err = errors.New("length unknown")
			return
		}
		ret = me.length + offset
	default:
		err = fmt.Errorf("unhandled whence: %d", whence)
		return
	}
	me.off = ret
	return
}

func (me *File) Write(b []byte) (n int, err error) {
	req, err := http.NewRequest("PATCH", me.url, bytes.NewReader(b))
	if err != nil {
		return
	}
	req.Header.Set("Content-Range", fmt.Sprintf("bytes=%d-", me.off))
	req.ContentLength = int64(len(b))
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return
	}
	resp.Body.Close()
	if resp.StatusCode != http.StatusPartialContent {
		err = errors.New(resp.Status)
		return
	}
	n = len(b)
	me.off += int64(n)
	return
}

var (
	ErrNotFound = errors.New("not found")
)

// Returns the length of the resource in bytes.
func GetLength(url string) (ret int64, err error) {
	resp, err := http.Head(url)
	if err != nil {
		return
	}
	resp.Body.Close()
	if resp.StatusCode == http.StatusNotFound {
		err = ErrNotFound
		return
	}
	return instanceLength(resp)
}

func (me *File) Close() error {
	me.url = ""
	if me.r != nil {
		me.r.Close()
		me.r = nil
	}
	return nil
}
<|endoftext|>


<commit_before>package mastodon

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"path"
	"strings"
	"time"
)

// Config is a setting for access mastodon APIs.
type Config struct {
	Server       string
	ClientID     string
	ClientSecret string
	AccessToken  string
}

// Client is a API client for mastodon.
type Client struct {
	http.Client
	config *Config
}

func (c *Client) doAPI(method string, uri string, params url.Values, res interface{}) error {
	url, err := url.Parse(c.config.Server)
	if err != nil {
		return err
	}
	url.Path = path.Join(url.Path, uri)

	var resp *http.Response
	req, err := http.NewRequest(method, url.String(), strings.NewReader(params.Encode()))
	if err != nil {
		return err
	}
	req.Header.Set("Authorization", "Bearer "+c.config.AccessToken)
	resp, err = c.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if res == nil {
		return nil
	}

	if method == "GET" && resp.StatusCode != http.StatusOK {
		return fmt.Errorf("bad request: %v", resp.Status)
	}

	return json.NewDecoder(resp.Body).Decode(&res)
}

// NewClient return new mastodon API client.
func NewClient(config *Config) *Client {
	return &Client{
		Client: *http.DefaultClient,
		config: config,
	}
}

// Authenticate get access-token to the API.
func (c *Client) Authenticate(username, password string) error {
	params := url.Values{}
	params.Set("client_id", c.config.ClientID)
	params.Set("client_secret", c.config.ClientSecret)
	params.Set("grant_type", "password")
	params.Set("username", username)
	params.Set("password", password)
	params.Set("scope", "read write follow")

	url, err := url.Parse(c.config.Server)
	if err != nil {
		return err
	}
	url.Path = path.Join(url.Path, "/oauth/token")

	req, err := http.NewRequest("POST", url.String(), strings.NewReader(params.Encode()))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	resp, err := c.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("bad authorization: %v", resp.Status)
	}

	res := struct {
		AccessToken string `json:"access_token"`
	}{}
	err = json.NewDecoder(resp.Body).Decode(&res)
	if err != nil {
		return err
	}
	c.config.AccessToken = res.AccessToken
	return nil
}

// AppConfig is a setting for registering applications.
type AppConfig struct {
	http.Client
	Server     string
	ClientName string

	// Where the user should be redirected after authorization (for no redirect, use urn:ietf:wg:oauth:2.0:oob)
	RedirectURIs string

	// This can be a space-separated list of the following items: "read", "write" and "follow".
	Scopes string

	// Optional.
	Website string
}

// Application is mastodon application.
type Application struct {
	ID           int64  `json:"id"`
	RedirectURI  string `json:"redirect_uri"`
	ClientID     string `json:"client_id"`
	ClientSecret string `json:"client_secret"`
}

// RegisterApp returns the mastodon application.
func RegisterApp(appConfig *AppConfig) (*Application, error) {
	params := url.Values{}
	params.Set("client_name", appConfig.ClientName)
	if appConfig.RedirectURIs == "" {
		params.Set("redirect_uris", "urn:ietf:wg:oauth:2.0:oob")
	} else {
		params.Set("redirect_uris", appConfig.RedirectURIs)
	}
	params.Set("scopes", appConfig.Scopes)
	params.Set("website", appConfig.Website)

	url, err := url.Parse(appConfig.Server)
	if err != nil {
		return nil, err
	}
	url.Path = path.Join(url.Path, "/api/v1/apps")

	req, err := http.NewRequest("POST", url.String(), strings.NewReader(params.Encode()))
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	resp, err := appConfig.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("bad request: %v", resp.Status)
	}

	var app Application
	err = json.NewDecoder(resp.Body).Decode(&app)
	if err != nil {
		return nil, err
	}

	return &app, nil
}

// Toot is struct to post status.
type Toot struct {
	Status      string  `json:"status"`
	InReplyToID int64   `json:"in_reply_to_id"`
	MediaIDs    []int64 `json:"media_ids"`
	Sensitive   bool    `json:"sensitive"`
	SpoilerText string  `json:"spoiler_text"`
	Visibility  string  `json:"visibility"`
}

// Status is struct to hold status.
type Status struct {
	ID                 int64         `json:"id"`
	CreatedAt          time.Time     `json:"created_at"`
	InReplyToID        interface{}   `json:"in_reply_to_id"`
	InReplyToAccountID interface{}   `json:"in_reply_to_account_id"`
	Sensitive          bool          `json:"sensitive"`
	SpoilerText        string        `json:"spoiler_text"`
	Visibility         string        `json:"visibility"`
	Application        interface{}   `json:"application"`
	Account            Account       `json:"account"`
	MediaAttachments   []interface{} `json:"media_attachments"`
	Mentions           []interface{} `json:"mentions"`
	Tags               []interface{} `json:"tags"`
	URI                string        `json:"uri"`
	Content            string        `json:"content"`
	URL                string        `json:"url"`
	ReblogsCount       int64         `json:"reblogs_count"`
	FavouritesCount    int64         `json:"favourites_count"`
	Reblog             interface{}   `json:"reblog"`
	Favourited         interface{}   `json:"favourited"`
	Reblogged          interface{}   `json:"reblogged"`
}

// GetTimelineHome return statuses from home timeline.
func (c *Client) GetTimelineHome() ([]*Status, error) {
	var statuses []*Status
	err := c.doAPI("GET", "/api/v1/timelines/home", nil, &statuses)
	if err != nil {
		return nil, err
	}
	return statuses, nil
}

// PostStatus post the toot.
func (c *Client) PostStatus(toot *Toot) (*Status, error) {
	params := url.Values{}
	params.Set("status", toot.Status)
	if toot.InReplyToID > 0 {
		params.Set("in_reply_to_id", fmt.Sprint(toot.InReplyToID))
	}
	// TODO: media_ids, senstitive, spoiler_text, visibility
	//params.Set("visibility", "public")

	var status Status
	err := c.doAPI("POST", "/api/v1/statuses", params, &status)
	if err != nil {
		return nil, err
	}
	return &status, nil
}
<commit_msg>separate files<commit_after>package mastodon

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"path"
	"strings"
	"time"
)

// Config is a setting for access mastodon APIs.
type Config struct {
	Server       string
	ClientID     string
	ClientSecret string
	AccessToken  string
}

// Client is a API client for mastodon.
type Client struct {
	http.Client
	config *Config
}

func (c *Client) doAPI(method string, uri string, params url.Values, res interface{}) error {
	url, err := url.Parse(c.config.Server)
	if err != nil {
		return err
	}
	url.Path = path.Join(url.Path, uri)

	var resp *http.Response
	req, err := http.NewRequest(method, url.String(), strings.NewReader(params.Encode()))
	if err != nil {
		return err
	}
	req.Header.Set("Authorization", "Bearer "+c.config.AccessToken)
	resp, err = c.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if res == nil {
		return nil
	}

	if method == "GET" && resp.StatusCode != http.StatusOK {
		return fmt.Errorf("bad request: %v", resp.Status)
	}

	return json.NewDecoder(resp.Body).Decode(&res)
}

// NewClient return new mastodon API client.
func NewClient(config *Config) *Client {
	return &Client{
		Client: *http.DefaultClient,
		config: config,
	}
}

// Authenticate get access-token to the API.
func (c *Client) Authenticate(username, password string) error {
	params := url.Values{}
	params.Set("client_id", c.config.ClientID)
	params.Set("client_secret", c.config.ClientSecret)
	params.Set("grant_type", "password")
	params.Set("username", username)
	params.Set("password", password)
	params.Set("scope", "read write follow")

	url, err := url.Parse(c.config.Server)
	if err != nil {
		return err
	}
	url.Path = path.Join(url.Path, "/oauth/token")

	req, err := http.NewRequest("POST", url.String(), strings.NewReader(params.Encode()))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	resp, err := c.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("bad authorization: %v", resp.Status)
	}

	res := struct {
		AccessToken string `json:"access_token"`
	}{}
	err = json.NewDecoder(resp.Body).Decode(&res)
	if err != nil {
		return err
	}
	c.config.AccessToken = res.AccessToken
	return nil
}

// Toot is struct to post status.
type Toot struct {
	Status      string  `json:"status"`
	InReplyToID int64   `json:"in_reply_to_id"`
	MediaIDs    []int64 `json:"media_ids"`
	Sensitive   bool    `json:"sensitive"`
	SpoilerText string  `json:"spoiler_text"`
	Visibility  string  `json:"visibility"`
}

// Status is struct to hold status.
type Status struct {
	ID                 int64         `json:"id"`
	CreatedAt          time.Time     `json:"created_at"`
	InReplyToID        interface{}   `json:"in_reply_to_id"`
	InReplyToAccountID interface{}   `json:"in_reply_to_account_id"`
	Sensitive          bool          `json:"sensitive"`
	SpoilerText        string        `json:"spoiler_text"`
	Visibility         string        `json:"visibility"`
	Application        interface{}   `json:"application"`
	Account            Account       `json:"account"`
	MediaAttachments   []interface{} `json:"media_attachments"`
	Mentions           []interface{} `json:"mentions"`
	Tags               []interface{} `json:"tags"`
	URI                string        `json:"uri"`
	Content            string        `json:"content"`
	URL                string        `json:"url"`
	ReblogsCount       int64         `json:"reblogs_count"`
	FavouritesCount    int64         `json:"favourites_count"`
	Reblog             interface{}   `json:"reblog"`
	Favourited         interface{}   `json:"favourited"`
	Reblogged          interface{}   `json:"reblogged"`
}

// GetTimelineHome return statuses from home timeline.
func (c *Client) GetTimelineHome() ([]*Status, error) {
	var statuses []*Status
	err := c.doAPI("GET", "/api/v1/timelines/home", nil, &statuses)
	if err != nil {
		return nil, err
	}
	return statuses, nil
}

// PostStatus post the toot.
func (c *Client) PostStatus(toot *Toot) (*Status, error) {
	params := url.Values{}
	params.Set("status", toot.Status)
	if toot.InReplyToID > 0 {
		params.Set("in_reply_to_id", fmt.Sprint(toot.InReplyToID))
	}
	// TODO: media_ids, senstitive, spoiler_text, visibility
	//params.Set("visibility", "public")

	var status Status
	err := c.doAPI("POST", "/api/v1/statuses", params, &status)
	if err != nil {
		return nil, err
	}
	return &status, nil
}
<|endoftext|>


<commit_before>package peco

import (
	"fmt"
	"regexp"
	"sort"
	"strings"
)

const (
	CaseSensitiveMatch = iota
	IgnoreCaseMatch
)

type StraightMatcher struct {
	flags []string
}

type CaseSensitiveMatcher struct {
	StraightMatcher
}

type IgnoreCaseMatcher struct {
	StraightMatcher
}

func NewCaseSensitiveMatcher() *CaseSensitiveMatcher {
	return &CaseSensitiveMatcher{StraightMatcher{nil}}
}

func NewIgnoreCaseMatcher() *IgnoreCaseMatcher {
	return &IgnoreCaseMatcher{StraightMatcher{[]string{"i"}}}
}

func regexpFor(q string, flags []string) (*regexp.Regexp, error) {
	var reTxt string
	if flags == nil || len(flags) <= 0 {
		reTxt = fmt.Sprintf("%s", regexp.QuoteMeta(q))
	} else {
		reTxt = fmt.Sprintf("(?%s)%s", strings.Join(flags, ""), regexp.QuoteMeta(q))
	}
	re, err := regexp.Compile(reTxt)
	if err != nil {
		return nil, err
	}
	return re, nil
}

func (m *StraightMatcher) QueryToRegexps(query string) ([]*regexp.Regexp, error) {
	queries := strings.Split(strings.TrimSpace(query), " ")
	regexps := make([]*regexp.Regexp, 0)

	for _, q := range queries {
		re, err := regexpFor(q, m.flags)
		if err != nil {
			return nil, err
		}
		regexps = append(regexps, re)
	}

	return regexps, nil
}

func (m *CaseSensitiveMatcher) String() string {
	return "CaseSentive"
}

func (m *IgnoreCaseMatcher) String() string {
	return "IgnoreCase"
}

// sort related stuff
type byStart [][]int

func (m byStart) Len() int {
	return len(m)
}

func (m byStart) Swap(i, j int) {
	m[i], m[j] = m[j], m[i]
}

func (m byStart) Less(i, j int) bool {
	return m[i][0] < m[j][0]
}

func (m *StraightMatcher) Match(q string, buffer []Match) []Match {
	results := []Match{}
	regexps, err := m.QueryToRegexps(q)
	if err != nil {
		return []Match{}
	}

	for _, line := range buffer {
		ms := m.MatchAllRegexps(regexps, line.line)
		if ms == nil {
			continue
		}
		results = append(results, Match{line.line, ms})
	}
	return results
}

func (m *StraightMatcher) MatchAllRegexps(regexps []*regexp.Regexp, line string) [][]int {
	matches := make([][]int, 0)

	allMatched := true
Match:
	for _, re := range regexps {
		match := re.FindAllStringSubmatchIndex(line, 1)
		if match == nil {
			allMatched = false
			break Match
		}

		start, end := match[0][0], match[0][1]
		for _, m := range matches {
			if start >= m[0] && start < m[1] {
				continue Match
			}

			if start < m[0] && end >= m[0] {
				continue Match
			}
		}

		matches = append(matches, match[0])
		sort.Sort(byStart(matches))
	}

	if !allMatched {
		return nil
	}

	return matches
}
<commit_msg>Apply syohex's fix<commit_after>package peco

import (
	"fmt"
	"regexp"
	"sort"
	"strings"
)

const (
	CaseSensitiveMatch = iota
	IgnoreCaseMatch
)

type StraightMatcher struct {
	flags []string
}

type CaseSensitiveMatcher struct {
	StraightMatcher
}

type IgnoreCaseMatcher struct {
	StraightMatcher
}

func NewCaseSensitiveMatcher() *CaseSensitiveMatcher {
	return &CaseSensitiveMatcher{StraightMatcher{nil}}
}

func NewIgnoreCaseMatcher() *IgnoreCaseMatcher {
	return &IgnoreCaseMatcher{StraightMatcher{[]string{"i"}}}
}

func regexpFor(q string, flags []string) (*regexp.Regexp, error) {
	var reTxt string
	if flags == nil || len(flags) <= 0 {
		reTxt = fmt.Sprintf("%s", regexp.QuoteMeta(q))
	} else {
		reTxt = fmt.Sprintf("(?%s)%s", strings.Join(flags, ""), regexp.QuoteMeta(q))
	}
	re, err := regexp.Compile(reTxt)
	if err != nil {
		return nil, err
	}
	return re, nil
}

func (m *StraightMatcher) QueryToRegexps(query string) ([]*regexp.Regexp, error) {
	queries := strings.Split(strings.TrimSpace(query), " ")
	regexps := make([]*regexp.Regexp, 0)

	for _, q := range queries {
		re, err := regexpFor(q, m.flags)
		if err != nil {
			return nil, err
		}
		regexps = append(regexps, re)
	}

	return regexps, nil
}

func (m *CaseSensitiveMatcher) String() string {
	return "CaseSentive"
}

func (m *IgnoreCaseMatcher) String() string {
	return "IgnoreCase"
}

// sort related stuff
type byStart [][]int

func (m byStart) Len() int {
	return len(m)
}

func (m byStart) Swap(i, j int) {
	m[i], m[j] = m[j], m[i]
}

func (m byStart) Less(i, j int) bool {
	return m[i][0] < m[j][0]
}

func (m *StraightMatcher) Match(q string, buffer []Match) []Match {
	results := []Match{}
	regexps, err := m.QueryToRegexps(q)
	if err != nil {
		return []Match{}
	}

	for _, line := range buffer {
		ms := m.MatchAllRegexps(regexps, line.line)
		if ms == nil {
			continue
		}
		results = append(results, Match{line.line, ms})
	}
	return results
}

func (m *StraightMatcher) MatchAllRegexps(regexps []*regexp.Regexp, line string) [][]int {
	matches := make([][]int, 0)

	allMatched := true
Match:
	for _, re := range regexps {
		match := re.FindAllStringSubmatchIndex(line, -1)
		if match == nil {
			allMatched = false
			break Match
		}

		for _, ma := range match {
			start, end := ma[0], ma[1]
			for _, m := range matches {
				if start >= m[0] && start < m[1] {
					continue Match
				}

				if start < m[0] && end >= m[0] {
					continue Match
				}
			}
			matches = append(matches, ma)
		}
	}

	if !allMatched {
		return nil
	}

	sort.Sort(byStart(matches))

	return matches
}
<|endoftext|>


<commit_before>package libvirt

import (
	"bytes"
	"fmt"
	"html/template"
	"net"
	"net/url"
	"strings"

	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/packer"
	"gopkg.in/alexzorin/libvirt-go.v2"
)

type stepCreateDomain struct{}

func (s *stepCreateDomain) Run(state multistep.StateBag) multistep.StepAction {
	config := state.Get("config").(*Config)
	ui := state.Get("ui").(packer.Ui)

	lv, err := libvirt.NewVirConnection(config.LibvirtUrl)
	if err != nil {
		err := fmt.Errorf("Error connecting to libvirt: %s", err)
		state.Put("error", err)
		ui.Error(err.Error())
		return multistep.ActionHalt
	}
	defer lv.CloseConnection()
	if lvd, err := lv.LookupDomainByName(config.VMName); err != nil {

		domainXml := bytes.NewBuffer(nil)
		tmpl, err := template.New("domain").Parse(config.DomainXml)
		if err != nil {
			err := fmt.Errorf("Error creating domain: %s", err)
			state.Put("error", err)
			ui.Error(err.Error())
			return multistep.ActionHalt
		}

		u, err := url.Parse(config.ISOUrl)
		if err != nil {
			err := fmt.Errorf("Error parse iso_url: %s", err)
			state.Put("error", err)
			ui.Error(err.Error())
			return multistep.ActionHalt
		}
		var h string
		var p string

		if strings.Index(u.Host, ":") == -1 {
			h = u.Host
		} else {
			h, p, _ = net.SplitHostPort(u.Host)
		}
		if p == "" {
			switch u.Scheme {
			case "https":
				p = "443"
			case "http":
				p = "80"
			}
		}

		data := struct {
			VMName      string
			DiskName    string
			DiskType    string
			PoolName    string
			MemorySize  uint
			ISOUrlProto string
			ISOUrlPath  string
			ISOUrlHost  string
			ISOUrlPort  string
			SSHPort     string
		}{
			config.VMName,
			config.DiskName,
			"raw",
			config.PoolName,
			config.MemorySize,
			u.Scheme,
			u.Path,
			h,
			p,
			"2022",
		}
		err = tmpl.Execute(domainXml, data)
		if err != nil {
			err := fmt.Errorf("Error creating domain: %s", err)
			state.Put("error", err)
			ui.Error(err.Error())
			return multistep.ActionHalt
		}

		lvd, err = lv.DomainCreateXML(string(domainXml.Bytes()), 0)
		if err != nil {
			err := fmt.Errorf("Error creating domain: %s", err)
			state.Put("error", err)
			ui.Error(err.Error())
			return multistep.ActionHalt
		}
	} else {
		defer lvd.Free()
	}
	return multistep.ActionContinue
}

func (s *stepCreateDomain) Cleanup(state multistep.StateBag) {

}
<commit_msg>fix<commit_after>package libvirt

import (
	"bytes"
	"fmt"
	"html/template"
	"net"
	"net/url"
	"strings"

	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/packer"
	"gopkg.in/alexzorin/libvirt-go.v2"
)

type stepCreateDomain struct{}

func (s *stepCreateDomain) Run(state multistep.StateBag) multistep.StepAction {
	config := state.Get("config").(*Config)
	ui := state.Get("ui").(packer.Ui)

	lv, err := libvirt.NewVirConnection(config.LibvirtUrl)
	if err != nil {
		err := fmt.Errorf("Error connecting to libvirt: %s", err)
		state.Put("error", err)
		ui.Error(err.Error())
		return multistep.ActionHalt
	}
	defer lv.CloseConnection()
	if lvd, err := lv.LookupDomainByName(config.VMName); err != nil {

		domainXml := bytes.NewBuffer(nil)
		tmpl, err := template.New("domain").Parse(config.DomainXml)
		if err != nil {
			err := fmt.Errorf("Error creating domain: %s", err)
			state.Put("error", err)
			ui.Error(err.Error())
			return multistep.ActionHalt
		}

		u, err := url.Parse(config.ISOUrl)
		if err != nil {
			err := fmt.Errorf("Error parse iso_url: %s", err)
			state.Put("error", err)
			ui.Error(err.Error())
			return multistep.ActionHalt
		}
		var h string
		var p string

		if strings.Index(u.Host, ":") == -1 {
			h = u.Host
		} else {
			h, p, _ = net.SplitHostPort(u.Host)
		}
		if p == "" {
			switch u.Scheme {
			case "https":
				p = "443"
			case "http":
				p = "80"
			}
		}

		data := struct {
			VMName      string
			DiskName    string
			DiskType    string
			PoolName    string
			MemorySize  uint
			ISOUrlProto string
			ISOUrlPath  string
			ISOUrlHost  string
			ISOUrlPort  string
			SSHPort     string
		}{
			config.VMName,
			config.DiskName,
			"raw",
			config.PoolName,
			config.MemorySize,
			u.Scheme,
			u.Path,
			h,
			p,
			"2022",
		}
		err = tmpl.Execute(domainXml, data)
		if err != nil {
			err := fmt.Errorf("Error creating domain: %s", err)
			state.Put("error", err)
			ui.Error(err.Error())
			return multistep.ActionHalt
		}

		ui.Say(fmt.Sprintf("domain config:\n%s", domainXml.Bytes()))

		lvd, err = lv.DomainCreateXML(string(domainXml.Bytes()), 0)
		if err != nil {
			err := fmt.Errorf("Error creating domain: %s", err)
			state.Put("error", err)
			ui.Error(err.Error())
			return multistep.ActionHalt
		}
	} else {
		defer lvd.Free()
	}
	return multistep.ActionContinue
}

func (s *stepCreateDomain) Cleanup(state multistep.StateBag) {

}
<|endoftext|>


<commit_before>package pages

import (
	"io"
	"io/ioutil"
)

// StaticPage is a static page.
type StaticPage struct {
	pageFields
}

// Static returns a bool indicating that the page is a static page.
func (p *StaticPage) Static() bool { return true }

func (p *StaticPage) Write(_ Context, w io.Writer) error {
	b, err := ioutil.ReadFile(p.filename)
	if err != nil {
		return err
	}
	_, err = w.Write(b)
	return err
}
<commit_msg>Use io.Copy to write static files<commit_after>package pages

import (
	"io"
	"os"
)

// StaticPage is a static page.
type StaticPage struct {
	pageFields
}

// Static returns a bool indicating that the page is a static page.
func (p *StaticPage) Static() bool { return true }

func (p *StaticPage) Write(_ Context, w io.Writer) error {
	in, err := os.Open(p.filename)
	if err != nil {
		return err
	}
	defer in.Close() // nolint: errcheck, gas
	_, err = io.Copy(w, in)
	return err
}
<|endoftext|>


<commit_before>package banner

import (
	"../zcrypto/ztls"
	"net"
	"fmt"
	"time"
	"regexp"
)

var smtpEndRegex = regexp.MustCompile(`(?:\r\n)|^[0-9]{3} .+\r\n$`)
var pop3EndRegex = regexp.MustCompile(`(?:\r\n\.\r\n$)|(?:\r\n$)`)
var imapStatusEndRegex = regexp.MustCompile(`\r\n$`)

const SMTP_COMMAND = "STARTTLS\r\n"
const POP3_COMMAND = "STLS\r\n"
const IMAP_COMMAND = "a001 STARTTLS\r\n"

// Implements the net.Conn interface
type Conn struct {
	// Underlying network connection
	conn net.Conn
	tlsConn *ztls.Conn
	isTls bool

	// Keep track of state / network operations
	operations []ConnectionOperation

	// Cache the deadlines so we can reapply after TLS handshake
	readDeadline time.Time
	writeDeadline time.Time
}

func (c *Conn) getUnderlyingConn() (net.Conn) {
	if c.isTls {
		return c.tlsConn
	}
	return c.conn
}

// Layer in the regular conn methods
func (c *Conn) LocalAddr() net.Addr {
	return c.getUnderlyingConn().LocalAddr()
}

func (c *Conn) RemoteAddr() net.Addr {
	return c.getUnderlyingConn().RemoteAddr()
}

func (c *Conn) SetDeadline(t time.Time) error {
	c.readDeadline = t
	c.writeDeadline = t
	return c.getUnderlyingConn().SetDeadline(t)
}

func (c *Conn) SetReadDeadline(t time.Time) error {
	c.readDeadline = t
	return c.getUnderlyingConn().SetReadDeadline(t)
}

func (c *Conn) SetWriteDeadline(t time.Time) error {
	c.writeDeadline = t
	return c.getUnderlyingConn().SetWriteDeadline(t)
}

// Delegate here, but record all the things
func (c *Conn) Write(b []byte) (int, error) {
	n, err := c.getUnderlyingConn().Write(b)
	ws := writeState{toSend: b, err: err}
	c.operations = append(c.operations, &ws)
	return n, err
}

func (c *Conn) Read(b []byte) (int, error) {
	n, err := c.getUnderlyingConn().Read(b)
	rs := readState{response: b[0:n], err: err}
	c.operations = append(c.operations, &rs)
	return n, err
}

func (c *Conn) Close() error {
	return c.getUnderlyingConn().Close()
}

// Extra method - Do a TLS Handshake and record progress
func (c *Conn) TlsHandshake() error {
	if c.isTls {
		return fmt.Errorf(
			"Attempted repeat handshake with remote host %s",
			c.RemoteAddr().String())
	}
	tlsConfig := new(ztls.Config)
	tlsConfig.InsecureSkipVerify = true
	tlsConfig.MinVersion = ztls.VersionSSL30
	c.tlsConn = ztls.Client(c.conn, tlsConfig)
	c.tlsConn.SetReadDeadline(c.readDeadline)
	c.tlsConn.SetWriteDeadline(c.writeDeadline)
	c.isTls = true
	err := c.tlsConn.Handshake()
	hl := c.tlsConn.HandshakeLog()
	ts := tlsState{handshake: hl, err: err}
	c.operations = append(c.operations, &ts)
	return err
}

func (c *Conn) sendStarttlsCommand(command string) error {
	// Don't doublehandshake
	if c.isTls {
		return fmt.Errorf(
			"Attempt STARTTLS after TLS handshake with remote host %s",
			c.RemoteAddr().String())
	}
	// Send the STARTTLS message
	starttls := []byte(command);
	_, err := c.conn.Write(starttls);
	return err
}

// Do a STARTTLS handshake
func (c *Conn) SmtpStarttlsHandshake() error {
	// Make the state
	ss := starttlsState{command: []byte(SMTP_COMMAND)}
	// Send the command
	ss.err = c.sendStarttlsCommand(SMTP_COMMAND)
	// Read the response on a successful send
	if ss.err == nil {
		buf := make([]byte, 256)
		n, err := c.readSmtpResponse(buf)
		ss.response = buf[0:n]
		ss.err = err
	}
	// No matter what happened, record the state
	c.operations = append(c.operations, &ss)
	// Stop if we failed already
	if ss.err != nil {
		return ss.err
	}
	// Successful so far, attempt to do the actual handshake
	return c.TlsHandshake()
}

func (c *Conn) Pop3StarttlsHandshake() error {
	ss := starttlsState{command: []byte(POP3_COMMAND)}
	ss.err = c.sendStarttlsCommand(POP3_COMMAND)
	if ss.err == nil {
		buf := make([]byte, 512)
		n, err := c.readPop3Response(buf)
		ss.response = buf[0:n]
		ss.err = err
	}
	c.operations = append(c.operations, &ss)
	if ss.err != nil {
		return ss.err
	}
	return c.TlsHandshake()
}

func (c *Conn) ImapStarttlsHandshake() error {
	ss := starttlsState{command: []byte(IMAP_COMMAND)}
	ss.err = c.sendStarttlsCommand(IMAP_COMMAND)
	if ss.err == nil {
		buf := make([]byte, 512)
		n, err := c.readImapStatusResponse(buf)
		ss.response = buf[0:n]
		ss.err = err
	}
	c.operations = append(c.operations, &ss)
	if ss.err != nil {
		return ss.err
	}
	return c.TlsHandshake()
}

func (c *Conn) readUntilRegex(res []byte, expr *regexp.Regexp) (int, error) {
	buf := res[0:]
	length := 0
	for finished := false; !finished; {
		n, err := c.getUnderlyingConn().Read(buf);
		length += n
		if err != nil {
			return length, err
		}
		if expr.Match(res[0:length]) {
			finished = true
		} else if length == len(res) {
			b := make([]byte, 3*length)
			copy(b, res)
			res = b
		}
		buf = res[length:]
	}
	return length, nil
}

func (c *Conn) readSmtpResponse(res []byte) (int, error) {
	return c.readUntilRegex(res, smtpEndRegex)
}

func (c *Conn) SmtpBanner(b []byte) (int, error) {
	n, err := c.readSmtpResponse(b)
	rs := readState{}
	rs.response = b[0:n]
	rs.err = err
	c.operations = append(c.operations, &rs)
	return n, err
}

func (c *Conn) Ehlo(domain string) error {
	cmd := []byte("EHLO " + domain + "\r\n")
	es := ehloState{}
	_, writeErr := c.getUnderlyingConn().Write(cmd)
	if writeErr != nil {
		es.err = writeErr
	} else {
		buf := make([]byte, 512)
		n, readErr := c.readSmtpResponse(buf)
		es.err = readErr
		es.response = buf[0:n]
	}
	c.operations = append(c.operations, &es)
	return es.err
}

func (c *Conn) SmtpHelp() error {
	cmd := []byte("HELP\r\n")
	hs := helpState{}
	_, writeErr := c.getUnderlyingConn().Write(cmd)
	if writeErr != nil {
		hs.err = writeErr
	} else {
		buf := make([]byte, 512)
		n, readErr := c.readSmtpResponse(buf)
		hs.err = readErr
		hs.response = buf[0:n]
	}
	c.operations = append(c.operations, &hs)
	return hs.err
}

func (c *Conn) readPop3Response(res []byte) (int, error) {
	return c.readUntilRegex(res, pop3EndRegex)
}

func (c *Conn) Pop3Banner(b []byte) (int, error) {
	n, err := c.readPop3Response(b)
	rs := readState{
		response: b[0:n],
		err: err,
	}
	c.operations = append(c.operations, &rs)
	return n, err
}

func (c *Conn) readImapStatusResponse(res []byte) (int, error) {
	return c.readUntilRegex(res, imapStatusEndRegex)
}

func (c *Conn) ImapBanner(b []byte) (int, error) {
	n, err := c.readImapStatusResponse(b)
	rs := readState {
		response: b[0:n],
		err: err,
	}
	c.operations = append(c.operations, &rs)
	return n, err
}

func (c *Conn) SendHeartbleedProbe(b []byte) (int, error) {
	if !c.isTls {
		return 0, fmt.Errorf(
			"Must perform TLS handshake before sending Heartbleed probe to %s",
			c.RemoteAddr().String())
	}
	n, err := c.tlsConn.CheckHeartbleed(b)
	hl := c.tlsConn.HeartbleedLog()
	if err == ztls.HeartbleedError {
		err = nil
	}
	hs := heartbleedState{probe: hl, err: err}
	c.operations = append(c.operations, &hs)
	return n, err
}

func (c *Conn) States() []StateLog {
	states := make([]StateLog, 0, len(c.operations))
	for _, state := range c.operations {
		states = append(states, state.StateLog())
	}
	return states
}
<commit_msg>Fix bug in readUntilRegex<commit_after>package banner

import (
	"../zcrypto/ztls"
	"net"
	"fmt"
	"time"
	"regexp"
)

var smtpEndRegex = regexp.MustCompile(`(?:\r\n)|^[0-9]{3} .+\r\n$`)
var pop3EndRegex = regexp.MustCompile(`(?:\r\n\.\r\n$)|(?:\r\n$)`)
var imapStatusEndRegex = regexp.MustCompile(`\r\n$`)

const SMTP_COMMAND = "STARTTLS\r\n"
const POP3_COMMAND = "STLS\r\n"
const IMAP_COMMAND = "a001 STARTTLS\r\n"

// Implements the net.Conn interface
type Conn struct {
	// Underlying network connection
	conn net.Conn
	tlsConn *ztls.Conn
	isTls bool

	// Keep track of state / network operations
	operations []ConnectionOperation

	// Cache the deadlines so we can reapply after TLS handshake
	readDeadline time.Time
	writeDeadline time.Time
}

func (c *Conn) getUnderlyingConn() (net.Conn) {
	if c.isTls {
		return c.tlsConn
	}
	return c.conn
}

// Layer in the regular conn methods
func (c *Conn) LocalAddr() net.Addr {
	return c.getUnderlyingConn().LocalAddr()
}

func (c *Conn) RemoteAddr() net.Addr {
	return c.getUnderlyingConn().RemoteAddr()
}

func (c *Conn) SetDeadline(t time.Time) error {
	c.readDeadline = t
	c.writeDeadline = t
	return c.getUnderlyingConn().SetDeadline(t)
}

func (c *Conn) SetReadDeadline(t time.Time) error {
	c.readDeadline = t
	return c.getUnderlyingConn().SetReadDeadline(t)
}

func (c *Conn) SetWriteDeadline(t time.Time) error {
	c.writeDeadline = t
	return c.getUnderlyingConn().SetWriteDeadline(t)
}

// Delegate here, but record all the things
func (c *Conn) Write(b []byte) (int, error) {
	n, err := c.getUnderlyingConn().Write(b)
	ws := writeState{toSend: b, err: err}
	c.operations = append(c.operations, &ws)
	return n, err
}

func (c *Conn) Read(b []byte) (int, error) {
	n, err := c.getUnderlyingConn().Read(b)
	rs := readState{response: b[0:n], err: err}
	c.operations = append(c.operations, &rs)
	return n, err
}

func (c *Conn) Close() error {
	return c.getUnderlyingConn().Close()
}

// Extra method - Do a TLS Handshake and record progress
func (c *Conn) TlsHandshake() error {
	if c.isTls {
		return fmt.Errorf(
			"Attempted repeat handshake with remote host %s",
			c.RemoteAddr().String())
	}
	tlsConfig := new(ztls.Config)
	tlsConfig.InsecureSkipVerify = true
	tlsConfig.MinVersion = ztls.VersionSSL30
	c.tlsConn = ztls.Client(c.conn, tlsConfig)
	c.tlsConn.SetReadDeadline(c.readDeadline)
	c.tlsConn.SetWriteDeadline(c.writeDeadline)
	c.isTls = true
	err := c.tlsConn.Handshake()
	hl := c.tlsConn.HandshakeLog()
	ts := tlsState{handshake: hl, err: err}
	c.operations = append(c.operations, &ts)
	return err
}

func (c *Conn) sendStarttlsCommand(command string) error {
	// Don't doublehandshake
	if c.isTls {
		return fmt.Errorf(
			"Attempt STARTTLS after TLS handshake with remote host %s",
			c.RemoteAddr().String())
	}
	// Send the STARTTLS message
	starttls := []byte(command);
	_, err := c.conn.Write(starttls);
	return err
}

// Do a STARTTLS handshake
func (c *Conn) SmtpStarttlsHandshake() error {
	// Make the state
	ss := starttlsState{command: []byte(SMTP_COMMAND)}
	// Send the command
	ss.err = c.sendStarttlsCommand(SMTP_COMMAND)
	// Read the response on a successful send
	if ss.err == nil {
		buf := make([]byte, 256)
		n, err := c.readSmtpResponse(buf)
		ss.response = buf[0:n]
		ss.err = err
	}
	// No matter what happened, record the state
	c.operations = append(c.operations, &ss)
	// Stop if we failed already
	if ss.err != nil {
		return ss.err
	}
	// Successful so far, attempt to do the actual handshake
	return c.TlsHandshake()
}

func (c *Conn) Pop3StarttlsHandshake() error {
	ss := starttlsState{command: []byte(POP3_COMMAND)}
	ss.err = c.sendStarttlsCommand(POP3_COMMAND)
	if ss.err == nil {
		buf := make([]byte, 512)
		n, err := c.readPop3Response(buf)
		ss.response = buf[0:n]
		ss.err = err
	}
	c.operations = append(c.operations, &ss)
	if ss.err != nil {
		return ss.err
	}
	return c.TlsHandshake()
}

func (c *Conn) ImapStarttlsHandshake() error {
	ss := starttlsState{command: []byte(IMAP_COMMAND)}
	ss.err = c.sendStarttlsCommand(IMAP_COMMAND)
	if ss.err == nil {
		buf := make([]byte, 512)
		n, err := c.readImapStatusResponse(buf)
		ss.response = buf[0:n]
		ss.err = err
	}
	c.operations = append(c.operations, &ss)
	if ss.err != nil {
		return ss.err
	}
	return c.TlsHandshake()
}

func (c *Conn) readUntilRegex(res []byte, expr *regexp.Regexp) (int, error) {
	buf := res[0:]
	length := 0
	for finished := false; !finished; {
		n, err := c.getUnderlyingConn().Read(buf);
		length += n
		if err != nil {
			return length, err
		}
		if expr.Match(res[0:length]) {
			finished = true
		}
		if length == len(res) {
			b := make([]byte, 3*length)
			copy(b, res)
			res = b
		}
		buf = res[length:]
	}
	return length, nil
}

func (c *Conn) readSmtpResponse(res []byte) (int, error) {
	return c.readUntilRegex(res, smtpEndRegex)
}

func (c *Conn) SmtpBanner(b []byte) (int, error) {
	n, err := c.readSmtpResponse(b)
	rs := readState{}
	rs.response = b[0:n]
	rs.err = err
	c.operations = append(c.operations, &rs)
	return n, err
}

func (c *Conn) Ehlo(domain string) error {
	cmd := []byte("EHLO " + domain + "\r\n")
	es := ehloState{}
	_, writeErr := c.getUnderlyingConn().Write(cmd)
	if writeErr != nil {
		es.err = writeErr
	} else {
		buf := make([]byte, 512)
		n, readErr := c.readSmtpResponse(buf)
		es.err = readErr
		es.response = buf[0:n]
	}
	c.operations = append(c.operations, &es)
	return es.err
}

func (c *Conn) SmtpHelp() error {
	cmd := []byte("HELP\r\n")
	hs := helpState{}
	_, writeErr := c.getUnderlyingConn().Write(cmd)
	if writeErr != nil {
		hs.err = writeErr
	} else {
		buf := make([]byte, 512)
		n, readErr := c.readSmtpResponse(buf)
		hs.err = readErr
		hs.response = buf[0:n]
	}
	c.operations = append(c.operations, &hs)
	return hs.err
}

func (c *Conn) readPop3Response(res []byte) (int, error) {
	return c.readUntilRegex(res, pop3EndRegex)
}

func (c *Conn) Pop3Banner(b []byte) (int, error) {
	n, err := c.readPop3Response(b)
	rs := readState{
		response: b[0:n],
		err: err,
	}
	c.operations = append(c.operations, &rs)
	return n, err
}

func (c *Conn) readImapStatusResponse(res []byte) (int, error) {
	return c.readUntilRegex(res, imapStatusEndRegex)
}

func (c *Conn) ImapBanner(b []byte) (int, error) {
	n, err := c.readImapStatusResponse(b)
	rs := readState {
		response: b[0:n],
		err: err,
	}
	c.operations = append(c.operations, &rs)
	return n, err
}

func (c *Conn) SendHeartbleedProbe(b []byte) (int, error) {
	if !c.isTls {
		return 0, fmt.Errorf(
			"Must perform TLS handshake before sending Heartbleed probe to %s",
			c.RemoteAddr().String())
	}
	n, err := c.tlsConn.CheckHeartbleed(b)
	hl := c.tlsConn.HeartbleedLog()
	if err == ztls.HeartbleedError {
		err = nil
	}
	hs := heartbleedState{probe: hl, err: err}
	c.operations = append(c.operations, &hs)
	return n, err
}

func (c *Conn) States() []StateLog {
	states := make([]StateLog, 0, len(c.operations))
	for _, state := range c.operations {
		states = append(states, state.StateLog())
	}
	return states
}
<|endoftext|>


<commit_before>// Copyright (c) 2018 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ipv4net

import (
	"net"
	"sync"

	podmodel "github.com/contiv/vpp/plugins/ksr/model/pod"
)

// MockIPv4Net is a mock for the ipv4net Plugin.
type MockIPv4Net struct {
	sync.Mutex

	podIf            map[podmodel.ID]string
	hostIPs          []net.IP
	nodeIP           *net.IPNet
	hostInterconnect string
	vxlanBVIIfName   string
}

// NewMockIPv4Net is a constructor for MockIPv4Net.
func NewMockIPv4Net() *MockIPv4Net {
	return &MockIPv4Net{podIf: make(map[podmodel.ID]string)}
}

// SetPodIfName allows to create a fake association between a pod and an interface.
func (mn *MockIPv4Net) SetPodIfName(pod podmodel.ID, ifName string) {
	mn.podIf[pod] = ifName
}

// SetNodeIP allows to set what tests will assume the node IP is.
func (mn *MockIPv4Net) SetNodeIP(nodeIP *net.IPNet) {
	mn.Lock()
	defer mn.Unlock()

	mn.nodeIP = nodeIP
}

// SetHostInterconnectIfName allows to set what tests will assume the name of the host-interconnect
// interface is.
func (mn *MockIPv4Net) SetHostInterconnectIfName(ifName string) {
	mn.hostInterconnect = ifName
}

// SetVxlanBVIIfName allows to set what tests will assume the name of the VXLAN BVI interface is.
func (mn *MockIPv4Net) SetVxlanBVIIfName(ifName string) {
	mn.vxlanBVIIfName = ifName
}

// SetHostIPs sets IP addresses of this node present in the host network namespace (Linux).
func (mn *MockIPv4Net) SetHostIPs(ips []net.IP) {
	mn.hostIPs = ips
}

// GetIfName returns pod's interface name as set previously using SetPodIfName.
func (mn *MockIPv4Net) GetIfName(podNamespace string, podName string) (name string, exists bool) {
	name, exists = mn.podIf[podmodel.ID{Name: podName, Namespace: podNamespace}]
	return name, exists
}

// GetPodByIf looks up podName and podNamespace that is associated with logical interface name.
func (mn *MockIPv4Net) GetPodByIf(ifname string) (podNamespace string, podName string, exists bool) {
	for podID, name := range mn.podIf {
		if name == ifname {
			return podID.Namespace, podID.Name, true
		}
	}
	return "", "", false
}

// GetNodeIP returns the IP+network address of this node.
func (mn *MockIPv4Net) GetNodeIP() (net.IP, *net.IPNet) {
	mn.Lock()
	defer mn.Unlock()

	if mn.nodeIP == nil {
		return net.IP{}, nil
	}

	return mn.nodeIP.IP, &net.IPNet{
		IP:   mn.nodeIP.IP.Mask(mn.nodeIP.Mask),
		Mask: mn.nodeIP.Mask,
	}
}

// GetHostInterconnectIfName returns the name of the TAP/AF_PACKET interface
// interconnecting VPP with the host stack.
func (mn *MockIPv4Net) GetHostInterconnectIfName() string {
	return mn.hostInterconnect
}

// GetVxlanBVIIfName returns the name of an BVI interface facing towards VXLAN tunnels to other hosts.
// Returns an empty string if VXLAN is not used (in L2 interconnect mode).
func (mn *MockIPv4Net) GetVxlanBVIIfName() string {
	return mn.vxlanBVIIfName
}

// GetHostIPs returns all IP addresses of this node present in the host network namespace (Linux).
func (mn *MockIPv4Net) GetHostIPs() []net.IP {
	return mn.hostIPs
}
<commit_msg>Fix ipv4net mock<commit_after>// Copyright (c) 2018 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ipv4net

import (
	"net"
	"sync"

	podmodel "github.com/contiv/vpp/plugins/ksr/model/pod"
)

// MockIPv4Net is a mock for the ipv4net Plugin.
type MockIPv4Net struct {
	sync.Mutex

	podIf             map[podmodel.ID]string
	hostIPs           []net.IP
	nodeIP            *net.IPNet
	podSubnetThisNode *net.IPNet
	hostInterconnect  string
	vxlanBVIIfName    string
}

// NewMockIPv4Net is a constructor for MockIPv4Net.
func NewMockIPv4Net() *MockIPv4Net {
	return &MockIPv4Net{podIf: make(map[podmodel.ID]string)}
}

// SetPodIfName allows to create a fake association between a pod and an interface.
func (mn *MockIPv4Net) SetPodIfName(pod podmodel.ID, ifName string) {
	mn.podIf[pod] = ifName
}

// SetNodeIP allows to set what tests will assume the node IP is.
func (mn *MockIPv4Net) SetNodeIP(nodeIP *net.IPNet) {
	mn.Lock()
	defer mn.Unlock()

	mn.nodeIP = nodeIP
}

// SetHostInterconnectIfName allows to set what tests will assume the name of the host-interconnect
// interface is.
func (mn *MockIPv4Net) SetHostInterconnectIfName(ifName string) {
	mn.hostInterconnect = ifName
}

// SetVxlanBVIIfName allows to set what tests will assume the name of the VXLAN BVI interface is.
func (mn *MockIPv4Net) SetVxlanBVIIfName(ifName string) {
	mn.vxlanBVIIfName = ifName
}

// SetHostIPs sets IP addresses of this node present in the host network namespace (Linux).
func (mn *MockIPv4Net) SetHostIPs(ips []net.IP) {
	mn.hostIPs = ips
}

// GetIfName returns pod's interface name as set previously using SetPodIfName.
func (mn *MockIPv4Net) GetIfName(podNamespace string, podName string) (name string, exists bool) {
	name, exists = mn.podIf[podmodel.ID{Name: podName, Namespace: podNamespace}]
	return name, exists
}

// GetPodByIf looks up podName and podNamespace that is associated with logical interface name.
func (mn *MockIPv4Net) GetPodByIf(ifname string) (podNamespace string, podName string, exists bool) {
	for podID, name := range mn.podIf {
		if name == ifname {
			return podID.Namespace, podID.Name, true
		}
	}
	return "", "", false
}

// GetNodeIP returns the IP+network address of this node.
func (mn *MockIPv4Net) GetNodeIP() (net.IP, *net.IPNet) {
	mn.Lock()
	defer mn.Unlock()

	if mn.nodeIP == nil {
		return net.IP{}, nil
	}

	return mn.nodeIP.IP, &net.IPNet{
		IP:   mn.nodeIP.IP.Mask(mn.nodeIP.Mask),
		Mask: mn.nodeIP.Mask,
	}
}

// GetHostInterconnectIfName returns the name of the TAP/AF_PACKET interface
// interconnecting VPP with the host stack.
func (mn *MockIPv4Net) GetHostInterconnectIfName() string {
	return mn.hostInterconnect
}

// GetVxlanBVIIfName returns the name of an BVI interface facing towards VXLAN tunnels to other hosts.
// Returns an empty string if VXLAN is not used (in L2 interconnect mode).
func (mn *MockIPv4Net) GetVxlanBVIIfName() string {
	return mn.vxlanBVIIfName
}

// GetHostIPs returns all IP addresses of this node present in the host network namespace (Linux).
func (mn *MockIPv4Net) GetHostIPs() []net.IP {
	return mn.hostIPs
}

// GetPodSubnetThisNode returns POD network for the current node
// (given by nodeID allocated for this node).
func (mn *MockIPv4Net) GetPodSubnetThisNode() *net.IPNet {
	return mn.podSubnetThisNode
}
<|endoftext|>


<commit_before>package zcl

import "fmt"

// Pos represents a single position in a source file, by addressing the
// start byte of a unicode character encoded in UTF-8.
//
// Pos is generally used only in the context of a Range, which then defines
// which source file the position is within.
type Pos struct {
	// Line is the source code line where this position points. Lines are
	// counted starting at 1 and incremented for each newline character
	// encountered.
	Line int

	// Column is the source code column where this position points, in
	// unicode characters, with counting starting at 1.
	//
	// Column counts characters as they appear visually, so for example a
	// latin letter with a combining diacritic mark counts as one character.
	// This is intended for rendering visual markers against source code in
	// contexts where these diacritics would be rendered in a single character
	// cell. Technically speaking, Column is counting grapheme clusters as
	// used in unicode normalization.
	Column int

	// Byte is the byte offset into the file where the indicated character
	// begins. This is a zero-based offset to the first byte of the first
	// UTF-8 codepoint sequence in the character, and thus gives a position
	// that can be resolved _without_ awareness of Unicode characters.
	Byte int
}

// Range represents a span of characters between two positions in a source
// file.
//
// This struct is usually used by value in types that represent AST nodes,
// but by pointer in types that refer to the positions of other objects,
// such as in diagnostics.
type Range struct {
	// Filename is the name of the file into which this range's positions
	// point.
	Filename string

	// Start and End represent the bounds of this range. Start is inclusive
	// and End is exclusive.
	Start, End Pos
}

// RangeBetween returns a new range that spans from the beginning of the
// start range to the end of the end range.
//
// The result is meaningless if the two ranges do not belong to the same
// source file or if the end range appears before the start range.
func RangeBetween(start, end Range) Range {
	return Range{
		Filename: start.Filename,
		Start:    start.Start,
		End:      end.End,
	}
}

// ContainsOffset returns true if and only if the given byte offset is within
// the receiving Range.
func (r *Range) ContainsOffset(offset int) bool {
	return offset >= r.Start.Byte && offset < r.End.Byte
}

// Ptr returns a pointer to a copy of the receiver. This is a convenience when
// ranges in places where pointers are required, such as in Diagnostic, but
// the range in question is returned from a method. Go would otherwise not
// allow one to take the address of a function call.
func (r Range) Ptr() *Range {
	return &r
}

// String returns a compact string representation of the receiver.
// Callers should generally prefer to present a range more visually,
// e.g. via markers directly on the relevant portion of source code.
func (r *Range) String() string {
	if r.Start.Line == r.End.Line {
		return fmt.Sprintf(
			"%s:%d,%d-%d",
			r.Filename,
			r.Start.Line, r.Start.Column,
			r.End.Column,
		)
	} else {
		return fmt.Sprintf(
			"%s:%d,%d-%d,%d",
			r.Filename,
			r.Start.Line, r.Start.Column,
			r.End.Line, r.End.Column,
		)
	}
}
<commit_msg>zcl: Range.ContainsOffset should not have pointer receiver<commit_after>package zcl

import "fmt"

// Pos represents a single position in a source file, by addressing the
// start byte of a unicode character encoded in UTF-8.
//
// Pos is generally used only in the context of a Range, which then defines
// which source file the position is within.
type Pos struct {
	// Line is the source code line where this position points. Lines are
	// counted starting at 1 and incremented for each newline character
	// encountered.
	Line int

	// Column is the source code column where this position points, in
	// unicode characters, with counting starting at 1.
	//
	// Column counts characters as they appear visually, so for example a
	// latin letter with a combining diacritic mark counts as one character.
	// This is intended for rendering visual markers against source code in
	// contexts where these diacritics would be rendered in a single character
	// cell. Technically speaking, Column is counting grapheme clusters as
	// used in unicode normalization.
	Column int

	// Byte is the byte offset into the file where the indicated character
	// begins. This is a zero-based offset to the first byte of the first
	// UTF-8 codepoint sequence in the character, and thus gives a position
	// that can be resolved _without_ awareness of Unicode characters.
	Byte int
}

// Range represents a span of characters between two positions in a source
// file.
//
// This struct is usually used by value in types that represent AST nodes,
// but by pointer in types that refer to the positions of other objects,
// such as in diagnostics.
type Range struct {
	// Filename is the name of the file into which this range's positions
	// point.
	Filename string

	// Start and End represent the bounds of this range. Start is inclusive
	// and End is exclusive.
	Start, End Pos
}

// RangeBetween returns a new range that spans from the beginning of the
// start range to the end of the end range.
//
// The result is meaningless if the two ranges do not belong to the same
// source file or if the end range appears before the start range.
func RangeBetween(start, end Range) Range {
	return Range{
		Filename: start.Filename,
		Start:    start.Start,
		End:      end.End,
	}
}

// ContainsOffset returns true if and only if the given byte offset is within
// the receiving Range.
func (r Range) ContainsOffset(offset int) bool {
	return offset >= r.Start.Byte && offset < r.End.Byte
}

// Ptr returns a pointer to a copy of the receiver. This is a convenience when
// ranges in places where pointers are required, such as in Diagnostic, but
// the range in question is returned from a method. Go would otherwise not
// allow one to take the address of a function call.
func (r Range) Ptr() *Range {
	return &r
}

// String returns a compact string representation of the receiver.
// Callers should generally prefer to present a range more visually,
// e.g. via markers directly on the relevant portion of source code.
func (r *Range) String() string {
	if r.Start.Line == r.End.Line {
		return fmt.Sprintf(
			"%s:%d,%d-%d",
			r.Filename,
			r.Start.Line, r.Start.Column,
			r.End.Column,
		)
	} else {
		return fmt.Sprintf(
			"%s:%d,%d-%d,%d",
			r.Filename,
			r.Start.Line, r.Start.Column,
			r.End.Line, r.End.Column,
		)
	}
}
<|endoftext|>


<commit_before>package vaultutil

import (
	vaultapi "github.com/hashicorp/vault/api"
)

func CreateUpdateSSHKey(c *vaultapi.Client, ssh_path, key_name, key_value string) (*vaultapi.Secret, error) {
	data := map[string]interface{}{"key": key_value}
	secret, err := c.Logical().Write(ssh_path+key_name, data)
	return secret, err
}

func DeleteSSHKey(c *vaultapi.Client, ssh_path, key_name string) (*vaultapi.Secret, error) {
	secret, err := c.Logical().Delete(ssh_path + key_name)
	return secret, err
}

func CreateUpdateSSHRole(c *vaultapi.Client, ssh_path, role_name string, role SSHRoleData) (*vaultapi.Secret, error) {
	data := map[string]interface{}{
		"key":        role.Key,
		"admin_user": role.Admin_User,
		"defaul_user", role.Default_User,
		"cidr_list ":        role.Cidr_List,
		"exclude_cidr_list": role.Exclude_Cidr_List,
		"port":              role.Port,
		"key_type":          role.key_type,
		"key_bits":          role.key_bits,
		"install_script":    role.install_script,
		"allowed_users":     role.allowed_users,
		"key_option_specs":  role.key_option_specs}
	secret, err := c.Logical().Write(ssh_path+key_name, data)
	return secret, err
}

func GetSSHRole(c *vaultapi.Client, ssh_path, role_name string) (*vaultapi.Secret, error) {
	secret, err := c.Logical().Read(ssh_path+key_name, data)
	return secret, err
}
<commit_msg>Fix compilation errors<commit_after>package vaultutil

import (
	vaultapi "github.com/hashicorp/vault/api"
)

func CreateUpdateSSHKey(c *vaultapi.Client, ssh_path, key_name, key_value string) (*vaultapi.Secret, error) {
	data := map[string]interface{}{"key": key_value}
	secret, err := c.Logical().Write(ssh_path+key_name, data)
	return secret, err
}

func DeleteSSHKey(c *vaultapi.Client, ssh_path, key_name string) (*vaultapi.Secret, error) {
	secret, err := c.Logical().Delete(ssh_path + key_name)
	return secret, err
}

func CreateUpdateSSHRole(c *vaultapi.Client, ssh_path, role_name string, role SSHRoleData) (*vaultapi.Secret, error) {
	data := map[string]interface{}{
		"key":               role.Key,
		"admin_user":        role.Admin_User,
		"defaul_user":       role.Default_User,
		"cidr_list ":        role.Cidr_List,
		"exclude_cidr_list": role.Exclude_Cidr_List,
		"port":              role.Port,
		"key_type":          role.key_type,
		"key_bits":          role.key_bits,
		"install_script":    role.install_script,
		"allowed_users":     role.allowed_users,
		"key_option_specs":  role.key_option_specs}
	secret, err := c.Logical().Write(ssh_path+role_name, data)
	return secret, err
}

func GetSSHRole(c *vaultapi.Client, ssh_path, role_name string) (*vaultapi.Secret, error) {
	secret, err := c.Logical().Read(ssh_path + role_name)
	return secret, err
}
<|endoftext|>


<commit_before>package utils

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/request"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudformation"
)

type AwsRequest interface {
	Send() error
}

type RequestHandler interface {
	HandleRequest(req AwsRequest) error
}

type AwsRequestHandler struct {
}

func (r *AwsRequestHandler) HandleRequest(req AwsRequest) error {
	s := 1
	var err error
	for err = req.Send(); err != nil; err = req.Send() {
		if reqerr, ok := err.(awserr.RequestFailure); ok {
			if reqerr.Code() == "RequestLimitExceeded" {
				time.Sleep(time.Duration(s) * time.Second)
				s = s * 2
				continue
			}
		}
		return err
	}
	return nil
}

type Input struct {
	Source struct {
		Name               string `json:"name"`
		AwsAccessKeyId     string `json:"aws_access_key_id"`
		AwsSecretAccessKey string `json:"aws_secret_access_key"`
		Region             string `json:"region"`
	} `json:"source"`
	Version struct {
		LastUpdatedTime string `json:"LastUpdatedTime"`
	} `json:"version"`
	Params struct {
		Template     string   `json:"template"`
		Parameters   string   `json:"parameters"`
		Tags         string   `json:"tags"`
		Capabilities []string `json:"capabilities"`
		Delete       bool     `json:"delete"`
		Wait         bool     `json:"wait"`
	} `json:"params"`
}

func GetInput() Input {
	bytes, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		panic(err)
	}
	input := Input{}
	err = json.Unmarshal(bytes, &input)
	if err != nil {
		panic(err)
	}
	return input
}

type AwsCloudformationSvc interface {
	DescribeStacksRequest(input *cloudformation.DescribeStacksInput) (req *request.Request, output *cloudformation.DescribeStacksOutput)
	CreateStackRequest(input *cloudformation.CreateStackInput) (req *request.Request, output *cloudformation.CreateStackOutput)
	UpdateStackRequest(input *cloudformation.UpdateStackInput) (req *request.Request, output *cloudformation.UpdateStackOutput)
	DescribeStackEventsRequest(input *cloudformation.DescribeStackEventsInput) (req *request.Request, output *cloudformation.DescribeStackEventsOutput)
	DeleteStackRequest(input *cloudformation.DeleteStackInput) (req *request.Request, output *cloudformation.DeleteStackOutput)
}

func GetCloudformationService(input Input) AwsCloudformationSvc {
	creds := credentials.NewStaticCredentials(input.Source.AwsAccessKeyId, input.Source.AwsSecretAccessKey, "")
	awsConfig := aws.NewConfig().WithCredentials(creds).WithRegion(input.Source.Region)
	sess := session.Must(session.NewSession(awsConfig))
	svc := cloudformation.New(sess)
	return svc
}

func GoToBuildDirectory() {
	files, err := ioutil.ReadDir("/tmp/build")
	if err != nil {
		panic(err)
	}

	if len(files) != 1 {
		fmt.Printf("Expected only 1 file in /tmp/build but found %d: %v\n", len(files), files)
		os.Exit(1)
	}

	os.Chdir("/tmp/build/" + files[0].Name())
}

func Logln(message interface{}) {
	Logf("%s\n", message)
}

func Logf(format string, a ...interface{}) {
	fmt.Fprintf(os.Stderr, format, a)
}
<commit_msg>Attempt to fix logging<commit_after>package utils

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/request"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudformation"
)

type AwsRequest interface {
	Send() error
}

type RequestHandler interface {
	HandleRequest(req AwsRequest) error
}

type AwsRequestHandler struct {
}

func (r *AwsRequestHandler) HandleRequest(req AwsRequest) error {
	s := 1
	var err error
	for err = req.Send(); err != nil; err = req.Send() {
		if reqerr, ok := err.(awserr.RequestFailure); ok {
			if reqerr.Code() == "RequestLimitExceeded" {
				time.Sleep(time.Duration(s) * time.Second)
				s = s * 2
				continue
			}
		}
		return err
	}
	return nil
}

type Input struct {
	Source struct {
		Name               string `json:"name"`
		AwsAccessKeyId     string `json:"aws_access_key_id"`
		AwsSecretAccessKey string `json:"aws_secret_access_key"`
		Region             string `json:"region"`
	} `json:"source"`
	Version struct {
		LastUpdatedTime string `json:"LastUpdatedTime"`
	} `json:"version"`
	Params struct {
		Template     string   `json:"template"`
		Parameters   string   `json:"parameters"`
		Tags         string   `json:"tags"`
		Capabilities []string `json:"capabilities"`
		Delete       bool     `json:"delete"`
		Wait         bool     `json:"wait"`
	} `json:"params"`
}

func GetInput() Input {
	bytes, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		panic(err)
	}
	input := Input{}
	err = json.Unmarshal(bytes, &input)
	if err != nil {
		panic(err)
	}
	return input
}

type AwsCloudformationSvc interface {
	DescribeStacksRequest(input *cloudformation.DescribeStacksInput) (req *request.Request, output *cloudformation.DescribeStacksOutput)
	CreateStackRequest(input *cloudformation.CreateStackInput) (req *request.Request, output *cloudformation.CreateStackOutput)
	UpdateStackRequest(input *cloudformation.UpdateStackInput) (req *request.Request, output *cloudformation.UpdateStackOutput)
	DescribeStackEventsRequest(input *cloudformation.DescribeStackEventsInput) (req *request.Request, output *cloudformation.DescribeStackEventsOutput)
	DeleteStackRequest(input *cloudformation.DeleteStackInput) (req *request.Request, output *cloudformation.DeleteStackOutput)
}

func GetCloudformationService(input Input) AwsCloudformationSvc {
	creds := credentials.NewStaticCredentials(input.Source.AwsAccessKeyId, input.Source.AwsSecretAccessKey, "")
	awsConfig := aws.NewConfig().WithCredentials(creds).WithRegion(input.Source.Region)
	sess := session.Must(session.NewSession(awsConfig))
	svc := cloudformation.New(sess)
	return svc
}

func GoToBuildDirectory() {
	files, err := ioutil.ReadDir("/tmp/build")
	if err != nil {
		panic(err)
	}

	if len(files) != 1 {
		fmt.Printf("Expected only 1 file in /tmp/build but found %d: %v\n", len(files), files)
		os.Exit(1)
	}

	os.Chdir("/tmp/build/" + files[0].Name())
}

func Logln(message ...interface{}) {
	Logf("%s\n", message...)
}

func Logf(format string, a ...interface{}) {
	fmt.Fprintf(os.Stderr, format, a)
}
<|endoftext|>


<commit_before>package utils

import (
	"errors"
	"fmt"
	"log"
	"math"
	"sort"
	"strconv"
	"strings"

	"github.com/vmware/govmomi/vim25/types"
)

var stdlog, errlog *log.Logger

// Min : get the minimum of values
func Min(n ...int64) int64 {
	var min int64 = -1
	for _, i := range n {
		if i >= 0 {
			if min == -1 {
				min = i
			} else {
				if i < min {
					min = i
				}
			}
		}
	}
	return min
}

// Max : get the maximum of the values
func Max(n ...int64) int64 {
	var max int64 = -1
	for _, i := range n {
		if i >= 0 {
			if max == -1 {
				max = i
			} else {
				if i > max {
					max = i
				}
			}
		}
	}
	return max
}

// Sum : Sum the values
func Sum(n ...int64) int64 {
	var total int64
	for _, i := range n {
		if i > 0 {
			total += i
		}
	}
	return total
}

// Average : average the values
func Average(n ...int64) int64 {
	var total int64
	var count int64
	for _, i := range n {
		if i >= 0 {
			count++
			total += i
		}
	}
	favg := float64(total) / float64(count)
	return int64(math.Floor(favg + .5))
}

// MapObjRefs fills in object references into a map to another object reference
func MapObjRefs(property string, sourceVal *types.AnyType, dest map[string]*[]types.ManagedObjectReference, index string) error {
	mors, ok := (*sourceVal).(types.ArrayOfManagedObjectReference)
	if ok {
		if len(mors.ManagedObjectReference) > 0 {
			dest[index] = &mors.ManagedObjectReference
			return nil
		}
		return errors.New("Property " + property + " of " + index + " didn't contain any object references")
	}
	return errors.New("Property " + property + " of " + index + " was not a ManagedObjectReferences, it was " + fmt.Sprintf("%T", sourceVal))
}

// MapObjRef fills in object reference into a map to another object reference
func MapObjRef(property string, sourceVal *types.AnyType, dest map[string]*string, index string) error {
	mor, ok := (*sourceVal).(types.ManagedObjectReference)
	if ok {
		dest[index] = &mor.Value
		return nil
	}
	return errors.New("Property " + property + " of " + index + " was not a ManagedObjectReference, it was " + fmt.Sprintf("%T", sourceVal))
}

// MapObjInt32 fills in an int32 into a map to another object reference
func MapObjInt32(property string, sourceVal *types.AnyType, dest map[string]*int32, index string) error {
	val, ok := (*sourceVal).(int32)
	if ok {
		dest[index] = &val
		return nil
	}
	return errors.New("Property " + property + " of " + index + " was not an int32, it was " + fmt.Sprintf("%T", sourceVal))
}

// StringMaptoString converts a string map to csv or get the first value
func StringMaptoString(value []string, separator string, noarray bool) string {
	if len(value) == 0 {
		return ""
	}
	if noarray {
		return value[0]
	}
	return strings.Join(value, separator)
}

// IntMaptoString converts a int map to csv or get the first value
func IntMaptoString(value []int, separator string, noarray bool) string {
	if len(value) == 0 {
		return ""
	}
	if noarray {
		return strconv.Itoa(value[0])
	}
	var strval []string
	for _, i := range value {
		strval = append(strval, strconv.Itoa(i))
	}
	return strings.Join(strval, separator)
}

// Int32MaptoString converts a int32 map to csv or get the first value
func Int32MaptoString(value []int32, separator string, noarray bool) string {
	if len(value) == 0 {
		return ""
	}
	if noarray {
		return strconv.FormatInt(int64(value[0]), 10)
	}
	var strval []string
	for _, i := range value {
		strval = append(strval, strconv.FormatInt(int64(i), 10))
	}
	return strings.Join(strval, separator)
}

// Int64MaptoString converts a int64 map to csv or get the first value
func Int64MaptoString(value []int64, separator string, noarray bool) string {
	if len(value) == 0 {
		return ""
	}
	if noarray {
		return strconv.FormatInt(value[0], 10)
	}
	var strval []string
	for _, i := range value {
		strval = append(strval, strconv.FormatInt(i, 10))
	}
	return strings.Join(strval, separator)
}

// ValToString : try to convert interface to string. Separated by separator if slice
func ValToString(value interface{}, separator string, noarray bool) string {
	switch v := value.(type) {
	case string:
		return v
	case []string:
		return StringMaptoString(v, separator, noarray)
	case int:
		return strconv.Itoa(v)
	case []int:
		return IntMaptoString(v, separator, noarray)
	case int32:
		return strconv.FormatInt(int64(v), 10)
	case []int32:
		return Int32MaptoString(v, separator, noarray)
	case int64:
		return strconv.FormatInt(v, 10)
	case []int64:
		return Int64MaptoString(v, separator, noarray)
	default:
		return ""
	}
}

// Join map[int]string into a string
func Join(values map[int]string, separator string) string {
	var keys []int
	for k := range values {
		keys = append(keys, k)
	}
	sort.Ints(keys)
	// create a map with the key parts in order
	var tmp []string
	for _, k := range keys {
		tmp = append(tmp, values[k])
	}
	return strings.Join(tmp, separator)
}

// MustAtoi converts a string to integer and return 0 i case of error
func MustAtoi(value string) int {
	i, err := strconv.Atoi(value)
	if err != nil {
		i = 0
	}
	return i
}

// FindHostAndCluster Find cluster for managed entity
func FindHostAndCluster(entity *string,
	vmToHost map[string]*string,
	morToParent map[string]*string,
	morToName map[string]*string) (*string, *string, error) {
	// get host
	var host *string
	var hostname *string
	var cluster *string
	if strings.HasPrefix(*entity, "vm-") {
		if vmhost, ok := vmToHost[*entity]; ok {
			host = vmhost
			if vmhostname, ok := morToName[*host]; ok {
				hostname = vmhostname
			}
		}
	} else if strings.HasPrefix(*entity, "host-") {
		host = entity
	}
	// check if a host was found
	if host == nil {
		return nil, nil, errors.New("No host found for " + *entity)
	}
	if parmor, ok := morToParent[*host]; ok {
		if strings.HasPrefix(*parmor, "domain-c") {
			cluster = morToName[*parmor]
		} else if !strings.HasPRefix(*parmor, "domain-s") {
			// ComputeRessource parent denotes a standalong host
			// Any other is weird
			return hostname, nil, errors.New("No suitable parent for host " + *host + " to determine cluster (" + *parmor  + ")")
		}
	}
	return hostname, cluster, nil
}

// ConvertToKV converts a map[string]string to a csv with k=v pairs
func ConvertToKV(values map[string]string) string {
	var tmp []string
	for key, val := range values {
		if len(val) == 0 {
			continue
		}
		tmp = append(tmp, fmt.Sprintf("%s=%s", key, val))
	}
	return strings.Join(tmp, ",")
}

// Reverse reverses the values of an array
func Reverse(array []string) {
	for i := len(array)/2 - 1; i >= 0; i-- {
		opp := len(array) - 1 - i
		array[i], array[opp] = array[opp], array[i]
	}
}

// CleanStringMap removes unkonw keys from a map of string
func CleanStringMap(m map[string]*string, refs []string) {
	for mor := range m {
		found := false
		for _, refmor := range refs {
			if refmor == mor {
				found = true
				break
			}
		}
		if !found {
			stdlog.Println("removing mor from string map: " + mor)
			delete(m, mor)
		}
	}
}

// CleanMorefsMap removes unkonw keys from a map of manged object references
func CleanMorefsMap(m map[string]*[]types.ManagedObjectReference, refs []string) {
	for mor := range m {
		found := false
		for _, refmor := range refs {
			if refmor == mor {
				found = true
				break
			}
		}
		if !found {
			stdlog.Println("removing mor from moref map: " + mor)
			delete(m, mor)
		}
	}
}

// CleanInt32Map removes unkonw keys from a map of int32
func CleanInt32Map(m map[string]*int32, refs []string) {
	for mor := range m {
		found := false
		for _, refmor := range refs {
			if refmor == mor {
				found = true
				break
			}
		}
		if !found {
			stdlog.Println("removing mor from int map: " + mor)
			delete(m, mor)
		}
	}
}

// CleanTagsMap removes unkonw keys from a map of tag
func CleanTagsMap(m map[string]*[]types.Tag, refs []string) {
	for mor := range m {
		found := false
		for _, refmor := range refs {
			if refmor == mor {
				found = true
				break
			}
		}
		if !found {
			stdlog.Println("removing mor from tags map: " + mor)
			delete(m, mor)
		}
	}
}

// CleanDiskInfosMap removes unkonw keys from a map of tag
func CleanDiskInfosMap(m map[string]*[]types.GuestDiskInfo, refs []string) {
	for mor := range m {
		found := false
		for _, refmor := range refs {
			if refmor == mor {
				found = true
				break
			}
		}
		if !found {
			stdlog.Println("removing mor from disk info map: " + mor)
			delete(m, mor)
		}
	}
}
<commit_msg>typo in HasPrefix<commit_after>package utils

import (
	"errors"
	"fmt"
	"log"
	"math"
	"sort"
	"strconv"
	"strings"

	"github.com/vmware/govmomi/vim25/types"
)

var stdlog, errlog *log.Logger

// Min : get the minimum of values
func Min(n ...int64) int64 {
	var min int64 = -1
	for _, i := range n {
		if i >= 0 {
			if min == -1 {
				min = i
			} else {
				if i < min {
					min = i
				}
			}
		}
	}
	return min
}

// Max : get the maximum of the values
func Max(n ...int64) int64 {
	var max int64 = -1
	for _, i := range n {
		if i >= 0 {
			if max == -1 {
				max = i
			} else {
				if i > max {
					max = i
				}
			}
		}
	}
	return max
}

// Sum : Sum the values
func Sum(n ...int64) int64 {
	var total int64
	for _, i := range n {
		if i > 0 {
			total += i
		}
	}
	return total
}

// Average : average the values
func Average(n ...int64) int64 {
	var total int64
	var count int64
	for _, i := range n {
		if i >= 0 {
			count++
			total += i
		}
	}
	favg := float64(total) / float64(count)
	return int64(math.Floor(favg + .5))
}

// MapObjRefs fills in object references into a map to another object reference
func MapObjRefs(property string, sourceVal *types.AnyType, dest map[string]*[]types.ManagedObjectReference, index string) error {
	mors, ok := (*sourceVal).(types.ArrayOfManagedObjectReference)
	if ok {
		if len(mors.ManagedObjectReference) > 0 {
			dest[index] = &mors.ManagedObjectReference
			return nil
		}
		return errors.New("Property " + property + " of " + index + " didn't contain any object references")
	}
	return errors.New("Property " + property + " of " + index + " was not a ManagedObjectReferences, it was " + fmt.Sprintf("%T", sourceVal))
}

// MapObjRef fills in object reference into a map to another object reference
func MapObjRef(property string, sourceVal *types.AnyType, dest map[string]*string, index string) error {
	mor, ok := (*sourceVal).(types.ManagedObjectReference)
	if ok {
		dest[index] = &mor.Value
		return nil
	}
	return errors.New("Property " + property + " of " + index + " was not a ManagedObjectReference, it was " + fmt.Sprintf("%T", sourceVal))
}

// MapObjInt32 fills in an int32 into a map to another object reference
func MapObjInt32(property string, sourceVal *types.AnyType, dest map[string]*int32, index string) error {
	val, ok := (*sourceVal).(int32)
	if ok {
		dest[index] = &val
		return nil
	}
	return errors.New("Property " + property + " of " + index + " was not an int32, it was " + fmt.Sprintf("%T", sourceVal))
}

// StringMaptoString converts a string map to csv or get the first value
func StringMaptoString(value []string, separator string, noarray bool) string {
	if len(value) == 0 {
		return ""
	}
	if noarray {
		return value[0]
	}
	return strings.Join(value, separator)
}

// IntMaptoString converts a int map to csv or get the first value
func IntMaptoString(value []int, separator string, noarray bool) string {
	if len(value) == 0 {
		return ""
	}
	if noarray {
		return strconv.Itoa(value[0])
	}
	var strval []string
	for _, i := range value {
		strval = append(strval, strconv.Itoa(i))
	}
	return strings.Join(strval, separator)
}

// Int32MaptoString converts a int32 map to csv or get the first value
func Int32MaptoString(value []int32, separator string, noarray bool) string {
	if len(value) == 0 {
		return ""
	}
	if noarray {
		return strconv.FormatInt(int64(value[0]), 10)
	}
	var strval []string
	for _, i := range value {
		strval = append(strval, strconv.FormatInt(int64(i), 10))
	}
	return strings.Join(strval, separator)
}

// Int64MaptoString converts a int64 map to csv or get the first value
func Int64MaptoString(value []int64, separator string, noarray bool) string {
	if len(value) == 0 {
		return ""
	}
	if noarray {
		return strconv.FormatInt(value[0], 10)
	}
	var strval []string
	for _, i := range value {
		strval = append(strval, strconv.FormatInt(i, 10))
	}
	return strings.Join(strval, separator)
}

// ValToString : try to convert interface to string. Separated by separator if slice
func ValToString(value interface{}, separator string, noarray bool) string {
	switch v := value.(type) {
	case string:
		return v
	case []string:
		return StringMaptoString(v, separator, noarray)
	case int:
		return strconv.Itoa(v)
	case []int:
		return IntMaptoString(v, separator, noarray)
	case int32:
		return strconv.FormatInt(int64(v), 10)
	case []int32:
		return Int32MaptoString(v, separator, noarray)
	case int64:
		return strconv.FormatInt(v, 10)
	case []int64:
		return Int64MaptoString(v, separator, noarray)
	default:
		return ""
	}
}

// Join map[int]string into a string
func Join(values map[int]string, separator string) string {
	var keys []int
	for k := range values {
		keys = append(keys, k)
	}
	sort.Ints(keys)
	// create a map with the key parts in order
	var tmp []string
	for _, k := range keys {
		tmp = append(tmp, values[k])
	}
	return strings.Join(tmp, separator)
}

// MustAtoi converts a string to integer and return 0 i case of error
func MustAtoi(value string) int {
	i, err := strconv.Atoi(value)
	if err != nil {
		i = 0
	}
	return i
}

// FindHostAndCluster Find cluster for managed entity
func FindHostAndCluster(entity *string,
	vmToHost map[string]*string,
	morToParent map[string]*string,
	morToName map[string]*string) (*string, *string, error) {
	// get host
	var host *string
	var hostname *string
	var cluster *string
	if strings.HasPrefix(*entity, "vm-") {
		if vmhost, ok := vmToHost[*entity]; ok {
			host = vmhost
			if vmhostname, ok := morToName[*host]; ok {
				hostname = vmhostname
			}
		}
	} else if strings.HasPrefix(*entity, "host-") {
		host = entity
	}
	// check if a host was found
	if host == nil {
		return nil, nil, errors.New("No host found for " + *entity)
	}
	if parmor, ok := morToParent[*host]; ok {
		if strings.HasPrefix(*parmor, "domain-c") {
			cluster = morToName[*parmor]
		} else if !strings.HasPrefix(*parmor, "domain-s") {
			// ComputeRessource parent denotes a standalong host
			// Any other is weird
			return hostname, nil, errors.New("No suitable parent for host " + *host + " to determine cluster (" + *parmor + ")")
		}
	}
	return hostname, cluster, nil
}

// ConvertToKV converts a map[string]string to a csv with k=v pairs
func ConvertToKV(values map[string]string) string {
	var tmp []string
	for key, val := range values {
		if len(val) == 0 {
			continue
		}
		tmp = append(tmp, fmt.Sprintf("%s=%s", key, val))
	}
	return strings.Join(tmp, ",")
}

// Reverse reverses the values of an array
func Reverse(array []string) {
	for i := len(array)/2 - 1; i >= 0; i-- {
		opp := len(array) - 1 - i
		array[i], array[opp] = array[opp], array[i]
	}
}

// CleanStringMap removes unkonw keys from a map of string
func CleanStringMap(m map[string]*string, refs []string) {
	for mor := range m {
		found := false
		for _, refmor := range refs {
			if refmor == mor {
				found = true
				break
			}
		}
		if !found {
			stdlog.Println("removing mor from string map: " + mor)
			delete(m, mor)
		}
	}
}

// CleanMorefsMap removes unkonw keys from a map of manged object references
func CleanMorefsMap(m map[string]*[]types.ManagedObjectReference, refs []string) {
	for mor := range m {
		found := false
		for _, refmor := range refs {
			if refmor == mor {
				found = true
				break
			}
		}
		if !found {
			stdlog.Println("removing mor from moref map: " + mor)
			delete(m, mor)
		}
	}
}

// CleanInt32Map removes unkonw keys from a map of int32
func CleanInt32Map(m map[string]*int32, refs []string) {
	for mor := range m {
		found := false
		for _, refmor := range refs {
			if refmor == mor {
				found = true
				break
			}
		}
		if !found {
			stdlog.Println("removing mor from int map: " + mor)
			delete(m, mor)
		}
	}
}

// CleanTagsMap removes unkonw keys from a map of tag
func CleanTagsMap(m map[string]*[]types.Tag, refs []string) {
	for mor := range m {
		found := false
		for _, refmor := range refs {
			if refmor == mor {
				found = true
				break
			}
		}
		if !found {
			stdlog.Println("removing mor from tags map: " + mor)
			delete(m, mor)
		}
	}
}

// CleanDiskInfosMap removes unkonw keys from a map of tag
func CleanDiskInfosMap(m map[string]*[]types.GuestDiskInfo, refs []string) {
	for mor := range m {
		found := false
		for _, refmor := range refs {
			if refmor == mor {
				found = true
				break
			}
		}
		if !found {
			stdlog.Println("removing mor from disk info map: " + mor)
			delete(m, mor)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"math/rand"
	"time"

	"github.com/bmizerany/perks/quantile"
	"github.com/garyburd/redigo/redis"
)

// Simple benchmark test. Generate random keys/vals and insert into the DB. For some % of these, wait for some
// amount of time and then query for them.

const (
	minChar = ' '
	maxChar = '~'
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

func randStr(n int) string {
	r := make([]rune, n)
	for i := range r {
		r[i] = rune(rand.Intn(maxChar-minChar+1) + minChar)
	}
	return string(r)
}

var randomValues [][]byte

func init() {
	for i := 0; i < 1000; i++ {
		randomValues = append(randomValues, []byte(randStr(rand.Intn(1000)+500)))
	}
}

func makeRequests(pool *redis.Pool, delay time.Duration, stats chan<- float64) {
	conn := pool.Get()
	defer conn.Close()

	for i := 0; ; i = (i + 1) % len(randomValues) {
		key := randStr(10)
		val := randomValues[i]
		start := time.Now()
		_, err := conn.Do("SET", key, val)
		stats <- time.Since(start).Seconds() * 1000
		if err != nil {
			log.Fatal(err)
		}
		//result, err := redis.String(conn.Do("GET", key))
		//if err != nil {
		//log.Fatal(err)
		//}
		//if result != val {
		//log.Fatal("result mismatch")
		//}
		time.Sleep(delay)
	}
}

func main() {
	pool := &redis.Pool{
		MaxIdle:     0,
		MaxActive:   0, // No limit
		IdleTimeout: time.Second,
		Dial:        func() (redis.Conn, error) { return redis.Dial("tcp", "localhost:5533") },
	}

	const P = 4
	const targetQPS = 3000
	delay := time.Second / time.Duration(float64(targetQPS)/float64(P))

	stats := make(chan float64)
	for i := 0; i < P; i++ {
		go func() {
			makeRequests(pool, delay, stats)
		}()
	}
	collectStats(stats)
}

func collectStats(c <-chan float64) {
	tick := time.Tick(30 * time.Second)
	stats := NewStats()
	for {
		select {
		case <-tick:
			fmt.Println(stats)
			stats = NewStats()
		case f := <-c:
			stats.Add(f)
		}
	}
}

type Stats struct {
	start     time.Time
	max       float64
	total     float64
	samples   float64
	quantiles *quantile.Stream
}

func NewStats() *Stats {
	return &Stats{
		start:     time.Now(),
		quantiles: quantile.NewTargeted(0.5, 0.9, 0.99, 0.999),
	}
}

func (s *Stats) Add(f float64) {
	if f > s.max {
		s.max = f
	}
	s.total += f
	s.samples++
	s.quantiles.Insert(f)
}

func (s *Stats) String() string {
	return fmt.Sprintf("%.0f samples; %.1f qps; max = %.2fms; mean = %.2fms; median = %.2fms; 0.90pct = %.2fms; 0.99pct = %.2fms; 0.999pct = %.2fms",
		s.samples, s.samples/time.Since(s.start).Seconds(), s.max, s.total/s.samples,
		s.quantiles.Query(0.5), s.quantiles.Query(0.9), s.quantiles.Query(0.99), s.quantiles.Query(0.999))
}
<commit_msg>Change benchmark to report exact counts above certain thresholds<commit_after>package main

import (
	"fmt"
	"log"
	"math/rand"
	"time"

	"github.com/garyburd/redigo/redis"
)

// Simple benchmark test. Generate random keys/vals and insert into the DB. For some % of these, wait for some
// amount of time and then query for them.

const (
	minChar = ' '
	maxChar = '~'
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

func randStr(n int) string {
	r := make([]rune, n)
	for i := range r {
		r[i] = rune(rand.Intn(maxChar-minChar+1) + minChar)
	}
	return string(r)
}

var randomValues [][]byte

func init() {
	for i := 0; i < 1000; i++ {
		randomValues = append(randomValues, []byte(randStr(rand.Intn(1000)+500)))
	}
}

func makeRequests(pool *redis.Pool, delay time.Duration, stats chan<- float64) {
	conn := pool.Get()
	defer conn.Close()

	for i := 0; ; i = (i + 1) % len(randomValues) {
		key := randStr(10)
		val := randomValues[i]
		start := time.Now()
		_, err := conn.Do("SET", key, val)
		stats <- time.Since(start).Seconds() * 1000
		if err != nil {
			log.Fatal(err)
		}
		//result, err := redis.String(conn.Do("GET", key))
		//if err != nil {
		//log.Fatal(err)
		//}
		//if result != val {
		//log.Fatal("result mismatch")
		//}
		time.Sleep(delay)
	}
}

func main() {
	pool := &redis.Pool{
		MaxIdle:     0,
		MaxActive:   0, // No limit
		IdleTimeout: time.Second,
		Dial:        func() (redis.Conn, error) { return redis.Dial("tcp", "localhost:5533") },
	}

	const P = 4
	const targetQPS = 3000
	delay := time.Second / time.Duration(float64(targetQPS)/float64(P))

	stats := make(chan float64)
	for i := 0; i < P; i++ {
		go func() {
			makeRequests(pool, delay, stats)
		}()
	}
	collectStats(stats)
}

func collectStats(c <-chan float64) {
	tick := time.Tick(30 * time.Second)
	stats := NewStats()
	for {
		select {
		case <-tick:
			fmt.Println(stats)
			stats = NewStats()
		case f := <-c:
			stats.Add(f)
		}
	}
}

var thresholdsMS = []float64{1, 10, 50}

type Stats struct {
	start       time.Time
	max         float64
	total       float64
	samples     float64
	aboveThresh []float64
}

func NewStats() *Stats {
	return &Stats{
		start:       time.Now(),
		aboveThresh: make([]float64, len(thresholdsMS)),
	}
}

func (s *Stats) Add(f float64) {
	if f > s.max {
		s.max = f
	}
	s.total += f
	s.samples++
	for i, thresh := range thresholdsMS {
		if f > thresh {
			s.aboveThresh[i]++
		}
	}
}

func (s *Stats) String() string {
	return fmt.Sprintf("%.0f samples; %.1f qps; max = %.2fms; mean = %.2fms; >%.1fms = %.2f%%; >%.1fms = %.2f%%; >%.1fms = %.2f%%",
		s.samples, s.samples/time.Since(s.start).Seconds(), s.max, s.total/s.samples,
		thresholdsMS[0], (s.aboveThresh[0]/s.total)*100,
		thresholdsMS[1], (s.aboveThresh[1]/s.total)*100,
		thresholdsMS[2], (s.aboveThresh[2]/s.total)*100,
	)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/oxtoacart/framed"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"runtime"
	"runtime/pprof"
	"sync"
)

const (
	serverAddr = "127.0.0.1:10081"
)

var (
	mode          = flag.String("mode", "server", "Mode (server or client)")
	shouldFrame   = flag.Bool("framed", false, "Whether or not run in framed mode")
	shouldProfile = flag.Bool("profile", false, "Whether or not to profile")
	wg            sync.WaitGroup
)

func main() {
	runtime.GOMAXPROCS(1)
	flag.Parse()
	file, err := os.Create(fmt.Sprintf("/tmp/framed_cpu_%s", *mode))
	if err != nil {
		log.Fatal("Unable to create CPU profile file: %s", err)
	}
	if *shouldProfile {
		log.Println("Profiling")
		pprof.StartCPUProfile(file)
		defer pprof.StopCPUProfile()
	}
	if *mode == "client" {
		client()
	} else {
		server()
	}
}

func server() {
	log.Printf("Starting server at %s", serverAddr)
	if listener, err := net.Listen("tcp", serverAddr); err != nil {
		log.Fatalf("Unable to listen: %s", err)
	} else {
		for {
			if conn, err := listener.Accept(); err != nil {
				log.Printf("Unable to accept: %s", err)
			} else {
				if *shouldFrame {
					log.Println("Framing data")
					f := framed.NewFramed(conn)
					go func() {
						if frame, err := f.ReadInitial(); err != nil {
							log.Printf("Unable to read initial frame: %s", frame)
						} else {
							for {
								if err := frame.CopyTo(conn); err != nil {
									if *shouldProfile {
										pprof.StopCPUProfile()
										file, err := os.Create(fmt.Sprintf("/tmp/framed_heap_%s", *mode))
										if err != nil {
											log.Printf("Unable to create heap profile file: %s", err)
										} else {
											if err := pprof.WriteHeapProfile(file); err != nil {
												log.Printf("Unable to write heap profile: %s", err)
											}
										}
									}
									log.Fatalf("Unable to copy: %s", err)
								} else {
									if frame, err = frame.Next(); err != nil {
										log.Fatalf("Unable to read next frame")
									}
								}
							}
						}
					}()
				} else {
					log.Println("Doing a straight copy")
					go io.Copy(conn, conn)
				}
			}
		}
	}
}

func client() {
	for i := 0; i < 1; i++ {
		wg.Add(1)
		go doClient()
	}
	wg.Wait()
}

func doClient() {
	log.Printf("Starting client connection to server at %s", serverAddr)
	header := []byte{}
	data := []byte("Hell World")
	if conn, err := net.Dial("tcp", serverAddr); err != nil {
		log.Fatalf("Unable to dial server: %s", err)
	} else {
		f := framed.NewFramed(conn)
		// Read
		go io.Copy(ioutil.Discard, conn)

		// Write
		for {
			if err := f.WriteFrame(header, data); err != nil {
				log.Fatalf("Unable to write frame: %s", err)
			}
		}
	}
	wg.Done()
}
<commit_msg>Added heap profile<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/oxtoacart/framed"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"runtime"
	"runtime/pprof"
	"sync"
)

const (
	serverAddr = "127.0.0.1:10081"
)

var (
	mode          = flag.String("mode", "server", "Mode (server or client)")
	shouldFrame   = flag.Bool("framed", false, "Whether or not run in framed mode")
	shouldProfile = flag.Bool("profile", false, "Whether or not to profile")
	wg            sync.WaitGroup
)

func main() {
	runtime.GOMAXPROCS(1)
	flag.Parse()
	file, err := os.Create(fmt.Sprintf("/tmp/framed_cpu_%s", *mode))
	if err != nil {
		log.Fatal("Unable to create CPU profile file: %s", err)
	}
	if *shouldProfile {
		log.Println("Profiling")
		pprof.StartCPUProfile(file)
		defer pprof.StopCPUProfile()
	}
	if *mode == "client" {
		client()
	} else {
		server()
	}
}

func server() {
	log.Printf("Starting server at %s", serverAddr)
	if listener, err := net.Listen("tcp", serverAddr); err != nil {
		log.Fatalf("Unable to listen: %s", err)
	} else {
		for {
			if conn, err := listener.Accept(); err != nil {
				log.Printf("Unable to accept: %s", err)
			} else {
				if *shouldFrame {
					log.Println("Framing data")
					f := framed.NewFramed(conn)
					go func() {
						if frame, err := f.ReadInitial(); err != nil {
							log.Printf("Unable to read initial frame: %s", frame)
						} else {
							for {
								if err := frame.CopyTo(conn); err != nil {
									if *shouldProfile {
										pprof.StopCPUProfile()
										file, err := os.Create(fmt.Sprintf("/tmp/framed_heap_%s", *mode))
										if err != nil {
											log.Printf("Unable to create heap profile file: %s", err)
										} else {
											if err := pprof.WriteHeapProfile(file); err != nil {
												log.Printf("Unable to write heap profile: %s", err)
											} else {
												log.Printf("Wrote heap profile")
											}
										}
									}
									log.Fatalf("Unable to copy: %s", err)
								} else {
									if frame, err = frame.Next(); err != nil {
										log.Fatalf("Unable to read next frame")
									}
								}
							}
						}
					}()
				} else {
					log.Println("Doing a straight copy")
					go io.Copy(conn, conn)
				}
			}
		}
	}
}

func client() {
	for i := 0; i < 1; i++ {
		wg.Add(1)
		go doClient()
	}
	wg.Wait()
}

func doClient() {
	log.Printf("Starting client connection to server at %s", serverAddr)
	header := []byte{}
	data := []byte("Hell World")
	if conn, err := net.Dial("tcp", serverAddr); err != nil {
		log.Fatalf("Unable to dial server: %s", err)
	} else {
		f := framed.NewFramed(conn)
		// Read
		go io.Copy(ioutil.Discard, conn)

		// Write
		for {
			if err := f.WriteFrame(header, data); err != nil {
				log.Fatalf("Unable to write frame: %s", err)
			}
		}
	}
	wg.Done()
}
<|endoftext|>