<commit_before>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backend

import (
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"sync"

	"github.com/coreos/etcdlabs/cluster"
)

var (
	rootPortMu sync.Mutex
	rootPort   = 2379
)

func startCluster(rootCtx context.Context, rootCancel func()) (*cluster.Cluster, error) {
	rootPortMu.Lock()
	port := rootPort
	rootPort += 10 // for testing
	rootPortMu.Unlock()

	dir, err := ioutil.TempDir(os.TempDir(), "backend-cluster")
	if err != nil {
		return nil, err
	}

	cfg := cluster.Config{
		Size:          5,
		RootDir:       dir,
		RootPort:      port,
		ClientAutoTLS: true,
		RootCtx:       rootCtx,
		RootCancel:    rootCancel,
	}
	return cluster.Start(cfg)
}

// Server warps http.Server.
type Server struct {
	mu         sync.RWMutex
	addrURL    url.URL
	httpServer *http.Server

	rootCancel func()
	stopc      chan struct{}
	donec      chan struct{}
}

var globalCluster *cluster.Cluster

// StartServer starts a backend webserver with stoppable listener.
func StartServer(port int) (*Server, error) {
	stopc := make(chan struct{})
	ln, err := NewListenerStoppable("http", fmt.Sprintf("localhost:%d", port), nil, stopc)
	if err != nil {
		return nil, err
	}

	rootCtx, rootCancel := context.WithCancel(context.Background())
	c, err := startCluster(rootCtx, rootCancel)
	if err != nil {
		return nil, err
	}
	globalCluster = c

	mux := http.NewServeMux()
	mux.Handle("/server-status", &ContextAdapter{
		ctx:     rootCtx,
		handler: ContextHandlerFunc(serverStatusHandler),
	})
	mux.Handle("/client-request", &ContextAdapter{
		ctx:     rootCtx,
		handler: ContextHandlerFunc(clientRequestHandler),
	})

	addrURL := url.URL{Scheme: "http", Host: fmt.Sprintf("localhost:%d", port)}
	plog.Infof("started server %s", addrURL.String())
	srv := &Server{
		addrURL:    addrURL,
		httpServer: &http.Server{Addr: addrURL.String(), Handler: mux},
		rootCancel: rootCancel,
		stopc:      stopc,
		donec:      make(chan struct{}),
	}

	go func() {
		defer func() {
			if err := recover(); err != nil {
				plog.Errorf("etcd-play error (%v)", err)
				os.Exit(0)
			}
			srv.rootCancel()
			close(srv.donec)
		}()

		if err := srv.httpServer.Serve(ln); err != nil && err != ErrListenerStopped {
			plog.Panic(err)
		}
	}()
	return srv, nil
}

// StopNotify returns receive-only stop channel to notify the server has stopped.
func (srv *Server) StopNotify() <-chan struct{} {
	return srv.stopc
}

// Stop stops the server. Useful for testing.
func (srv *Server) Stop() {
	plog.Warningf("stopping server %s", srv.addrURL.String())
	srv.mu.Lock()
	if srv.httpServer == nil {
		srv.mu.Unlock()
		return
	}
	close(srv.stopc)
	<-srv.donec
	srv.httpServer = nil
	srv.mu.Unlock()
	plog.Warningf("stopped server %s", srv.addrURL.String())

	plog.Warning("stopping cluster")
	globalCluster.Shutdown()
	globalCluster = nil
	plog.Warning("stopped cluster")
}
<commit_msg>backend: pass root context to request limiter<commit_after>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backend

import (
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"sync"

	"github.com/coreos/etcdlabs/cluster"
)

var (
	rootPortMu sync.Mutex
	rootPort   = 2379
)

func startCluster(rootCtx context.Context, rootCancel func()) (*cluster.Cluster, error) {
	rootPortMu.Lock()
	port := rootPort
	rootPort += 10 // for testing
	rootPortMu.Unlock()

	dir, err := ioutil.TempDir(os.TempDir(), "backend-cluster")
	if err != nil {
		return nil, err
	}

	cfg := cluster.Config{
		Size:          5,
		RootDir:       dir,
		RootPort:      port,
		ClientAutoTLS: true,
		RootCtx:       rootCtx,
		RootCancel:    rootCancel,
	}
	return cluster.Start(cfg)
}

// Server warps http.Server.
type Server struct {
	mu         sync.RWMutex
	addrURL    url.URL
	httpServer *http.Server

	rootCancel func()
	stopc      chan struct{}
	donec      chan struct{}
}

var globalCluster *cluster.Cluster

// StartServer starts a backend webserver with stoppable listener.
func StartServer(port int) (*Server, error) {
	stopc := make(chan struct{})
	ln, err := NewListenerStoppable("http", fmt.Sprintf("localhost:%d", port), nil, stopc)
	if err != nil {
		return nil, err
	}

	rootCtx, rootCancel := context.WithCancel(context.Background())
	c, err := startCluster(rootCtx, rootCancel)
	if err != nil {
		return nil, err
	}
	globalCluster = c
	globalRequestLimiter.ctx = rootCtx

	mux := http.NewServeMux()
	mux.Handle("/server-status", &ContextAdapter{
		ctx:     rootCtx,
		handler: ContextHandlerFunc(serverStatusHandler),
	})
	mux.Handle("/client-request", &ContextAdapter{
		ctx:     rootCtx,
		handler: ContextHandlerFunc(clientRequestHandler),
	})

	addrURL := url.URL{Scheme: "http", Host: fmt.Sprintf("localhost:%d", port)}
	plog.Infof("started server %s", addrURL.String())
	srv := &Server{
		addrURL:    addrURL,
		httpServer: &http.Server{Addr: addrURL.String(), Handler: mux},
		rootCancel: rootCancel,
		stopc:      stopc,
		donec:      make(chan struct{}),
	}

	go func() {
		defer func() {
			if err := recover(); err != nil {
				plog.Errorf("etcd-play error (%v)", err)
				os.Exit(0)
			}
			srv.rootCancel()
			close(srv.donec)
		}()

		if err := srv.httpServer.Serve(ln); err != nil && err != ErrListenerStopped {
			plog.Panic(err)
		}
	}()
	return srv, nil
}

// StopNotify returns receive-only stop channel to notify the server has stopped.
func (srv *Server) StopNotify() <-chan struct{} {
	return srv.stopc
}

// Stop stops the server. Useful for testing.
func (srv *Server) Stop() {
	plog.Warningf("stopping server %s", srv.addrURL.String())
	srv.mu.Lock()
	if srv.httpServer == nil {
		srv.mu.Unlock()
		return
	}
	close(srv.stopc)
	<-srv.donec
	srv.httpServer = nil
	srv.mu.Unlock()
	plog.Warningf("stopped server %s", srv.addrURL.String())

	plog.Warning("stopping cluster")
	globalCluster.Shutdown()
	globalCluster = nil
	plog.Warning("stopped cluster")
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"math/rand"
	"socialapi/models"
	"strconv"
	"testing"
	. "github.com/smartystreets/goconvey/convey"
)

func TestPrivateMesssage(t *testing.T) {
	Convey("while testing private messages", t, func() {
		account := models.NewAccount()
		account.OldId = AccountOldId.Hex()
		account, err := createAccount(account)
		So(err, ShouldBeNil)
		So(account, ShouldNotBeNil)

		recipient := models.NewAccount()
		recipient.OldId = AccountOldId2.Hex()
		recipient, err = createAccount(recipient)
		So(err, ShouldBeNil)
		So(recipient, ShouldNotBeNil)

		recipient2 := models.NewAccount()
		recipient2.OldId = AccountOldId3.Hex()
		recipient2, err = createAccount(recipient2)
		So(err, ShouldBeNil)
		So(recipient2, ShouldNotBeNil)

		groupName := "testgroup" + strconv.FormatInt(rand.Int63(), 10)

		Convey("one can send private message to one person", func() {
			cmc, err := sendPrivateMessage(
				account.Id,
				"this is a body for private message",
				[]int64{recipient.Id},
				groupName,
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)

		})

		Convey("0 recipient should fail", func() {
			cmc, err := sendPrivateMessage(
				account.Id,
				"this is a body for private message",
				[]int64{},
				groupName,
			)
			So(err, ShouldNotBeNil)
			So(cmc, ShouldBeNil)

		})
		Convey("if body is nil, should fail to create PM", func() {
			cmc, err := sendPrivateMessage(
				account.Id,
				"",
				[]int64{recipient.Id},
				groupName,
			)
			So(err, ShouldNotBeNil)
			So(cmc, ShouldBeNil)
		})
		Convey("if group name is nil, should not fail to create PM", func() {
			cmc, err := sendPrivateMessage(
				account.Id,
				"this is a body for private message",
				[]int64{recipient.Id},
				"",
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)
		})

		Convey("if sender is not defined should fail to create PM", func() {
			cmc, err := sendPrivateMessage(
				0,
				"this is a body for private message",
				[]int64{recipient.Id},
				"",
			)
			So(err, ShouldNotBeNil)
			So(cmc, ShouldBeNil)
		})

		Convey("one can send private message to multiple person", func() {
			cmc, err := sendPrivateMessage(
				account.Id,
				"this is a body for private message",
				[]int64{recipient.Id, recipient2.Id},
				groupName,
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)

		})
		Convey("private message response should have created channel", func() {
			cmc, err := sendPrivateMessage(
				account.Id,
				"this is a body for private message",
				[]int64{recipient.Id, recipient2.Id},
				groupName,
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)
			So(cmc.Channel.TypeConstant, ShouldEqual, models.Channel_TYPE_PRIVATE_MESSAGE)
			So(cmc.Channel.Id, ShouldBeGreaterThan, 0)
			So(cmc.Channel.GroupName, ShouldEqual, groupName)
			So(cmc.Channel.PrivacyConstant, ShouldEqual, models.Channel_PRIVACY_PRIVATE)

		})

		Convey("private message response should have participant status data", func() {
			cmc, err := sendPrivateMessage(
				account.Id,
				"this is a body for private message",
				[]int64{recipient.Id, recipient2.Id},
				groupName,
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)
			So(cmc.IsParticipant, ShouldBeTrue)
		})

		Convey("private message response should have participant count", func() {
			cmc, err := sendPrivateMessage(
				account.Id,
				"this is a body for private message",
				[]int64{recipient.Id, recipient2.Id},
				groupName,
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)
			So(cmc.ParticipantCount, ShouldEqual, 3)
		})

		Convey("private message response should have participant preview", func() {
			cmc, err := sendPrivateMessage(
				account.Id,
				"this is a body for private message",
				[]int64{recipient.Id, recipient2.Id},
				groupName,
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)
			So(len(cmc.ParticipantsPreview), ShouldEqual, 3)
		})

		Convey("private message response should have last Message", func() {
			body := "this is a body for private message"
			cmc, err := sendPrivateMessage(
				account.Id,
				body,
				[]int64{recipient.Id, recipient2.Id},
				groupName,
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)
			So(cmc.LastMessage.Body, ShouldEqual, body)
		})

		Convey("private message should be listed by all recipients", func() {
			// use a different group name
			// in order not to interfere with another request
			groupName := "testgroup" + strconv.FormatInt(rand.Int63(), 10)

			body := "this is a body for private message"
			cmc, err := sendPrivateMessage(
				account.Id,
				body,
				[]int64{recipient.Id, recipient2.Id},
				groupName,
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)

			pm, err := getPrivateMessages(account.Id, groupName)
			So(err, ShouldBeNil)
			So(pm, ShouldNotBeNil)
			So(pm[0], ShouldNotBeNil)
			So(pm[0].Channel.TypeConstant, ShouldEqual, models.Channel_TYPE_PRIVATE_MESSAGE)
			So(pm[0].Channel.Id, ShouldEqual, cmc.Channel.Id)
			So(pm[0].Channel.GroupName, ShouldEqual, cmc.Channel.GroupName)
			So(pm[0].LastMessage.Body, ShouldEqual, cmc.LastMessage.Body)
			So(pm[0].Channel.PrivacyConstant, ShouldEqual, models.Channel_PRIVACY_PRIVATE)
			So(len(pm[0].ParticipantsPreview), ShouldEqual, 3)
			So(pm[0].IsParticipant, ShouldBeTrue)

		})

		Convey("targetted account should be able to list private message channel of himself", nil)

	})
}

func sendPrivateMessage(senderId int64, body string, recipients []int64, groupName string) (*models.ChannelContainer, error) {

	pmr := models.PrivateMessageRequest{}
	pmr.AccountId = senderId
	pmr.Body = body
	pmr.Recipients = recipients
	pmr.GroupName = groupName

	url := "/privatemessage/send"
	res, err := marshallAndSendRequest("POST", url, pmr)
	if err != nil {
		return nil, err
	}

	model := models.NewChannelContainer()
	err = json.Unmarshal(res, model)
	if err != nil {
		return nil, err
	}

	return model, nil
}

func getPrivateMessages(accountId int64, groupName string) ([]models.ChannelContainer, error) {
	url := fmt.Sprintf("/privatemessage/list?accountId=%d&groupName=%s", accountId, groupName)
	res, err := sendRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	var privateMessages []models.ChannelContainer
	err = json.Unmarshal(res, &privateMessages)
	if err != nil {
		return nil, err
	}

	return privateMessages, nil
}
<commit_msg>Social: replace participant list with @mentions<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"math/rand"
	"socialapi/models"
	"strconv"
	"testing"
	. "github.com/smartystreets/goconvey/convey"
)

func TestPrivateMesssage(t *testing.T) {
	Convey("while testing private messages", t, func() {
		account := models.NewAccount()
		account.OldId = AccountOldId.Hex()
		account, err := createAccount(account)
		So(err, ShouldBeNil)
		So(account, ShouldNotBeNil)

		recipient := models.NewAccount()
		recipient.OldId = AccountOldId2.Hex()
		recipient, err = createAccount(recipient)
		So(err, ShouldBeNil)
		So(recipient, ShouldNotBeNil)

		recipient2 := models.NewAccount()
		recipient2.OldId = AccountOldId3.Hex()
		recipient2, err = createAccount(recipient2)
		So(err, ShouldBeNil)
		So(recipient2, ShouldNotBeNil)

		groupName := "testgroup" + strconv.FormatInt(rand.Int63(), 10)

		Convey("one can send private message to one person", func() {
			cmc, err := sendPrivateMessage(
				account.Id,
				"this is a body message for private message @chris @devrim @sinan",
				groupName,
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)

		})

		Convey("0 recipient should fail", func() {
			cmc, err := sendPrivateMessage(
				account.Id,
				"this is a body for private message",
				groupName,
			)
			So(err, ShouldNotBeNil)
			So(cmc, ShouldBeNil)

		})
		Convey("if body is nil, should fail to create PM", func() {
			cmc, err := sendPrivateMessage(
				account.Id,
				"",
				groupName,
			)
			So(err, ShouldNotBeNil)
			So(cmc, ShouldBeNil)
		})
		Convey("if group name is nil, should not fail to create PM", func() {
			cmc, err := sendPrivateMessage(
				account.Id,
				"this is a body for private message @chris @devrim @sinan",
				"",
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)
		})

		Convey("if sender is not defined should fail to create PM", func() {
			cmc, err := sendPrivateMessage(
				0,
				"this is a body for private message",
				"",
			)
			So(err, ShouldNotBeNil)
			So(cmc, ShouldBeNil)
		})

		Convey("one can send private message to multiple person", func() {
			cmc, err := sendPrivateMessage(
				account.Id,
				"this is a body for private message @sinan",
				groupName,
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)

		})
		Convey("private message response should have created channel", func() {
			cmc, err := sendPrivateMessage(
				account.Id,
				"this is a body for private message @devrim @sinan",
				groupName,
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)
			So(cmc.Channel.TypeConstant, ShouldEqual, models.Channel_TYPE_PRIVATE_MESSAGE)
			So(cmc.Channel.Id, ShouldBeGreaterThan, 0)
			So(cmc.Channel.GroupName, ShouldEqual, groupName)
			So(cmc.Channel.PrivacyConstant, ShouldEqual, models.Channel_PRIVACY_PRIVATE)

		})

		Convey("private message response should have participant status data", func() {
			cmc, err := sendPrivateMessage(
				account.Id,
				"this is a body for private message @chris @devrim @sinan",
				groupName,
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)
			So(cmc.IsParticipant, ShouldBeTrue)
		})

		Convey("private message response should have participant count", func() {
			cmc, err := sendPrivateMessage(
				account.Id,
				"this is a body for @sinan private message @devrim",
				groupName,
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)
			So(cmc.ParticipantCount, ShouldEqual, 3)
		})

		Convey("private message response should have participant preview", func() {
			cmc, err := sendPrivateMessage(
				account.Id,
				"this is @chris a body for @devrim private message",
				groupName,
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)
			So(len(cmc.ParticipantsPreview), ShouldEqual, 3)
		})

		Convey("private message response should have last Message", func() {
			body := "hi @devrim this is a body for private message also for @chris"
			cmc, err := sendPrivateMessage(
				account.Id,
				body,
				groupName,
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)
			So(cmc.LastMessage.Body, ShouldEqual, body)
		})

		Convey("private message should be listed by all recipients", func() {
			// use a different group name
			// in order not to interfere with another request
			groupName := "testgroup" + strconv.FormatInt(rand.Int63(), 10)

			body := "hi @devrim this is a body for private message also for @chris"
			cmc, err := sendPrivateMessage(
				account.Id,
				body,
				groupName,
			)
			So(err, ShouldBeNil)
			So(cmc, ShouldNotBeNil)

			pm, err := getPrivateMessages(account.Id, groupName)
			So(err, ShouldBeNil)
			So(pm, ShouldNotBeNil)
			So(pm[0], ShouldNotBeNil)
			So(pm[0].Channel.TypeConstant, ShouldEqual, models.Channel_TYPE_PRIVATE_MESSAGE)
			So(pm[0].Channel.Id, ShouldEqual, cmc.Channel.Id)
			So(pm[0].Channel.GroupName, ShouldEqual, cmc.Channel.GroupName)
			So(pm[0].LastMessage.Body, ShouldEqual, cmc.LastMessage.Body)
			So(pm[0].Channel.PrivacyConstant, ShouldEqual, models.Channel_PRIVACY_PRIVATE)
			So(len(pm[0].ParticipantsPreview), ShouldEqual, 3)
			So(pm[0].IsParticipant, ShouldBeTrue)

		})

		Convey("targetted account should be able to list private message channel of himself", nil)

	})
}

func sendPrivateMessage(senderId int64, body string, groupName string) (*models.ChannelContainer, error) {

	pmr := models.PrivateMessageRequest{}
	pmr.AccountId = senderId
	pmr.Body = body
	pmr.GroupName = groupName

	url := "/privatemessage/send"
	res, err := marshallAndSendRequest("POST", url, pmr)
	if err != nil {
		return nil, err
	}

	model := models.NewChannelContainer()
	err = json.Unmarshal(res, model)
	if err != nil {
		return nil, err
	}

	return model, nil
}

func getPrivateMessages(accountId int64, groupName string) ([]models.ChannelContainer, error) {
	url := fmt.Sprintf("/privatemessage/list?accountId=%d&groupName=%s", accountId, groupName)
	res, err := sendRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	var privateMessages []models.ChannelContainer
	err = json.Unmarshal(res, &privateMessages)
	if err != nil {
		return nil, err
	}

	return privateMessages, nil
}
<|endoftext|>


<commit_before>package hammer

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"os"
	"sort"
	"sync"
	"time"

	"github.com/bmizerany/perks/quantile"
)

type Request *http.Request

type RequestGenerator func(*Hammer, chan<- Request, <-chan int)

type Hammer struct {
	Name             string
	RunFor           float64
	Threads          int
	Backlog          int
	QPS              float64
	ReadBody         bool
	LogErrors        bool
	GenerateFunction RequestGenerator
}

type result struct {
	Status     int
	Start      time.Time
	GotHeaders time.Time
	GotBody    time.Time
}

func (hammer *Hammer) warn(msg string) {
	log.Println(msg)
}

func (hammer *Hammer) warnf(fmt string, args ...interface{}) {
	log.Printf(fmt, args)
}

func (hammer *Hammer) sendRequests(requests <-chan Request, results chan<- result, wg *sync.WaitGroup) {
	defer wg.Done()

	client := &http.Client{}

	for req := range requests {
		var result result
		result.Start = time.Now()
		res, err := client.Do(req)
		result.GotHeaders = time.Now()
		if err != nil {
			result.Status = 499
			result.GotBody = result.GotHeaders
			hammer.warn(err.Error())
		} else {
			result.Status = res.StatusCode
			if result.Status >= 400 {
				if hammer.LogErrors {
					// TODO: refactor this into a method
					logOut, err := ioutil.TempFile(".", "error.log.")
					if err == nil {
						res.Write(logOut)
					} else {
						hammer.warnf("%s writing error log\n", err.Error())
					}
				}
				hammer.warnf("Got status %s for %s\n", res.Status, req.URL.String())
			} else if hammer.ReadBody {
				io.Copy(ioutil.Discard, res.Body)
				result.GotBody = time.Now()
			} else {
				res.Body.Close()
			}
		}
		results <- result
	}
}

type Stats struct {
	begin          time.Time
	end            time.Time
	statuses       map[int]int
	headerStats    BasicStats
	headerQuantile quantile.Stream
	bodyStats      BasicStats
	bodyQuantile   quantile.Stream
}

func newStats(name string, quantiles ...float64) Stats {
	return Stats{
		statuses:       make(map[int]int),
		headerStats:    BasicStats{},
		headerQuantile: *(quantile.NewTargeted(quantiles...)),
		bodyStats:      BasicStats{},
		bodyQuantile:   *(quantile.NewTargeted(quantiles...)),
	}
}

func (hammer *Hammer) ReportPrinter(format string) func(Stats) {
	return func(stats Stats) {
		file, err := os.Create(fmt.Sprintf(format, hammer.Name))
		if err != nil {
			hammer.warn(err.Error())
			return
		}
		runTime := stats.end.Sub(stats.begin).Seconds()
		count := stats.headerStats.Count
		fmt.Fprintf(
			file,
			`Hammer REPORT FOR %s:

Run time: %.3f
Total hits: %.0f
Hits/sec: %.3f

Status totals:
`,
			hammer.Name,
			runTime,
			count,
			count/runTime,
		)
		statusCodes := []int{}
		for code, _ := range stats.statuses {
			statusCodes = append(statusCodes, code)
		}
		sort.Ints(statusCodes)
		for _, code := range statusCodes {
			fmt.Fprintf(file, "%d\t%d\n", code, stats.statuses[code])
		}
		if count > 0 {
			fmt.Fprintf(
				file,
				"\nFirst byte mean +/- SD: %.2f +/- %.2f ms\n",
				1000*stats.headerStats.Mean(),
				1000*stats.headerStats.StdDev(),
			)
			fmt.Fprintf(
				file,
				"First byte 5-95 pct: (%.2f, %.2f) ms\n",
				1000*stats.headerQuantile.Query(0.05),
				1000*stats.headerQuantile.Query(0.95),
			)
			if hammer.ReadBody {
				fmt.Fprintf(
					file,
					"\nFull response mean +/- SD: %.2f +/- %.2f ms\n",
					1000*stats.bodyStats.Mean(),
					1000*stats.bodyStats.StdDev(),
				)
				fmt.Fprintf(
					file,
					"First byte 5-95 pct: (%.2f, %.2f) ms\n",
					1000*stats.bodyQuantile.Query(0.05),
					1000*stats.bodyQuantile.Query(0.95),
				)
			}
		}
		file.Close()
	}
}

func (hammer *Hammer) StatsPrinter(filename string) func(Stats) {
	return func(stats Stats) {
		statsFile, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			hammer.warn(err.Error())
			return
		}
		runTime := stats.end.Sub(stats.begin).Seconds()
		count := stats.headerStats.Count
		fmt.Fprintf(
			statsFile,
			"%s\t%d\t%f\t%f\t%f\t%f\t%f\t%f\t%f\t%f",
			hammer.Name,
			hammer.Threads,
			hammer.QPS,
			runTime,
			count,
			count/runTime,
			1000*stats.headerStats.Mean(),
			1000*stats.headerStats.StdDev(),
			1000*stats.headerQuantile.Query(0.05),
			1000*stats.headerQuantile.Query(0.95),
		)
		if hammer.ReadBody {
			fmt.Fprintf(
				statsFile,
				"%f\t%f\t%f\t%f\n",
				1000*stats.bodyStats.Mean(),
				1000*stats.bodyStats.StdDev(),
				1000*stats.bodyQuantile.Query(0.05),
				1000*stats.bodyQuantile.Query(0.95),
			)
		} else {
			fmt.Fprintf(statsFile, "\n")
		}
		statsFile.Close()
	}
}

func (hammer *Hammer) collectResults(results <-chan result, statschan chan<- Stats, wg *sync.WaitGroup) {
	defer wg.Done()

	var timeinit bool
	stats := newStats(hammer.Name, 0.05, 0.95)

	ticker := time.NewTicker(1 * time.Second)
	defer ticker.Stop()

	defer func() {
		statschan <- stats
		close(statschan)
	}()

	for {
		select {
		case res, ok := <-results:
			if !ok {
				return
			}
			stats.statuses[res.Status]++

			start := res.Start
			end := res.GotHeaders
			dur := end.Sub(start).Seconds()
			stats.headerStats.Add(dur)
			stats.headerQuantile.Insert(dur)
			if hammer.ReadBody {
				end = res.GotBody
				dur := end.Sub(start).Seconds()
				stats.bodyStats.Add(dur)
				stats.bodyQuantile.Insert(dur)
			}
			if !timeinit {
				stats.begin = start
				stats.end = end
				timeinit = true
			} else {
				if start.Before(stats.begin) {
					stats.begin = start
				}
				if start.After(stats.end) {
					stats.end = start
				}
			}
		case <-ticker.C:
			statschan <- stats
		}
	}
}

func RandomURLGenerator(URLs []string, Headers map[string][]string) RequestGenerator {
	readiedRequests := make([]Request, len(URLs))
	for i, url := range URLs {
		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			panic(err)
		}
		req.Header = Headers
		readiedRequests[i] = req
	}
	num := len(readiedRequests)

	return func(hammer *Hammer, requests chan<- Request, exit <-chan int) {
		defer func() { close(requests) }()

		ticker := time.NewTicker(time.Duration(float64(time.Second) / hammer.QPS))
		defer ticker.Stop()

		for {
			select {
			case <-exit:
				return
			case <-ticker.C:
				var idx int
				if num == 1 {
					idx = 0
				} else {
					idx = rand.Intn(len(readiedRequests))
				}
				requests <- readiedRequests[idx]
			}
		}
	}
}

func (hammer *Hammer) Run(statschan chan<- Stats) {
	exit := make(chan int)
	var requestWorkers, finishedResults sync.WaitGroup

	requests := make(chan Request, hammer.Backlog)
	results := make(chan result, hammer.Threads*2)

	for i := 0; i < hammer.Threads; i++ {
		requestWorkers.Add(1)
		go hammer.sendRequests(requests, results, &requestWorkers)
	}
	finishedResults.Add(1)
	go hammer.collectResults(results, statschan, &finishedResults)
	go hammer.GenerateFunction(hammer, requests, exit)
	go func() {
		requestWorkers.Wait()
		close(results)
	}()

	// Give it time to run...
	time.Sleep(time.Duration(hammer.RunFor * float64(time.Second)))
	// And then signal GenerateRequests to stop.
	close(exit)
	finishedResults.Wait()
}
<commit_msg>Fix warnf<commit_after>package hammer

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"os"
	"sort"
	"sync"
	"time"

	"github.com/bmizerany/perks/quantile"
)

type Request *http.Request

type RequestGenerator func(*Hammer, chan<- Request, <-chan int)

type Hammer struct {
	Name             string
	RunFor           float64
	Threads          int
	Backlog          int
	QPS              float64
	ReadBody         bool
	LogErrors        bool
	GenerateFunction RequestGenerator
}

type result struct {
	Status     int
	Start      time.Time
	GotHeaders time.Time
	GotBody    time.Time
}

func (hammer *Hammer) warn(msg string) {
	log.Println(msg)
}

func (hammer *Hammer) warnf(fmt string, args ...interface{}) {
	log.Printf(fmt, args...)
}

func (hammer *Hammer) sendRequests(requests <-chan Request, results chan<- result, wg *sync.WaitGroup) {
	defer wg.Done()

	client := &http.Client{}

	for req := range requests {
		var result result
		result.Start = time.Now()
		res, err := client.Do(req)
		result.GotHeaders = time.Now()
		if err != nil {
			result.Status = 499
			result.GotBody = result.GotHeaders
			hammer.warn(err.Error())
		} else {
			result.Status = res.StatusCode
			if result.Status >= 400 {
				if hammer.LogErrors {
					// TODO: refactor this into a method
					logOut, err := ioutil.TempFile(".", "error.log.")
					if err == nil {
						res.Write(logOut)
					} else {
						hammer.warnf("%s writing error log\n", err.Error())
					}
				}
				hammer.warnf("Got status %s for %s\n", res.Status, req.URL.String())
			} else if hammer.ReadBody {
				io.Copy(ioutil.Discard, res.Body)
				result.GotBody = time.Now()
			} else {
				res.Body.Close()
			}
		}
		results <- result
	}
}

type Stats struct {
	begin          time.Time
	end            time.Time
	statuses       map[int]int
	headerStats    BasicStats
	headerQuantile quantile.Stream
	bodyStats      BasicStats
	bodyQuantile   quantile.Stream
}

func newStats(name string, quantiles ...float64) Stats {
	return Stats{
		statuses:       make(map[int]int),
		headerStats:    BasicStats{},
		headerQuantile: *(quantile.NewTargeted(quantiles...)),
		bodyStats:      BasicStats{},
		bodyQuantile:   *(quantile.NewTargeted(quantiles...)),
	}
}

func (hammer *Hammer) ReportPrinter(format string) func(Stats) {
	return func(stats Stats) {
		file, err := os.Create(fmt.Sprintf(format, hammer.Name))
		if err != nil {
			hammer.warn(err.Error())
			return
		}
		runTime := stats.end.Sub(stats.begin).Seconds()
		count := stats.headerStats.Count
		fmt.Fprintf(
			file,
			`Hammer REPORT FOR %s:

Run time: %.3f
Total hits: %.0f
Hits/sec: %.3f

Status totals:
`,
			hammer.Name,
			runTime,
			count,
			count/runTime,
		)
		statusCodes := []int{}
		for code, _ := range stats.statuses {
			statusCodes = append(statusCodes, code)
		}
		sort.Ints(statusCodes)
		for _, code := range statusCodes {
			fmt.Fprintf(file, "%d\t%d\n", code, stats.statuses[code])
		}
		if count > 0 {
			fmt.Fprintf(
				file,
				"\nFirst byte mean +/- SD: %.2f +/- %.2f ms\n",
				1000*stats.headerStats.Mean(),
				1000*stats.headerStats.StdDev(),
			)
			fmt.Fprintf(
				file,
				"First byte 5-95 pct: (%.2f, %.2f) ms\n",
				1000*stats.headerQuantile.Query(0.05),
				1000*stats.headerQuantile.Query(0.95),
			)
			if hammer.ReadBody {
				fmt.Fprintf(
					file,
					"\nFull response mean +/- SD: %.2f +/- %.2f ms\n",
					1000*stats.bodyStats.Mean(),
					1000*stats.bodyStats.StdDev(),
				)
				fmt.Fprintf(
					file,
					"First byte 5-95 pct: (%.2f, %.2f) ms\n",
					1000*stats.bodyQuantile.Query(0.05),
					1000*stats.bodyQuantile.Query(0.95),
				)
			}
		}
		file.Close()
	}
}

func (hammer *Hammer) StatsPrinter(filename string) func(Stats) {
	return func(stats Stats) {
		statsFile, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			hammer.warn(err.Error())
			return
		}
		runTime := stats.end.Sub(stats.begin).Seconds()
		count := stats.headerStats.Count
		fmt.Fprintf(
			statsFile,
			"%s\t%d\t%f\t%f\t%f\t%f\t%f\t%f\t%f\t%f",
			hammer.Name,
			hammer.Threads,
			hammer.QPS,
			runTime,
			count,
			count/runTime,
			1000*stats.headerStats.Mean(),
			1000*stats.headerStats.StdDev(),
			1000*stats.headerQuantile.Query(0.05),
			1000*stats.headerQuantile.Query(0.95),
		)
		if hammer.ReadBody {
			fmt.Fprintf(
				statsFile,
				"%f\t%f\t%f\t%f\n",
				1000*stats.bodyStats.Mean(),
				1000*stats.bodyStats.StdDev(),
				1000*stats.bodyQuantile.Query(0.05),
				1000*stats.bodyQuantile.Query(0.95),
			)
		} else {
			fmt.Fprintf(statsFile, "\n")
		}
		statsFile.Close()
	}
}

func (hammer *Hammer) collectResults(results <-chan result, statschan chan<- Stats, wg *sync.WaitGroup) {
	defer wg.Done()

	var timeinit bool
	stats := newStats(hammer.Name, 0.05, 0.95)

	ticker := time.NewTicker(1 * time.Second)
	defer ticker.Stop()

	defer func() {
		statschan <- stats
		close(statschan)
	}()

	for {
		select {
		case res, ok := <-results:
			if !ok {
				return
			}
			stats.statuses[res.Status]++

			start := res.Start
			end := res.GotHeaders
			dur := end.Sub(start).Seconds()
			stats.headerStats.Add(dur)
			stats.headerQuantile.Insert(dur)
			if hammer.ReadBody {
				end = res.GotBody
				dur := end.Sub(start).Seconds()
				stats.bodyStats.Add(dur)
				stats.bodyQuantile.Insert(dur)
			}
			if !timeinit {
				stats.begin = start
				stats.end = end
				timeinit = true
			} else {
				if start.Before(stats.begin) {
					stats.begin = start
				}
				if start.After(stats.end) {
					stats.end = start
				}
			}
		case <-ticker.C:
			statschan <- stats
		}
	}
}

func RandomURLGenerator(URLs []string, Headers map[string][]string) RequestGenerator {
	readiedRequests := make([]Request, len(URLs))
	for i, url := range URLs {
		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			panic(err)
		}
		req.Header = Headers
		readiedRequests[i] = req
	}
	num := len(readiedRequests)

	return func(hammer *Hammer, requests chan<- Request, exit <-chan int) {
		defer func() { close(requests) }()

		ticker := time.NewTicker(time.Duration(float64(time.Second) / hammer.QPS))
		defer ticker.Stop()

		for {
			select {
			case <-exit:
				return
			case <-ticker.C:
				var idx int
				if num == 1 {
					idx = 0
				} else {
					idx = rand.Intn(len(readiedRequests))
				}
				requests <- readiedRequests[idx]
			}
		}
	}
}

func (hammer *Hammer) Run(statschan chan<- Stats) {
	exit := make(chan int)
	var requestWorkers, finishedResults sync.WaitGroup

	requests := make(chan Request, hammer.Backlog)
	results := make(chan result, hammer.Threads*2)

	for i := 0; i < hammer.Threads; i++ {
		requestWorkers.Add(1)
		go hammer.sendRequests(requests, results, &requestWorkers)
	}
	finishedResults.Add(1)
	go hammer.collectResults(results, statschan, &finishedResults)
	go hammer.GenerateFunction(hammer, requests, exit)
	go func() {
		requestWorkers.Wait()
		close(results)
	}()

	// Give it time to run...
	time.Sleep(time.Duration(hammer.RunFor * float64(time.Second)))
	// And then signal GenerateRequests to stop.
	close(exit)
	finishedResults.Wait()
}
<|endoftext|>


<commit_before>package hanapi

import (
    "fmt"
    "os"
    "sort"
    "math"
    "github.com/nlopes/slack"
    "github.com/kellydunn/golang-geo"
    "github.com/oliveroneill/hanserver/hanapi/imagedata"
    "github.com/oliveroneill/hanserver/hanapi/db"
    "github.com/oliveroneill/hanserver/hanapi/feedsort"
)

// RegionSize is radius of a region in meters
const RegionSize = 5000

// ContainsRegion - determines whether a point is within a specific region
func ContainsRegion(db db.DatabaseInterface, lat float64, lng float64) bool {
    return GetRegion(db, lat, lng) != nil
}

// GetRegion - returns the region which the specified lat, lng lies in
func GetRegion(db db.DatabaseInterface, lat float64, lng float64) *imagedata.Location {
    regions := db.GetRegions()
    currentPoint := geo.NewPoint(lat, lng)
    // loop through each region and return the first one that the point is
    // enclosed in
    for _, r := range regions {
        p := geo.NewPoint(r.Lat, r.Lng)
        if p.GreatCircleDistance(currentPoint) <= RegionSize / 1000 {
            return &r
        }
    }
    return nil
}

// GetRegions - returns the currently used regions
func GetRegions(db db.DatabaseInterface) []imagedata.Location {
    return db.GetRegions()
}

// AddRegion - adds a new region for image population
func AddRegion(db db.DatabaseInterface, lat float64, lng float64) {
    if !ContainsRegion(db, lat, lng) {
        db.AddRegion(lat, lng)
    }
}

// GetImages - get images near the location sorted by distance and recency
func GetImages(db db.DatabaseInterface, lat float64, lng float64) []imagedata.ImageData {
    return GetImagesWithRange(db, lat, lng, -1, -1)
}

// GetImagesWithStart - get images starting at a certain point
func GetImagesWithStart(db db.DatabaseInterface, lat float64, lng float64,
    start int) []imagedata.ImageData {
    return GetImagesWithRange(db, lat, lng, start, -1)
}

// GetImagesWithEnd - get images from the beginning to the specified end
func GetImagesWithEnd(db db.DatabaseInterface, lat float64, lng float64,
    end int) []imagedata.ImageData {
    return GetImagesWithRange(db, lat, lng, -1, end)
}

// GetImagesWithRange - Specify a range, so that you can query a portion of the image list
// @param start - start is optional, use -1 to signify no value, indexing starts at zero
// @param end - end is optional, use -1 to signify no value
func GetImagesWithRange(db db.DatabaseInterface, lat float64, lng float64,
    start int, end int) []imagedata.ImageData {
    // 100 images will be sorted at a time
    return getImagesWithRangeAndSampleSize(db, lat, lng, start, end, 100)
}

// SampleSize will determine the bias on distance, as we aren't sorting the
// whole database on recency on distance. Only the closest images are then
// sorted
func getImagesWithRangeAndSampleSize(db db.DatabaseInterface, lat float64,
    lng float64, start int, end int, sampleSize int) []imagedata.ImageData {
    // fix input values
    if start < 0 {
        start = 0
    }
    if end < 0 {
        end = start + sampleSize
    }
    // remove incorrect requests
    if end < start {
        return []imagedata.ImageData{}
    }
    startSort, endSort := getRange(sampleSize, start, end)

    images := []imagedata.ImageData{}
    // if the request is larger than our sample size then we need to sort
    // multiple sets of our sample size individually to avoid duplicate
    // images
    if endSort - startSort > sampleSize {
        // calculate the range queries that need to be made to only ever sort
        // arrays of sample size
        rangeEnd := end
        rangeStart := start
        // edge case will not return the previous sampleSize boundary, so we
        // subtract and add 1 to the range to get new start and end values
        // which specify where to query and sort
        if end % sampleSize == 0 {
            rangeEnd--
        }
        if start % sampleSize == 0 {
            rangeStart++
        }
        // work out the nearest sample size from the start position and the
        // next smallest sample size from the end position
        closestEnd, closestStart := getRange(sampleSize, rangeEnd, rangeStart)
        if start == 0 {
            closestEnd = sampleSize
        }
        // call image range recursively
        // get the first portion
        images = append(images, getImagesWithRangeAndSampleSize(db, lat, lng, start, closestEnd, sampleSize)...)
        // go through all sample size chunks in between the start and end portion
        for i := closestEnd; i < closestStart; i += sampleSize {
            images = append(images, getImagesWithRangeAndSampleSize(db, lat, lng, i, i+sampleSize, sampleSize)...)
        }
        // get the last portion
        images = append(images, getImagesWithRangeAndSampleSize(db, lat, lng, closestStart, end, sampleSize)...)
        return images
    }
    // this is the base case where we get the images and sort
    images = db.GetImages(lat, lng, startSort, endSort)
    // sort
    sort.Sort(feedsort.BySum(images))
    // figure out where to slice the array
    sliceStart := start - startSort
    sliceEnd := sliceStart + (end - start)
    // return empty if we're out of range
    if sliceStart > len(images) {
        return []imagedata.ImageData{}
    }
    // set relevant values since start and end can be optional
    if sliceStart < 0 {
        sliceStart = 0
    }
    if sliceEnd < 0 || sliceEnd > len(images) {
        sliceEnd = len(images)
    }
    return images[sliceStart:sliceEnd]
}

func getRange(sampleSize int, start int, end int) (int, int) {
    startSort := int(math.Floor(float64(start)/float64(sampleSize)) * float64(sampleSize))
    endSort := int(math.Ceil(float64(end)/float64(sampleSize)) * float64(sampleSize))
    return startSort, endSort
}

// ReportImage - report an image to be removed
// @param id - the image ID which should match one in imagedata.ImageData
// @param reason - reason for reporting
func ReportImage(db db.DatabaseInterface, id string, reason string) {
    db.SoftDelete(id, reason)

    // notify through Slack bot
    apiToken := os.Getenv("SLACK_API_TOKEN")
    if len(apiToken) == 0 {
        return
    }
    channelName := "hanserver"
    api := slack.New(apiToken)
    params := slack.PostMessageParameters{}
    _, _, err := api.PostMessage(channelName, fmt.Sprintf("Image %s reported because: %s", id, reason), params)
    if err != nil {
        fmt.Println(err)
        return
    }
}
<commit_msg>better comments<commit_after>package hanapi

import (
    "fmt"
    "os"
    "sort"
    "math"
    "github.com/nlopes/slack"
    "github.com/kellydunn/golang-geo"
    "github.com/oliveroneill/hanserver/hanapi/imagedata"
    "github.com/oliveroneill/hanserver/hanapi/db"
    "github.com/oliveroneill/hanserver/hanapi/feedsort"
)

// RegionSize is radius of a region in meters
const RegionSize = 5000

// ContainsRegion - determines whether a point is within a specific region
func ContainsRegion(db db.DatabaseInterface, lat float64, lng float64) bool {
    return GetRegion(db, lat, lng) != nil
}

// GetRegion - returns the region which the specified lat, lng lies in
func GetRegion(db db.DatabaseInterface, lat float64, lng float64) *imagedata.Location {
    regions := db.GetRegions()
    currentPoint := geo.NewPoint(lat, lng)
    // loop through each region and return the first one that the point is
    // enclosed in
    for _, r := range regions {
        p := geo.NewPoint(r.Lat, r.Lng)
        if p.GreatCircleDistance(currentPoint) <= RegionSize / 1000 {
            return &r
        }
    }
    return nil
}

// GetRegions - returns the currently used regions
func GetRegions(db db.DatabaseInterface) []imagedata.Location {
    return db.GetRegions()
}

// AddRegion - adds a new region for image population
func AddRegion(db db.DatabaseInterface, lat float64, lng float64) {
    if !ContainsRegion(db, lat, lng) {
        db.AddRegion(lat, lng)
    }
}

// GetImages - get images near the location sorted by distance and recency
func GetImages(db db.DatabaseInterface, lat float64, lng float64) []imagedata.ImageData {
    return GetImagesWithRange(db, lat, lng, -1, -1)
}

// GetImagesWithStart - get images starting at a certain point
func GetImagesWithStart(db db.DatabaseInterface, lat float64, lng float64,
    start int) []imagedata.ImageData {
    return GetImagesWithRange(db, lat, lng, start, -1)
}

// GetImagesWithEnd - get images from the beginning to the specified end
func GetImagesWithEnd(db db.DatabaseInterface, lat float64, lng float64,
    end int) []imagedata.ImageData {
    return GetImagesWithRange(db, lat, lng, -1, end)
}

// GetImagesWithRange - Specify a range, so that you can query a portion of the image list
// @param start - start is optional, use -1 to signify no value, indexing starts at zero
// @param end - end is optional, use -1 to signify no value
func GetImagesWithRange(db db.DatabaseInterface, lat float64, lng float64,
    start int, end int) []imagedata.ImageData {
    // 100 images will be sorted at a time
    return getImagesWithRangeAndSampleSize(db, lat, lng, start, end, 100)
}

// SampleSize will determine the bias on distance, as we aren't sorting the
// whole database on recency on distance. Only the closest images are then
// sorted.
// Sticking to this sample size is important in avoiding inconsistent
// sort that could result in duplicate images due to an image being
// ranked highly in a smaller subset than an earlier query.
// This system allows for a deterministic way to sort and return
// these images consistently
// This function ensures that `sampleSize` worth of images is only ever
// sorted and nothing smaller or inbetween is ever queried.
func getImagesWithRangeAndSampleSize(db db.DatabaseInterface, lat float64,
    lng float64, start int, end int, sampleSize int) []imagedata.ImageData {
    // fix input values
    if start < 0 {
        start = 0
    }
    if end < 0 {
        end = start + sampleSize
    }
    // remove incorrect requests
    if end < start {
        return []imagedata.ImageData{}
    }
    startSort, endSort := getRange(sampleSize, start, end)

    images := []imagedata.ImageData{}
    // if the request is larger than our sample size then we need to sort
    // multiple sets of our sample size individually to avoid duplicate
    // images
    if endSort - startSort > sampleSize {
        // calculate the range queries that need to be made to only ever sort
        // arrays of sample size
        rangeEnd := end
        rangeStart := start
        // edge case will not return the previous sampleSize boundary, so we
        // subtract and add 1 to the range to get new start and end values
        // which specify where to query and sort
        if end % sampleSize == 0 {
            rangeEnd--
        }
        if start % sampleSize == 0 {
            rangeStart++
        }
        // work out the nearest sample size from the start position and the
        // next smallest sample size from the end position
        closestEnd, closestStart := getRange(sampleSize, rangeEnd, rangeStart)
        if start == 0 {
            closestEnd = sampleSize
        }
        // call image range recursively
        // get the first portion
        images = append(images, getImagesWithRangeAndSampleSize(db, lat, lng, start, closestEnd, sampleSize)...)
        // go through all sample size chunks in between the start and end portion
        for i := closestEnd; i < closestStart; i += sampleSize {
            images = append(images, getImagesWithRangeAndSampleSize(db, lat, lng, i, i+sampleSize, sampleSize)...)
        }
        // get the last portion
        images = append(images, getImagesWithRangeAndSampleSize(db, lat, lng, closestStart, end, sampleSize)...)
        return images
    }
    // this is the base case where we get the images and sort
    images = db.GetImages(lat, lng, startSort, endSort)
    // sort
    sort.Sort(feedsort.BySum(images))
    // figure out where to slice the array
    sliceStart := start - startSort
    sliceEnd := sliceStart + (end - start)
    // return empty if we're out of range
    if sliceStart > len(images) {
        return []imagedata.ImageData{}
    }
    // set relevant values since start and end can be optional
    if sliceStart < 0 {
        sliceStart = 0
    }
    if sliceEnd < 0 || sliceEnd > len(images) {
        sliceEnd = len(images)
    }
    return images[sliceStart:sliceEnd]
}

func getRange(sampleSize int, start int, end int) (int, int) {
    startSort := int(math.Floor(float64(start)/float64(sampleSize)) * float64(sampleSize))
    endSort := int(math.Ceil(float64(end)/float64(sampleSize)) * float64(sampleSize))
    return startSort, endSort
}

// ReportImage - report an image to be removed
// @param id - the image ID which should match one in imagedata.ImageData
// @param reason - reason for reporting
func ReportImage(db db.DatabaseInterface, id string, reason string) {
    db.SoftDelete(id, reason)

    // notify through Slack bot
    apiToken := os.Getenv("SLACK_API_TOKEN")
    if len(apiToken) == 0 {
        return
    }
    channelName := "hanserver"
    api := slack.New(apiToken)
    params := slack.PostMessageParameters{}
    _, _, err := api.PostMessage(channelName, fmt.Sprintf("Image %s reported because: %s", id, reason), params)
    if err != nil {
        fmt.Println(err)
        return
    }
}
<|endoftext|>


<commit_before>package integration_test

import (
	"path/filepath"

	"github.com/cloudfoundry/libbuildpack/cutlass"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("deploy a staticfile app", func() {
	var app *cutlass.App
	AfterEach(func() {
		if app != nil {
			app.Destroy()
		}
		app = nil
	})

	BeforeEach(func() {
		app = cutlass.New(filepath.Join(bpDir, "fixtures", "reverse_proxy"))
		PushAppAndConfirm(app)
	})

	It("proxies", func() {
		Expect(app.GetBody("/intl/en/policies")).To(ContainSubstring("Welcome to the Google Privacy Policy"))

		By("hides the nginx.conf file", func() {
			Expect(app.GetBody("/nginx.conf")).To(ContainSubstring("<title>404 Not Found</title>"))
		})
	})
})
<commit_msg>Fix integration tests due to Google Privacy Policy content changes.<commit_after>package integration_test

import (
	"path/filepath"

	"github.com/cloudfoundry/libbuildpack/cutlass"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("deploy a staticfile app", func() {
	var app *cutlass.App
	AfterEach(func() {
		if app != nil {
			app.Destroy()
		}
		app = nil
	})

	BeforeEach(func() {
		app = cutlass.New(filepath.Join(bpDir, "fixtures", "reverse_proxy"))
		PushAppAndConfirm(app)
	})

	It("proxies", func() {
		Expect(app.GetBody("/intl/en/policies")).To(ContainSubstring("Google Privacy Policy"))

		By("hides the nginx.conf file", func() {
			Expect(app.GetBody("/nginx.conf")).To(ContainSubstring("<title>404 Not Found</title>"))
		})
	})
})
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build amd64 386

package sha1

func block(dig *digest, p []byte)
<commit_msg>crypto/sha1: mark block as non-escaping The compiler still gets the escape analysis wrong, but the annotation here is correct.<commit_after>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build amd64 386

package sha1

//go:noescape

func block(dig *digest, p []byte)
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2015, Henrik Mattsson
 * All rights reserved. See LICENSE.
 */

/*
 * scrabbled client
 */

package main

import (
	"github.com/hjmat/scrabbled/condlog"
	"github.com/hjmat/scrabbled/keyloader"
	scrabble "github.com/hjmat/scrabbled/proto"

	"github.com/golang/protobuf/proto"
	zmq "github.com/pebbe/zmq4"

	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path"
)

func solve(hand string, sock *zmq.Socket) []string {
	req := &scrabble.Request{Hand: hand}

	reqMsg, err := proto.Marshal(req)
	condlog.Fatal(err, "Unable to marshal message")

	_, err = sock.Send(string(reqMsg), 0)
	condlog.Fatal(err, "Unable to marshal message")

	resMsg, err := sock.Recv(0)
	condlog.Fatal(err, "Unable to receive response")

	res := &scrabble.Response{}
	err = proto.Unmarshal([]byte(resMsg), res)
	condlog.Fatal(err, "Unable to unmarshal request")

	return res.Words
}

func initSecurity(private_key_path string, server_key_path string, sock *zmq.Socket) {
	zmq.AuthStart()
	private_key, public_key, err := keyloader.InitKeys(private_key_path)
	condlog.Fatal(err, fmt.Sprintf("Unable to read key pair for private key '%v'", private_key_path))
	zmq.AuthCurveAdd("scrabble", public_key)

	server_key_buf, err := ioutil.ReadFile(server_key_path)
	condlog.Fatal(err, fmt.Sprintf("Unable to load public server key '%v'", server_key_path))
	server_key := string(server_key_buf)
	sock.ClientAuthCurve(server_key, public_key, private_key)
}

func main() {
	hostPtr := flag.String("host", "localhost", "hostname of the server")
	portPtr := flag.Int("port", 30000, "port that the server runs on")
	keyPtr := flag.String("key", "client.private", "private key for authentication")
	servKeyPtr := flag.String("servkey", "server.public", "public key of the server")
	flag.Parse()

	sock, err := zmq.NewSocket(zmq.REQ)
	condlog.Fatal(err, "Unable to create socket")
	defer sock.Close()
	initSecurity(path.Clean(*keyPtr), path.Clean(*servKeyPtr), sock)

	err = sock.Connect(fmt.Sprintf("tcp://%s:%d", *hostPtr, *portPtr))
	condlog.Fatal(err, "Unable to connect")

	in := bufio.NewScanner(os.Stdin)
	fmt.Print("> ")
	for in.Scan() {
		hand := in.Text()

		results := solve(hand, sock)
		for _, r := range results {
			fmt.Println(r)
		}
		fmt.Print("> ")
	}

	condlog.Fatal(in.Err(), "Unable to read from stdin")
}
<commit_msg>check permissions in client too<commit_after>/*
 * Copyright (c) 2015, Henrik Mattsson
 * All rights reserved. See LICENSE.
 */

/*
 * scrabbled client
 */

package main

import (
	"github.com/hjmat/scrabbled/condlog"
	"github.com/hjmat/scrabbled/keyloader"
	scrabble "github.com/hjmat/scrabbled/proto"

	"github.com/golang/protobuf/proto"
	zmq "github.com/pebbe/zmq4"

	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path"
)

func solve(hand string, sock *zmq.Socket) []string {
	req := &scrabble.Request{Hand: hand}

	reqMsg, err := proto.Marshal(req)
	condlog.Fatal(err, "Unable to marshal message")

	_, err = sock.Send(string(reqMsg), 0)
	condlog.Fatal(err, "Unable to marshal message")

	resMsg, err := sock.Recv(0)
	condlog.Fatal(err, "Unable to receive response")

	res := &scrabble.Response{}
	err = proto.Unmarshal([]byte(resMsg), res)
	condlog.Fatal(err, "Unable to unmarshal request")

	return res.Words
}

func initSecurity(private_key_path string, server_key_path string, sock *zmq.Socket) {
	zmq.AuthStart()
	private_key, public_key, err := keyloader.InitKeys(private_key_path)
	condlog.Fatal(err, fmt.Sprintf("Unable to read key pair for private key '%v'", private_key_path))
	zmq.AuthCurveAdd("scrabble", public_key)

	err = keyloader.CheckPermissions(server_key_path)
	condlog.Fatal(err, "Untrustworthy key file")
	server_key_buf, err := ioutil.ReadFile(server_key_path)
	condlog.Fatal(err, fmt.Sprintf("Unable to load public server key '%v'", server_key_path))
	server_key := string(server_key_buf)
	sock.ClientAuthCurve(server_key, public_key, private_key)
}

func main() {
	hostPtr := flag.String("host", "localhost", "hostname of the server")
	portPtr := flag.Int("port", 30000, "port that the server runs on")
	keyPtr := flag.String("key", "client.private", "private key for authentication")
	servKeyPtr := flag.String("servkey", "server.public", "public key of the server")
	flag.Parse()

	sock, err := zmq.NewSocket(zmq.REQ)
	condlog.Fatal(err, "Unable to create socket")
	defer sock.Close()
	initSecurity(path.Clean(*keyPtr), path.Clean(*servKeyPtr), sock)

	err = sock.Connect(fmt.Sprintf("tcp://%s:%d", *hostPtr, *portPtr))
	condlog.Fatal(err, "Unable to connect")

	in := bufio.NewScanner(os.Stdin)
	fmt.Print("> ")
	for in.Scan() {
		hand := in.Text()

		results := solve(hand, sock)
		for _, r := range results {
			fmt.Println(r)
		}
		fmt.Print("> ")
	}

	condlog.Fatal(in.Err(), "Unable to read from stdin")
}
<|endoftext|>


<commit_before>package glock

import (
	"bufio"
	"errors"
	"fmt"
	"log"
	"net"
	"strconv"
	"strings"
	"time"
)

type Client struct {
	endpoint string
	conn     net.Conn
	reader   *bufio.Reader
}

func NewClient(endpoint string) (*Client, error) {
	conn, err := net.Dial("tcp", endpoint)
	reader := bufio.NewReader(conn)
	return &Client{endpoint, conn, reader}, err
}

func (c *Client) redial() error {
	conn, err := net.Dial("tcp", c.endpoint)
	if err != nil {
		return err
	}
	c.conn = conn
	c.reader = bufio.NewReader(conn)

	return nil
}

func (c *Client) Lock(key string, duration time.Duration) (id int64, err error) {
	err = c.checkConn()
	if err != nil {
		return id, err
	}

	fmt.Fprintf(c.conn, "LOCK %s %d \r\n", key, int(duration/time.Millisecond))

	splits, err := c.readResponse()
	if err != nil {
		return id, err
	}

	id, err = strconv.ParseInt(splits[1], 10, 64)
	if err != nil {
		return id, err
	}

	return id, nil
}

func (c *Client) Unlock(key string, id int64) (err error) {
	err = c.checkConn()
	if err != nil {
		return err
	}

	fmt.Fprintf(c.conn, "UNLOCK %s %d \r\n", key, id)

	splits, err := c.readResponse()
	if err != nil {
		return err
	}

	cmd := splits[0]
	switch cmd {
	case "LOCKED":
		return errors.New("LOCKED")
	case "UNLOCKED":
		return nil
	}
	return errors.New("Unknown reponse format")
}

func (c *Client) Close() error {
	err := c.conn.Close()
	return err
}

func (c *Client) readResponse() (splits []string, err error) {
	response, err := c.reader.ReadString('\n')
	log.Println("glockResponse: ", response)
	if err != nil {
		return nil, err
	}

	trimmedResponse := strings.TrimRight(response, "\r\n")
	splits = strings.Split(trimmedResponse, " ")
	if splits[0] == "ERROR" {
		return nil, errors.New(trimmedResponse)
	}

	return splits, nil
}

func (c *Client) checkConn() error {
	buffer := make([]byte, 0)
	_, err := c.conn.Read(buffer)
	if err != nil {
		c.Close()
		err = c.redial()
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Method to redial 3 times if connection drops<commit_after>package glock

import (
	"bufio"
	"errors"
	"fmt"
	"log"
	"net"
	"strconv"
	"strings"
	"time"
)

type Client struct {
	endpoint string
	conn     net.Conn
	reader   *bufio.Reader
}

func NewClient(endpoint string) (*Client, error) {
	conn, err := net.Dial("tcp", endpoint)
	reader := bufio.NewReader(conn)
	return &Client{endpoint, conn, reader}, err
}

func (c *Client) redial() error {
	c.Close()
	conn, err := net.Dial("tcp", c.endpoint)
	if err != nil {
		return err
	}
	c.conn = conn
	c.reader = bufio.NewReader(conn)

	return nil
}

func (c *Client) Lock(key string, duration time.Duration) (id int64, err error) {
	err = c.fprintf("LOCK %s %d \r\n", key, int(duration/time.Millisecond))
	if err != nil {
		return id, err
	}

	splits, err := c.readResponse()
	if err != nil {
		return id, err
	}

	id, err = strconv.ParseInt(splits[1], 10, 64)
	if err != nil {
		return id, err
	}

	return id, nil
}

func (c *Client) Unlock(key string, id int64) (err error) {
	err = c.fprintf("UNLOCK %s %d \r\n", key, id)
	if err != nil {
		return err
	}

	splits, err := c.readResponse()
	if err != nil {
		return err
	}

	cmd := splits[0]
	switch cmd {
	case "LOCKED":
		return errors.New("LOCKED")
	case "UNLOCKED":
		return nil
	}
	return errors.New("Unknown reponse format")
}

func (c *Client) Close() error {
	err := c.conn.Close()
	return err
}

func (c *Client) readResponse() (splits []string, err error) {
	response, err := c.reader.ReadString('\n')
	log.Println("glockResponse: ", response)
	if err != nil {
		return nil, err
	}

	trimmedResponse := strings.TrimRight(response, "\r\n")
	splits = strings.Split(trimmedResponse, " ")
	if splits[0] == "ERROR" {
		return nil, errors.New(trimmedResponse)
	}

	return splits, nil
}

func (c *Client) fprintf(format string, a ...interface{}) error {
	for i := 0; i < 3; i++ {
		_, err := fmt.Fprintf(c.conn, format, a...)
		if err != nil {
			err = c.redial()
			if err != nil {
				return err
			}
		} else {
			return nil
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package client

import (
	"errors"
	"fmt"
	"os"
	"time"

	log "github.com/cihub/seelog"
	"github.com/golang/protobuf/proto"
	"github.com/nu7hatch/gouuid"
	"github.com/streadway/amqp"
	"github.com/vinceprignano/bunny/rabbit"
)

var Exchange string

type Client struct {
	Name       string
	dispatcher *dispatcher
	replyTo    string
	connection *rabbit.RabbitConnection
}

func init() {
	Exchange = os.Getenv("RABBIT_EXCHANGE")
}

var NewClient = func(name string) *Client {
	uuidQueue, err := uuid.NewV4()
	if err != nil {
		log.Criticalf("[Client] Failed to create UUID for reply queue")
		os.Exit(1)
	}
	return &Client{
		Name:       name,
		dispatcher: newDispatcher(),
		connection: rabbit.NewRabbitConnection(),
		replyTo:    fmt.Sprintf("replyTo-%s-%s", name, uuidQueue.String()),
	}
}

func (c *Client) Init() {
	select {
	case <-c.connection.Init():
		log.Info("[Client] Connected to transport")
	case <-time.After(10 * time.Second):
		log.Critical("[Client] Failed to connect to transport")
		os.Exit(1)
	}
	c.initConsume()
}

func (c *Client) initConsume() {
	err := c.connection.Channel.DeclareReplyQueue(c.replyTo)
	if err != nil {
		log.Critical("[Client] Failed to declare reply queue")
		log.Critical(err.Error())
		os.Exit(1)
	}
	deliveries, err := c.connection.Channel.ConsumeQueue(c.replyTo)
	if err != nil {
		log.Critical("[Client] Failed to consume from reply queue")
		log.Critical(err.Error())
		os.Exit(1)
	}
	go func() {
		log.Infof("[Client] Listening for deliveries on %s", c.replyTo)
		for delivery := range deliveries {
			go c.handleDelivery(delivery)
		}
	}()
}

func (c *Client) handleDelivery(delivery amqp.Delivery) {
	channel := c.dispatcher.pop(delivery.CorrelationId)
	if channel == nil {
		log.Errorf("[Client] CorrelationID -> %s does not exist in dispatcher", delivery.CorrelationId)
		return
	}
	select {
	case channel <- delivery:
	default:
		log.Errorf("[Client] Error in delivery for correlation %s", delivery.CorrelationId)
	}
}

func (c *Client) Call(routingKey string, req proto.Message, res proto.Message) error {
	correlation, err := uuid.NewV4()
	if err != nil {
		log.Error("[Client] Failed to create correlationId in client")
		return errors.New("client.call.uuid.error")
	}

	replyChannel := c.dispatcher.add(correlation.String())
	defer close(replyChannel)

	requestBody, err := proto.Marshal(req)
	if err != nil {
		log.Error("[Client] Failed to marshal request")
		return errors.New("client.call.marshal.error")
	}

	message := amqp.Publishing{
		CorrelationId: correlation.String(),
		Timestamp:     time.Now().UTC(),
		Body:          requestBody,
		ReplyTo:       c.replyTo,
	}

	err = c.connection.Publish(Exchange, routingKey, message)
	if err != nil {
		log.Errorf("[Client] Failed to publish to %s", routingKey)
		return fmt.Errorf("client.call.publish.%s.error", routingKey)
	}

	select {
	case delivery := <-replyChannel:
		if err := proto.Unmarshal(delivery.Body, res); err != nil {
			return fmt.Errorf("client.unmarshal.%s-reply.error", routingKey)
		}
		return nil
	case <-time.After(10 * time.Second):
		log.Criticalf("[Client] Client timeout on delivery")
		return fmt.Errorf("client.call.timeout.%s.error", routingKey)
	}
}
<commit_msg>Remove exchange in client<commit_after>package client

import (
	"errors"
	"fmt"
	"os"
	"time"

	log "github.com/cihub/seelog"
	"github.com/golang/protobuf/proto"
	"github.com/nu7hatch/gouuid"
	"github.com/streadway/amqp"
	"github.com/vinceprignano/bunny/rabbit"
)

type Client struct {
	Name       string
	dispatcher *dispatcher
	replyTo    string
	connection *rabbit.RabbitConnection
}

var NewClient = func(name string) *Client {
	uuidQueue, err := uuid.NewV4()
	if err != nil {
		log.Criticalf("[Client] Failed to create UUID for reply queue")
		os.Exit(1)
	}
	return &Client{
		Name:       name,
		dispatcher: newDispatcher(),
		connection: rabbit.NewRabbitConnection(),
		replyTo:    fmt.Sprintf("replyTo-%s-%s", name, uuidQueue.String()),
	}
}

func (c *Client) Init() {
	select {
	case <-c.connection.Init():
		log.Info("[Client] Connected to transport")
	case <-time.After(10 * time.Second):
		log.Critical("[Client] Failed to connect to transport")
		os.Exit(1)
	}
	c.initConsume()
}

func (c *Client) initConsume() {
	err := c.connection.Channel.DeclareReplyQueue(c.replyTo)
	if err != nil {
		log.Critical("[Client] Failed to declare reply queue")
		log.Critical(err.Error())
		os.Exit(1)
	}
	deliveries, err := c.connection.Channel.ConsumeQueue(c.replyTo)
	if err != nil {
		log.Critical("[Client] Failed to consume from reply queue")
		log.Critical(err.Error())
		os.Exit(1)
	}
	go func() {
		log.Infof("[Client] Listening for deliveries on %s", c.replyTo)
		for delivery := range deliveries {
			go c.handleDelivery(delivery)
		}
	}()
}

func (c *Client) handleDelivery(delivery amqp.Delivery) {
	channel := c.dispatcher.pop(delivery.CorrelationId)
	if channel == nil {
		log.Errorf("[Client] CorrelationID -> %s does not exist in dispatcher", delivery.CorrelationId)
		return
	}
	select {
	case channel <- delivery:
	default:
		log.Errorf("[Client] Error in delivery for correlation %s", delivery.CorrelationId)
	}
}

func (c *Client) Call(routingKey string, req proto.Message, res proto.Message) error {
	correlation, err := uuid.NewV4()
	if err != nil {
		log.Error("[Client] Failed to create correlationId in client")
		return errors.New("client.call.uuid.error")
	}

	replyChannel := c.dispatcher.add(correlation.String())
	defer close(replyChannel)

	requestBody, err := proto.Marshal(req)
	if err != nil {
		log.Error("[Client] Failed to marshal request")
		return errors.New("client.call.marshal.error")
	}

	message := amqp.Publishing{
		CorrelationId: correlation.String(),
		Timestamp:     time.Now().UTC(),
		Body:          requestBody,
		ReplyTo:       c.replyTo,
	}

	err = c.connection.Publish(rabbit.Exchange, routingKey, message)
	if err != nil {
		log.Errorf("[Client] Failed to publish to %s", routingKey)
		return fmt.Errorf("client.call.publish.%s.error", routingKey)
	}

	select {
	case delivery := <-replyChannel:
		if err := proto.Unmarshal(delivery.Body, res); err != nil {
			return fmt.Errorf("client.unmarshal.%s-reply.error", routingKey)
		}
		return nil
	case <-time.After(10 * time.Second):
		log.Criticalf("[Client] Client timeout on delivery")
		return fmt.Errorf("client.call.timeout.%s.error", routingKey)
	}
}
<|endoftext|>


<commit_before>package client

import (
	"code.google.com/p/go-uuid/uuid"

	"github.com/dnaeon/gru/minion"
)

type MinionClient interface {
	// Gets the minion name
	GetName(u uuid.UUID) (string, error)

	// Gets the time the minion was last seen
	GetLastseen(u uuid.UUID) (int64, error)

	// Submits a task to a minion
	SubmitTask(u uuid.UUID, t minion.MinionTask) error
}
<commit_msg>GetClassifier() method is required for implementing MinionClient interface<commit_after>package client

import (
	"code.google.com/p/go-uuid/uuid"

	"github.com/dnaeon/gru/minion"
)

type MinionClient interface {
	// Gets the minion name
	GetName(u uuid.UUID) (string, error)

	// Gets the time the minion was last seen
	GetLastseen(u uuid.UUID) (int64, error)

	// Gets a classifier of a minion
	GetClassifier(u uuid.UUID, key string) (minion.MinionClassifier, error)

	// Submits a task to a minion
	SubmitTask(u uuid.UUID, t minion.MinionTask) error
}
<|endoftext|>


<commit_before>/*
Package client is a Go client for the Docker Engine API.

For more information about the Engine API, see the documentation:
https://docs.docker.com/engine/api/

Usage

You use the library by creating a client object and calling methods on it. The
client can be created either from environment variables with NewClientWithOpts(client.FromEnv),
or configured manually with NewClient().

For example, to list running containers (the equivalent of "docker ps"):

	package main

	import (
		"context"
		"fmt"

		"github.com/docker/docker/api/types"
		"github.com/docker/docker/client"
	)

	func main() {
		cli, err := client.NewClientWithOpts(client.FromEnv)
		if err != nil {
			panic(err)
		}

		containers, err := cli.ContainerList(context.Background(), types.ContainerListOptions{})
		if err != nil {
			panic(err)
		}

		for _, container := range containers {
			fmt.Printf("%s %s\n", container.ID[:10], container.Image)
		}
	}

*/
package client // import "github.com/docker/docker/client"

import (
	"context"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"path"
	"strings"

	"github.com/docker/docker/api"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/versions"
	"github.com/docker/go-connections/sockets"
	"github.com/pkg/errors"
)

// ErrRedirect is the error returned by checkRedirect when the request is non-GET.
var ErrRedirect = errors.New("unexpected redirect in response")

// Client is the API client that performs all operations
// against a docker server.
type Client struct {
	// scheme sets the scheme for the client
	scheme string
	// host holds the server address to connect to
	host string
	// proto holds the client protocol i.e. unix.
	proto string
	// addr holds the client address.
	addr string
	// basePath holds the path to prepend to the requests.
	basePath string
	// client used to send and receive http requests.
	client *http.Client
	// version of the server to talk to.
	version string
	// custom http headers configured by users.
	customHTTPHeaders map[string]string
	// manualOverride is set to true when the version was set by users.
	manualOverride bool

	// negotiateVersion indicates if the client should automatically negotiate
	// the API version to use when making requests. API version negotiation is
	// performed on the first request, after which negotiated is set to "true"
	// so that subsequent requests do not re-negotiate.
	negotiateVersion bool

	// negotiated indicates that API version negotiation took place
	negotiated bool
}

// CheckRedirect specifies the policy for dealing with redirect responses:
// If the request is non-GET return `ErrRedirect`. Otherwise use the last response.
//
// Go 1.8 changes behavior for HTTP redirects (specifically 301, 307, and 308) in the client .
// The Docker client (and by extension docker API client) can be made to send a request
// like POST /containers//start where what would normally be in the name section of the URL is empty.
// This triggers an HTTP 301 from the daemon.
// In go 1.8 this 301 will be converted to a GET request, and ends up getting a 404 from the daemon.
// This behavior change manifests in the client in that before the 301 was not followed and
// the client did not generate an error, but now results in a message like Error response from daemon: page not found.
func CheckRedirect(req *http.Request, via []*http.Request) error {
	if via[0].Method == http.MethodGet {
		return http.ErrUseLastResponse
	}
	return ErrRedirect
}

// NewClientWithOpts initializes a new API client with default values. It takes functors
// to modify values when creating it, like `NewClientWithOpts(WithVersion(…))`
// It also initializes the custom http headers to add to each request.
//
// It won't send any version information if the version number is empty. It is
// highly recommended that you set a version or your client may break if the
// server is upgraded.
func NewClientWithOpts(ops ...Opt) (*Client, error) {
	client, err := defaultHTTPClient(DefaultDockerHost)
	if err != nil {
		return nil, err
	}
	c := &Client{
		host:    DefaultDockerHost,
		version: api.DefaultVersion,
		client:  client,
		proto:   defaultProto,
		addr:    defaultAddr,
	}

	for _, op := range ops {
		if err := op(c); err != nil {
			return nil, err
		}
	}

	if _, ok := c.client.Transport.(http.RoundTripper); !ok {
		return nil, fmt.Errorf("unable to verify TLS configuration, invalid transport %v", c.client.Transport)
	}
	if c.scheme == "" {
		c.scheme = "http"

		tlsConfig := resolveTLSConfig(c.client.Transport)
		if tlsConfig != nil {
			// TODO(stevvooe): This isn't really the right way to write clients in Go.
			// `NewClient` should probably only take an `*http.Client` and work from there.
			// Unfortunately, the model of having a host-ish/url-thingy as the connection
			// string has us confusing protocol and transport layers. We continue doing
			// this to avoid breaking existing clients but this should be addressed.
			c.scheme = "https"
		}
	}

	return c, nil
}

func defaultHTTPClient(host string) (*http.Client, error) {
	url, err := ParseHostURL(host)
	if err != nil {
		return nil, err
	}
	transport := new(http.Transport)
	sockets.ConfigureTransport(transport, url.Scheme, url.Host)
	return &http.Client{
		Transport:     transport,
		CheckRedirect: CheckRedirect,
	}, nil
}

// Close the transport used by the client
func (cli *Client) Close() error {
	if t, ok := cli.client.Transport.(*http.Transport); ok {
		t.CloseIdleConnections()
	}
	return nil
}

// getAPIPath returns the versioned request path to call the api.
// It appends the query parameters to the path if they are not empty.
func (cli *Client) getAPIPath(ctx context.Context, p string, query url.Values) string {
	var apiPath string
	if cli.negotiateVersion && !cli.negotiated {
		cli.NegotiateAPIVersion(ctx)
	}
	if cli.version != "" {
		v := strings.TrimPrefix(cli.version, "v")
		apiPath = path.Join(cli.basePath, "/v"+v, p)
	} else {
		apiPath = path.Join(cli.basePath, p)
	}
	return (&url.URL{Path: apiPath, RawQuery: query.Encode()}).String()
}

// ClientVersion returns the API version used by this client.
func (cli *Client) ClientVersion() string {
	return cli.version
}

// NegotiateAPIVersion queries the API and updates the version to match the
// API version. Any errors are silently ignored. If a manual override is in place,
// either through the `DOCKER_API_VERSION` environment variable, or if the client
// was initialized with a fixed version (`opts.WithVersion(xx)`), no negotiation
// will be performed.
func (cli *Client) NegotiateAPIVersion(ctx context.Context) {
	if !cli.manualOverride {
		ping, _ := cli.Ping(ctx)
		cli.negotiateAPIVersionPing(ping)
	}
}

// NegotiateAPIVersionPing updates the client version to match the Ping.APIVersion
// if the ping version is less than the default version.  If a manual override is
// in place, either through the `DOCKER_API_VERSION` environment variable, or if
// the client was initialized with a fixed version (`opts.WithVersion(xx)`), no
// negotiation is performed.
func (cli *Client) NegotiateAPIVersionPing(p types.Ping) {
	if !cli.manualOverride {
		cli.negotiateAPIVersionPing(p)
	}
}

// negotiateAPIVersionPing queries the API and updates the version to match the
// API version. Any errors are silently ignored.
func (cli *Client) negotiateAPIVersionPing(p types.Ping) {
	// try the latest version before versioning headers existed
	if p.APIVersion == "" {
		p.APIVersion = "1.24"
	}

	// if the client is not initialized with a version, start with the latest supported version
	if cli.version == "" {
		cli.version = api.DefaultVersion
	}

	// if server version is lower than the client version, downgrade
	if versions.LessThan(p.APIVersion, cli.version) {
		cli.version = p.APIVersion
	}

	// Store the results, so that automatic API version negotiation (if enabled)
	// won't be performed on the next request.
	if cli.negotiateVersion {
		cli.negotiated = true
	}
}

// DaemonHost returns the host address used by the client
func (cli *Client) DaemonHost() string {
	return cli.host
}

// HTTPClient returns a copy of the HTTP client bound to the server
func (cli *Client) HTTPClient() *http.Client {
	c := *cli.client
	return &c
}

// ParseHostURL parses a url string, validates the string is a host url, and
// returns the parsed URL
func ParseHostURL(host string) (*url.URL, error) {
	protoAddrParts := strings.SplitN(host, "://", 2)
	if len(protoAddrParts) == 1 {
		return nil, fmt.Errorf("unable to parse docker host `%s`", host)
	}

	var basePath string
	proto, addr := protoAddrParts[0], protoAddrParts[1]
	if proto == "tcp" {
		parsed, err := url.Parse("tcp://" + addr)
		if err != nil {
			return nil, err
		}
		addr = parsed.Host
		basePath = parsed.Path
	}
	return &url.URL{
		Scheme: proto,
		Host:   addr,
		Path:   basePath,
	}, nil
}

// Dialer returns a dialer for a raw stream connection, with HTTP/1.1 header, that can be used for proxying the daemon connection.
// Used by `docker dial-stdio` (docker/cli#889).
func (cli *Client) Dialer() func(context.Context) (net.Conn, error) {
	return func(ctx context.Context) (net.Conn, error) {
		if transport, ok := cli.client.Transport.(*http.Transport); ok {
			if transport.DialContext != nil && transport.TLSClientConfig == nil {
				return transport.DialContext(ctx, cli.proto, cli.addr)
			}
		}
		return fallbackDial(cli.proto, cli.addr, resolveTLSConfig(cli.client.Transport))
	}
}
<commit_msg>client.NewClientWithOpts(): remove redundant type assertion (gosimple)<commit_after>/*
Package client is a Go client for the Docker Engine API.

For more information about the Engine API, see the documentation:
https://docs.docker.com/engine/api/

Usage

You use the library by creating a client object and calling methods on it. The
client can be created either from environment variables with NewClientWithOpts(client.FromEnv),
or configured manually with NewClient().

For example, to list running containers (the equivalent of "docker ps"):

	package main

	import (
		"context"
		"fmt"

		"github.com/docker/docker/api/types"
		"github.com/docker/docker/client"
	)

	func main() {
		cli, err := client.NewClientWithOpts(client.FromEnv)
		if err != nil {
			panic(err)
		}

		containers, err := cli.ContainerList(context.Background(), types.ContainerListOptions{})
		if err != nil {
			panic(err)
		}

		for _, container := range containers {
			fmt.Printf("%s %s\n", container.ID[:10], container.Image)
		}
	}

*/
package client // import "github.com/docker/docker/client"

import (
	"context"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"path"
	"strings"

	"github.com/docker/docker/api"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/versions"
	"github.com/docker/go-connections/sockets"
	"github.com/pkg/errors"
)

// ErrRedirect is the error returned by checkRedirect when the request is non-GET.
var ErrRedirect = errors.New("unexpected redirect in response")

// Client is the API client that performs all operations
// against a docker server.
type Client struct {
	// scheme sets the scheme for the client
	scheme string
	// host holds the server address to connect to
	host string
	// proto holds the client protocol i.e. unix.
	proto string
	// addr holds the client address.
	addr string
	// basePath holds the path to prepend to the requests.
	basePath string
	// client used to send and receive http requests.
	client *http.Client
	// version of the server to talk to.
	version string
	// custom http headers configured by users.
	customHTTPHeaders map[string]string
	// manualOverride is set to true when the version was set by users.
	manualOverride bool

	// negotiateVersion indicates if the client should automatically negotiate
	// the API version to use when making requests. API version negotiation is
	// performed on the first request, after which negotiated is set to "true"
	// so that subsequent requests do not re-negotiate.
	negotiateVersion bool

	// negotiated indicates that API version negotiation took place
	negotiated bool
}

// CheckRedirect specifies the policy for dealing with redirect responses:
// If the request is non-GET return `ErrRedirect`. Otherwise use the last response.
//
// Go 1.8 changes behavior for HTTP redirects (specifically 301, 307, and 308) in the client .
// The Docker client (and by extension docker API client) can be made to send a request
// like POST /containers//start where what would normally be in the name section of the URL is empty.
// This triggers an HTTP 301 from the daemon.
// In go 1.8 this 301 will be converted to a GET request, and ends up getting a 404 from the daemon.
// This behavior change manifests in the client in that before the 301 was not followed and
// the client did not generate an error, but now results in a message like Error response from daemon: page not found.
func CheckRedirect(req *http.Request, via []*http.Request) error {
	if via[0].Method == http.MethodGet {
		return http.ErrUseLastResponse
	}
	return ErrRedirect
}

// NewClientWithOpts initializes a new API client with default values. It takes functors
// to modify values when creating it, like `NewClientWithOpts(WithVersion(…))`
// It also initializes the custom http headers to add to each request.
//
// It won't send any version information if the version number is empty. It is
// highly recommended that you set a version or your client may break if the
// server is upgraded.
func NewClientWithOpts(ops ...Opt) (*Client, error) {
	client, err := defaultHTTPClient(DefaultDockerHost)
	if err != nil {
		return nil, err
	}
	c := &Client{
		host:    DefaultDockerHost,
		version: api.DefaultVersion,
		client:  client,
		proto:   defaultProto,
		addr:    defaultAddr,
	}

	for _, op := range ops {
		if err := op(c); err != nil {
			return nil, err
		}
	}

	if c.scheme == "" {
		c.scheme = "http"

		tlsConfig := resolveTLSConfig(c.client.Transport)
		if tlsConfig != nil {
			// TODO(stevvooe): This isn't really the right way to write clients in Go.
			// `NewClient` should probably only take an `*http.Client` and work from there.
			// Unfortunately, the model of having a host-ish/url-thingy as the connection
			// string has us confusing protocol and transport layers. We continue doing
			// this to avoid breaking existing clients but this should be addressed.
			c.scheme = "https"
		}
	}

	return c, nil
}

func defaultHTTPClient(host string) (*http.Client, error) {
	url, err := ParseHostURL(host)
	if err != nil {
		return nil, err
	}
	transport := new(http.Transport)
	sockets.ConfigureTransport(transport, url.Scheme, url.Host)
	return &http.Client{
		Transport:     transport,
		CheckRedirect: CheckRedirect,
	}, nil
}

// Close the transport used by the client
func (cli *Client) Close() error {
	if t, ok := cli.client.Transport.(*http.Transport); ok {
		t.CloseIdleConnections()
	}
	return nil
}

// getAPIPath returns the versioned request path to call the api.
// It appends the query parameters to the path if they are not empty.
func (cli *Client) getAPIPath(ctx context.Context, p string, query url.Values) string {
	var apiPath string
	if cli.negotiateVersion && !cli.negotiated {
		cli.NegotiateAPIVersion(ctx)
	}
	if cli.version != "" {
		v := strings.TrimPrefix(cli.version, "v")
		apiPath = path.Join(cli.basePath, "/v"+v, p)
	} else {
		apiPath = path.Join(cli.basePath, p)
	}
	return (&url.URL{Path: apiPath, RawQuery: query.Encode()}).String()
}

// ClientVersion returns the API version used by this client.
func (cli *Client) ClientVersion() string {
	return cli.version
}

// NegotiateAPIVersion queries the API and updates the version to match the
// API version. Any errors are silently ignored. If a manual override is in place,
// either through the `DOCKER_API_VERSION` environment variable, or if the client
// was initialized with a fixed version (`opts.WithVersion(xx)`), no negotiation
// will be performed.
func (cli *Client) NegotiateAPIVersion(ctx context.Context) {
	if !cli.manualOverride {
		ping, _ := cli.Ping(ctx)
		cli.negotiateAPIVersionPing(ping)
	}
}

// NegotiateAPIVersionPing updates the client version to match the Ping.APIVersion
// if the ping version is less than the default version.  If a manual override is
// in place, either through the `DOCKER_API_VERSION` environment variable, or if
// the client was initialized with a fixed version (`opts.WithVersion(xx)`), no
// negotiation is performed.
func (cli *Client) NegotiateAPIVersionPing(p types.Ping) {
	if !cli.manualOverride {
		cli.negotiateAPIVersionPing(p)
	}
}

// negotiateAPIVersionPing queries the API and updates the version to match the
// API version. Any errors are silently ignored.
func (cli *Client) negotiateAPIVersionPing(p types.Ping) {
	// try the latest version before versioning headers existed
	if p.APIVersion == "" {
		p.APIVersion = "1.24"
	}

	// if the client is not initialized with a version, start with the latest supported version
	if cli.version == "" {
		cli.version = api.DefaultVersion
	}

	// if server version is lower than the client version, downgrade
	if versions.LessThan(p.APIVersion, cli.version) {
		cli.version = p.APIVersion
	}

	// Store the results, so that automatic API version negotiation (if enabled)
	// won't be performed on the next request.
	if cli.negotiateVersion {
		cli.negotiated = true
	}
}

// DaemonHost returns the host address used by the client
func (cli *Client) DaemonHost() string {
	return cli.host
}

// HTTPClient returns a copy of the HTTP client bound to the server
func (cli *Client) HTTPClient() *http.Client {
	c := *cli.client
	return &c
}

// ParseHostURL parses a url string, validates the string is a host url, and
// returns the parsed URL
func ParseHostURL(host string) (*url.URL, error) {
	protoAddrParts := strings.SplitN(host, "://", 2)
	if len(protoAddrParts) == 1 {
		return nil, fmt.Errorf("unable to parse docker host `%s`", host)
	}

	var basePath string
	proto, addr := protoAddrParts[0], protoAddrParts[1]
	if proto == "tcp" {
		parsed, err := url.Parse("tcp://" + addr)
		if err != nil {
			return nil, err
		}
		addr = parsed.Host
		basePath = parsed.Path
	}
	return &url.URL{
		Scheme: proto,
		Host:   addr,
		Path:   basePath,
	}, nil
}

// Dialer returns a dialer for a raw stream connection, with HTTP/1.1 header, that can be used for proxying the daemon connection.
// Used by `docker dial-stdio` (docker/cli#889).
func (cli *Client) Dialer() func(context.Context) (net.Conn, error) {
	return func(ctx context.Context) (net.Conn, error) {
		if transport, ok := cli.client.Transport.(*http.Transport); ok {
			if transport.DialContext != nil && transport.TLSClientConfig == nil {
				return transport.DialContext(ctx, cli.proto, cli.addr)
			}
		}
		return fallbackDial(cli.proto, cli.addr, resolveTLSConfig(cli.client.Transport))
	}
}
<|endoftext|>


<commit_before>// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0

package kio

import (
	"encoding/json"
	"io"
	"path/filepath"
	"sigs.k8s.io/kustomize/kyaml/errors"
	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
	"sigs.k8s.io/kustomize/kyaml/yaml"
)

// ByteWriter writes ResourceNodes to bytes. Generally YAML encoding will be used but in the special
// case of writing a single, bare yaml.RNode that has a kioutil.PathAnnotation indicating that the
// target is a JSON file JSON encoding is used. See shouldJSONEncodeSingleBareNode below for more
// information.
type ByteWriter struct {
	// Writer is where ResourceNodes are encoded.
	Writer io.Writer

	// KeepReaderAnnotations if set will keep the Reader specific annotations when writing
	// the Resources, otherwise they will be cleared.
	KeepReaderAnnotations bool

	// ClearAnnotations is a list of annotations to clear when writing the Resources.
	ClearAnnotations []string

	// Style is a style that is set on the Resource Node Document.
	Style yaml.Style

	// FunctionConfig is the function config for an ResourceList.  If non-nil
	// wrap the results in an ResourceList.
	FunctionConfig *yaml.RNode

	Results *yaml.RNode

	// WrappingKind if set will cause ByteWriter to wrap the Resources in
	// an 'items' field in this kind.  e.g. if WrappingKind is 'List',
	// ByteWriter will wrap the Resources in a List .items field.
	WrappingKind string

	// WrappingAPIVersion is the apiVersion for WrappingKind
	WrappingAPIVersion string

	// Sort if set, will cause ByteWriter to sort the the nodes before writing them.
	Sort bool
}

var _ Writer = ByteWriter{}

func (w ByteWriter) Write(nodes []*yaml.RNode) error {
	yaml.DoSerializationHacksOnNodes(nodes)
	if w.Sort {
		if err := kioutil.SortNodes(nodes); err != nil {
			return errors.Wrap(err)
		}
	}

	// Even though we use the this value further down we must check this before removing annotations
	jsonEncodeSingleBareNode := w.shouldJSONEncodeSingleBareNode(nodes)

	for i := range nodes {
		// clean resources by removing annotations set by the Reader
		if !w.KeepReaderAnnotations {
			_, err := nodes[i].Pipe(yaml.ClearAnnotation(kioutil.IndexAnnotation))
			if err != nil {
				return errors.Wrap(err)
			}
		}
		for _, a := range w.ClearAnnotations {
			_, err := nodes[i].Pipe(yaml.ClearAnnotation(a))
			if err != nil {
				return errors.Wrap(err)
			}
		}

		if err := yaml.ClearEmptyAnnotations(nodes[i]); err != nil {
			return err
		}

		if w.Style != 0 {
			nodes[i].YNode().Style = w.Style
		}
	}

	if jsonEncodeSingleBareNode {
		encoder := json.NewEncoder(w.Writer)
		encoder.SetIndent("", "  ")
		return errors.Wrap(encoder.Encode(nodes[0]))
	}

	encoder := yaml.NewEncoder(w.Writer)
	defer encoder.Close()
	// don't wrap the elements
	if w.WrappingKind == "" {
		for i := range nodes {
			if err := encoder.Encode(nodes[i].Document()); err != nil {
				return err
			}
		}
		return nil
	}
	// wrap the elements in a list
	items := &yaml.Node{Kind: yaml.SequenceNode}
	list := &yaml.Node{
		Kind:  yaml.MappingNode,
		Style: w.Style,
		Content: []*yaml.Node{
			{Kind: yaml.ScalarNode, Value: "apiVersion"},
			{Kind: yaml.ScalarNode, Value: w.WrappingAPIVersion},
			{Kind: yaml.ScalarNode, Value: "kind"},
			{Kind: yaml.ScalarNode, Value: w.WrappingKind},
			{Kind: yaml.ScalarNode, Value: "items"}, items,
		}}
	if w.FunctionConfig != nil {
		list.Content = append(list.Content,
			&yaml.Node{Kind: yaml.ScalarNode, Value: "functionConfig"},
			w.FunctionConfig.YNode())
	}
	if w.Results != nil {
		list.Content = append(list.Content,
			&yaml.Node{Kind: yaml.ScalarNode, Value: "results"},
			w.Results.YNode())
	}
	doc := &yaml.Node{
		Kind:    yaml.DocumentNode,
		Content: []*yaml.Node{list}}
	for i := range nodes {
		items.Content = append(items.Content, nodes[i].YNode())
	}
	err := encoder.Encode(doc)
	yaml.UndoSerializationHacksOnNodes(nodes)
	return err
}

// shouldJSONEncodeSingleBareNode determines if nodes contain a single node that should not be
// wrapped and has a JSON file extension, which in turn means that the node should be JSON encoded.
// Note 1: this must be checked before any annotations to avoid losing information about the target
//         filename extension.
// Note 2: JSON encoding should only be used for single, unwrapped nodes because multiple unwrapped
//         nodes cannot be represented in JSON (no multi doc support). Furthermore, the typical use
//         cases for wrapping nodes would likely not include later writing the whole wrapper to a
//         .json file, i.e. there is no point risking any edge case information loss e.g. comments
//         disappearing, that could come from JSON encoding the whole wrapper just to ensure that
//         one (or all nodes) can be read as JSON.
func (w ByteWriter) shouldJSONEncodeSingleBareNode(nodes []*yaml.RNode) bool {
	if w.WrappingKind == "" && len(nodes) == 1 {
		if path, _, _ := kioutil.GetFileAnnotations(nodes[0]); path != "" {
			filename := filepath.Base(path)
			for _, glob := range JSONMatch {
				if match, _ := filepath.Match(glob, filename); match {
					return true
				}
			}
		}
	}
	return false
}<commit_msg>Update byteio_writer.go<commit_after>// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0

package kio

import (
	"encoding/json"
	"io"
	"path/filepath"
	"sigs.k8s.io/kustomize/kyaml/errors"
	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
	"sigs.k8s.io/kustomize/kyaml/yaml"
)

// ByteWriter writes ResourceNodes to bytes. Generally YAML encoding will be used but in the special
// case of writing a single, bare yaml.RNode that has a kioutil.PathAnnotation indicating that the
// target is a JSON file JSON encoding is used. See shouldJSONEncodeSingleBareNode below for more
// information.
type ByteWriter struct {
	// Writer is where ResourceNodes are encoded.
	Writer io.Writer

	// KeepReaderAnnotations if set will keep the Reader specific annotations when writing
	// the Resources, otherwise they will be cleared.
	KeepReaderAnnotations bool

	// ClearAnnotations is a list of annotations to clear when writing the Resources.
	ClearAnnotations []string

	// Style is a style that is set on the Resource Node Document.
	Style yaml.Style

	// FunctionConfig is the function config for an ResourceList.  If non-nil
	// wrap the results in an ResourceList.
	FunctionConfig *yaml.RNode

	Results *yaml.RNode

	// WrappingKind if set will cause ByteWriter to wrap the Resources in
	// an 'items' field in this kind.  e.g. if WrappingKind is 'List',
	// ByteWriter will wrap the Resources in a List .items field.
	WrappingKind string

	// WrappingAPIVersion is the apiVersion for WrappingKind
	WrappingAPIVersion string

	// Sort if set, will cause ByteWriter to sort the the nodes before writing them.
	Sort bool
}

var _ Writer = ByteWriter{}

func (w ByteWriter) Write(nodes []*yaml.RNode) error {
	yaml.DoSerializationHacksOnNodes(nodes)
	if w.Sort {
		if err := kioutil.SortNodes(nodes); err != nil {
			return errors.Wrap(err)
		}
	}

	// Even though we use the this value further down we must check this before removing annotations
	jsonEncodeSingleBareNode := w.shouldJSONEncodeSingleBareNode(nodes)

	for i := range nodes {
		// clean resources by removing annotations set by the Reader
		if !w.KeepReaderAnnotations {
			_, err := nodes[i].Pipe(yaml.ClearAnnotation(kioutil.IndexAnnotation))
			if err != nil {
				return errors.Wrap(err)
			}
		}
		for _, a := range w.ClearAnnotations {
			_, err := nodes[i].Pipe(yaml.ClearAnnotation(a))
			if err != nil {
				return errors.Wrap(err)
			}
		}

		if err := yaml.ClearEmptyAnnotations(nodes[i]); err != nil {
			return err
		}

		if w.Style != 0 {
			nodes[i].YNode().Style = w.Style
		}
	}

	if jsonEncodeSingleBareNode {
		encoder := json.NewEncoder(w.Writer)
		encoder.SetIndent("", "  ")
		return errors.Wrap(encoder.Encode(nodes[0]))
	}

	encoder := yaml.NewEncoder(w.Writer)
	defer encoder.Close()
	// don't wrap the elements
	if w.WrappingKind == "" {
		for i := range nodes {
			if err := encoder.Encode(nodes[i].Document()); err != nil {
				return err
			}
		}
		return nil
	}
	// wrap the elements in a list
	items := &yaml.Node{Kind: yaml.SequenceNode}
	list := &yaml.Node{
		Kind:  yaml.MappingNode,
		Style: w.Style,
		Content: []*yaml.Node{
			{Kind: yaml.ScalarNode, Value: "apiVersion"},
			{Kind: yaml.ScalarNode, Value: w.WrappingAPIVersion},
			{Kind: yaml.ScalarNode, Value: "kind"},
			{Kind: yaml.ScalarNode, Value: w.WrappingKind},
			{Kind: yaml.ScalarNode, Value: "items"}, items,
		}}
	if w.FunctionConfig != nil {
		list.Content = append(list.Content,
			&yaml.Node{Kind: yaml.ScalarNode, Value: "functionConfig"},
			w.FunctionConfig.YNode())
	}
	if w.Results != nil {
		list.Content = append(list.Content,
			&yaml.Node{Kind: yaml.ScalarNode, Value: "results"},
			w.Results.YNode())
	}
	doc := &yaml.Node{
		Kind:    yaml.DocumentNode,
		Content: []*yaml.Node{list}}
	for i := range nodes {
		items.Content = append(items.Content, nodes[i].YNode())
	}
	err := encoder.Encode(doc)
	yaml.UndoSerializationHacksOnNodes(nodes)
	return err
}

// shouldJSONEncodeSingleBareNode determines if nodes contain a single node that should not be
// wrapped and has a JSON file extension, which in turn means that the node should be JSON encoded.
// Note 1: this must be checked before any annotations to avoid losing information about the target
//         filename extension.
// Note 2: JSON encoding should only be used for single, unwrapped nodes because multiple unwrapped
//         nodes cannot be represented in JSON (no multi doc support). Furthermore, the typical use
//         cases for wrapping nodes would likely not include later writing the whole wrapper to a
//         .json file, i.e. there is no point risking any edge case information loss e.g. comments
//         disappearing, that could come from JSON encoding the whole wrapper just to ensure that
//         one (or all nodes) can be read as JSON.
func (w ByteWriter) shouldJSONEncodeSingleBareNode(nodes []*yaml.RNode) bool {
	if w.WrappingKind == "" && len(nodes) == 1 {
		if path, _, _ := kioutil.GetFileAnnotations(nodes[0]); path != "" {
			filename := filepath.Base(path)
			for _, glob := range JSONMatch {
				if match, _ := filepath.Match(glob, filename); match {
					return true
				}
			}
		}
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/concourse/atc"
	"github.com/jessevdk/go-flags"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
)

type GardenBackend struct{}

func (cmd WorkerCommand) lessenRequirements(command *flags.Command) {}

func (cmd *WorkerCommand) gardenRunner(logger lager.Logger, args []string) (atc.Worker, ifrit.Runner, error) {
	err := cmd.checkRoot()
	if err != nil {
		return atc.Worker{}, nil, err
	}

	return cmd.houdiniRunner(logger, "solaris")
}

func (cmd *WorkerCommand) baggageclaimRunner(logger lager.Logger) (ifrit.Runner, error) {
	return cmd.naiveBaggageclaimRunner(logger)
}
<commit_msg>Sync up Solaris code with mainline changes<commit_after>package main

import (
	"code.cloudfoundry.org/lager"
	"github.com/concourse/atc"
	"github.com/jessevdk/go-flags"
	"github.com/tedsuo/ifrit"
)

type GardenBackend struct{}

func (cmd WorkerCommand) lessenRequirements(command *flags.Command) {
	command.FindOptionByLongName("baggageclaim-volumes").Required = false
}

func (cmd *WorkerCommand) gardenRunner(logger lager.Logger, args []string) (atc.Worker, ifrit.Runner, error) {
	return cmd.houdiniRunner(logger, "solaris")
}

func (cmd *WorkerCommand) baggageclaimRunner(logger lager.Logger) (ifrit.Runner, error) {
	return cmd.naiveBaggageclaimRunner(logger)
}
<|endoftext|>