<commit_before>package self_test

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"math"
	mrand "math/rand"
	"net"
	"sync/atomic"
	"time"

	quic "github.com/lucas-clemente/quic-go"
	quicproxy "github.com/lucas-clemente/quic-go/integrationtests/tools/proxy"
	"github.com/lucas-clemente/quic-go/integrationtests/tools/testserver"
	"github.com/lucas-clemente/quic-go/internal/protocol"
	"github.com/lucas-clemente/quic-go/internal/wire"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("MITM test", func() {
	for _, v := range protocol.SupportedVersions {
		version := v

		Context(fmt.Sprintf("with QUIC version %s", version), func() {
			const connIDLen = 6 // explicitly set the connection ID length, so the proxy can parse it

			var (
				proxy                  *quicproxy.QuicProxy
				serverConn, clientConn *net.UDPConn
				serverSess             quic.Session
				serverConfig           *quic.Config
			)

			startServerAndProxy := func(delayCb quicproxy.DelayCallback, dropCb quicproxy.DropCallback) {
				addr, err := net.ResolveUDPAddr("udp", "localhost:0")
				Expect(err).ToNot(HaveOccurred())
				serverConn, err = net.ListenUDP("udp", addr)
				Expect(err).ToNot(HaveOccurred())
				ln, err := quic.Listen(serverConn, getTLSConfig(), serverConfig)
				Expect(err).ToNot(HaveOccurred())
				go func() {
					defer GinkgoRecover()
					var err error
					serverSess, err = ln.Accept(context.Background())
					Expect(err).ToNot(HaveOccurred())
					str, err := serverSess.OpenUniStream()
					Expect(err).ToNot(HaveOccurred())
					_, err = str.Write(testserver.PRData)
					Expect(err).ToNot(HaveOccurred())
					Expect(str.Close()).To(Succeed())
				}()
				serverPort := ln.Addr().(*net.UDPAddr).Port
				proxy, err = quicproxy.NewQuicProxy("localhost:0", &quicproxy.Opts{
					RemoteAddr:  fmt.Sprintf("localhost:%d", serverPort),
					DelayPacket: delayCb,
					DropPacket:  dropCb,
				})
				Expect(err).ToNot(HaveOccurred())
			}

			BeforeEach(func() {
				serverConfig = &quic.Config{
					Versions:           []protocol.VersionNumber{version},
					ConnectionIDLength: connIDLen,
				}
				addr, err := net.ResolveUDPAddr("udp", "localhost:0")
				Expect(err).ToNot(HaveOccurred())
				clientConn, err = net.ListenUDP("udp", addr)
				Expect(err).ToNot(HaveOccurred())
			})

			AfterEach(func() {
				Eventually(serverSess.Context().Done()).Should(BeClosed())
				// Test shutdown is tricky due to the proxy. Just wait for a bit.
				time.Sleep(50 * time.Millisecond)
				Expect(clientConn.Close()).To(Succeed())
				Expect(serverConn.Close()).To(Succeed())
				Expect(proxy.Close()).To(Succeed())
			})

			Context("injecting invalid packets", func() {
				const rtt = 20 * time.Millisecond

				sendRandomPacketsOfSameType := func(conn net.PacketConn, remoteAddr net.Addr, raw []byte) {
					defer GinkgoRecover()
					hdr, _, _, err := wire.ParsePacket(raw, connIDLen)
					Expect(err).ToNot(HaveOccurred())
					replyHdr := &wire.ExtendedHeader{
						Header: wire.Header{
							IsLongHeader:     hdr.IsLongHeader,
							DestConnectionID: hdr.DestConnectionID,
							SrcConnectionID:  hdr.SrcConnectionID,
							Type:             hdr.Type,
							Version:          hdr.Version,
						},
						PacketNumber:    protocol.PacketNumber(mrand.Int31n(math.MaxInt32 / 4)),
						PacketNumberLen: protocol.PacketNumberLen(mrand.Int31n(4) + 1),
					}

					const numPackets = 10
					ticker := time.NewTicker(rtt / numPackets)
					for i := 0; i < numPackets; i++ {
						payloadLen := mrand.Int31n(100)
						replyHdr.Length = protocol.ByteCount(mrand.Int31n(payloadLen + 1))
						buf := &bytes.Buffer{}
						Expect(replyHdr.Write(buf, v)).To(Succeed())
						b := make([]byte, payloadLen)
						mrand.Read(b)
						buf.Write(b)
						if _, err := conn.WriteTo(buf.Bytes(), remoteAddr); err != nil {
							return
						}
						<-ticker.C
					}
				}

				runTest := func(delayCb quicproxy.DelayCallback) {
					startServerAndProxy(delayCb, nil)
					raddr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("localhost:%d", proxy.LocalPort()))
					Expect(err).ToNot(HaveOccurred())
					sess, err := quic.Dial(
						clientConn,
						raddr,
						fmt.Sprintf("localhost:%d", proxy.LocalPort()),
						getTLSClientConfig(),
						&quic.Config{
							Versions:           []protocol.VersionNumber{version},
							ConnectionIDLength: connIDLen,
						},
					)
					Expect(err).ToNot(HaveOccurred())
					str, err := sess.AcceptUniStream(context.Background())
					Expect(err).ToNot(HaveOccurred())
					data, err := ioutil.ReadAll(str)
					Expect(err).ToNot(HaveOccurred())
					Expect(data).To(Equal(testserver.PRData))
					Expect(sess.Close()).To(Succeed())
				}

				It("downloads a message when the packets are injected towards the server", func() {
					delayCb := func(dir quicproxy.Direction, raw []byte) time.Duration {
						if dir == quicproxy.DirectionIncoming {
							defer GinkgoRecover()
							go sendRandomPacketsOfSameType(clientConn, serverConn.LocalAddr(), raw)
						}
						return rtt / 2
					}
					runTest(delayCb)
				})

				It("downloads a message when the packets are injected towards the client", func() {
					delayCb := func(dir quicproxy.Direction, raw []byte) time.Duration {
						if dir == quicproxy.DirectionOutgoing {
							defer GinkgoRecover()
							go sendRandomPacketsOfSameType(serverConn, clientConn.LocalAddr(), raw)
						}
						return rtt / 2
					}
					runTest(delayCb)
				})
			})

			runTest := func(dropCb quicproxy.DropCallback) {
				startServerAndProxy(nil, dropCb)
				raddr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("localhost:%d", proxy.LocalPort()))
				Expect(err).ToNot(HaveOccurred())
				sess, err := quic.Dial(
					clientConn,
					raddr,
					fmt.Sprintf("localhost:%d", proxy.LocalPort()),
					getTLSClientConfig(),
					&quic.Config{
						Versions:           []protocol.VersionNumber{version},
						ConnectionIDLength: connIDLen,
					},
				)
				Expect(err).ToNot(HaveOccurred())
				str, err := sess.AcceptUniStream(context.Background())
				Expect(err).ToNot(HaveOccurred())
				data, err := ioutil.ReadAll(str)
				Expect(err).ToNot(HaveOccurred())
				Expect(data).To(Equal(testserver.PRData))
				Expect(sess.Close()).To(Succeed())
			}

			Context("duplicating packets", func() {
				It("downloads a message when packets are duplicated towards the server", func() {
					dropCb := func(dir quicproxy.Direction, raw []byte) bool {
						defer GinkgoRecover()
						if dir == quicproxy.DirectionIncoming {
							_, err := clientConn.WriteTo(raw, serverConn.LocalAddr())
							Expect(err).ToNot(HaveOccurred())
						}
						return false
					}
					runTest(dropCb)
				})

				It("downloads a message when packets are duplicated towards the client", func() {
					dropCb := func(dir quicproxy.Direction, raw []byte) bool {
						defer GinkgoRecover()
						if dir == quicproxy.DirectionOutgoing {
							_, err := serverConn.WriteTo(raw, clientConn.LocalAddr())
							Expect(err).ToNot(HaveOccurred())
						}
						return false
					}
					runTest(dropCb)
				})
			})

			Context("corrupting packets", func() {
				const interval = 10 // corrupt every 10th packet (stochastically)
				const idleTimeout = time.Second

				var numCorrupted int32

				BeforeEach(func() {
					numCorrupted = 0
					serverConfig.IdleTimeout = idleTimeout
				})

				AfterEach(func() {
					num := atomic.LoadInt32(&numCorrupted)
					fmt.Fprintf(GinkgoWriter, "Corrupted %d packets.", num)
					Expect(num).To(BeNumerically(">=", 1))
					// If the packet containing the CONNECTION_CLOSE is corrupted,
					// we have to wait for the session to time out.
					Eventually(serverSess.Context().Done(), 3*idleTimeout).Should(BeClosed())
				})

				It("downloads a message when packet are corrupted towards the server", func() {
					dropCb := func(dir quicproxy.Direction, raw []byte) bool {
						defer GinkgoRecover()
						if dir == quicproxy.DirectionIncoming && mrand.Intn(interval) == 0 {
							pos := mrand.Intn(len(raw))
							raw[pos] = byte(mrand.Intn(256))
							_, err := clientConn.WriteTo(raw, serverConn.LocalAddr())
							Expect(err).ToNot(HaveOccurred())
							atomic.AddInt32(&numCorrupted, 1)
							return true
						}
						return false
					}
					runTest(dropCb)
				})

				It("downloads a message when packet are corrupted towards the client", func() {
					dropCb := func(dir quicproxy.Direction, raw []byte) bool {
						defer GinkgoRecover()
						isRetry := raw[0]&0xc0 == 0xc0 // don't corrupt Retry packets
						if dir == quicproxy.DirectionOutgoing && mrand.Intn(interval) == 0 && !isRetry {
							pos := mrand.Intn(len(raw))
							raw[pos] = byte(mrand.Intn(256))
							_, err := serverConn.WriteTo(raw, clientConn.LocalAddr())
							Expect(err).ToNot(HaveOccurred())
							atomic.AddInt32(&numCorrupted, 1)
							return true
						}
						return false
					}
					runTest(dropCb)
				})
			})
		})
	}
})
<commit_msg>restructured contexts in mitm tests<commit_after>package self_test

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"math"
	mrand "math/rand"
	"net"
	"sync/atomic"
	"time"

	quic "github.com/lucas-clemente/quic-go"
	quicproxy "github.com/lucas-clemente/quic-go/integrationtests/tools/proxy"
	"github.com/lucas-clemente/quic-go/integrationtests/tools/testserver"
	"github.com/lucas-clemente/quic-go/internal/protocol"
	"github.com/lucas-clemente/quic-go/internal/wire"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("MITM test", func() {
	for _, v := range protocol.SupportedVersions {
		version := v

		Context(fmt.Sprintf("with QUIC version %s", version), func() {
			const connIDLen = 6 // explicitly set the connection ID length, so the proxy can parse it

			var (
				proxy                  *quicproxy.QuicProxy
				serverConn, clientConn *net.UDPConn
				serverSess             quic.Session
				serverConfig           *quic.Config
			)

			startServerAndProxy := func(delayCb quicproxy.DelayCallback, dropCb quicproxy.DropCallback) {
				addr, err := net.ResolveUDPAddr("udp", "localhost:0")
				Expect(err).ToNot(HaveOccurred())
				serverConn, err = net.ListenUDP("udp", addr)
				Expect(err).ToNot(HaveOccurred())
				ln, err := quic.Listen(serverConn, getTLSConfig(), serverConfig)
				Expect(err).ToNot(HaveOccurred())
				go func() {
					defer GinkgoRecover()
					var err error
					serverSess, err = ln.Accept(context.Background())
					Expect(err).ToNot(HaveOccurred())
					str, err := serverSess.OpenUniStream()
					Expect(err).ToNot(HaveOccurred())
					_, err = str.Write(testserver.PRData)
					Expect(err).ToNot(HaveOccurred())
					Expect(str.Close()).To(Succeed())
				}()
				serverPort := ln.Addr().(*net.UDPAddr).Port
				proxy, err = quicproxy.NewQuicProxy("localhost:0", &quicproxy.Opts{
					RemoteAddr:  fmt.Sprintf("localhost:%d", serverPort),
					DelayPacket: delayCb,
					DropPacket:  dropCb,
				})
				Expect(err).ToNot(HaveOccurred())
			}

			BeforeEach(func() {
				serverConfig = &quic.Config{
					Versions:           []protocol.VersionNumber{version},
					ConnectionIDLength: connIDLen,
				}
				addr, err := net.ResolveUDPAddr("udp", "localhost:0")
				Expect(err).ToNot(HaveOccurred())
				clientConn, err = net.ListenUDP("udp", addr)
				Expect(err).ToNot(HaveOccurred())
			})

			Context("unsuccessful attacks", func() {
				AfterEach(func() {
					Eventually(serverSess.Context().Done()).Should(BeClosed())
					// Test shutdown is tricky due to the proxy. Just wait for a bit.
					time.Sleep(50 * time.Millisecond)
					Expect(clientConn.Close()).To(Succeed())
					Expect(serverConn.Close()).To(Succeed())
					Expect(proxy.Close()).To(Succeed())
				})

				Context("injecting invalid packets", func() {
					const rtt = 20 * time.Millisecond

					sendRandomPacketsOfSameType := func(conn net.PacketConn, remoteAddr net.Addr, raw []byte) {
						defer GinkgoRecover()
						hdr, _, _, err := wire.ParsePacket(raw, connIDLen)
						Expect(err).ToNot(HaveOccurred())
						replyHdr := &wire.ExtendedHeader{
							Header: wire.Header{
								IsLongHeader:     hdr.IsLongHeader,
								DestConnectionID: hdr.DestConnectionID,
								SrcConnectionID:  hdr.SrcConnectionID,
								Type:             hdr.Type,
								Version:          hdr.Version,
							},
							PacketNumber:    protocol.PacketNumber(mrand.Int31n(math.MaxInt32 / 4)),
							PacketNumberLen: protocol.PacketNumberLen(mrand.Int31n(4) + 1),
						}

						const numPackets = 10
						ticker := time.NewTicker(rtt / numPackets)
						for i := 0; i < numPackets; i++ {
							payloadLen := mrand.Int31n(100)
							replyHdr.Length = protocol.ByteCount(mrand.Int31n(payloadLen + 1))
							buf := &bytes.Buffer{}
							Expect(replyHdr.Write(buf, v)).To(Succeed())
							b := make([]byte, payloadLen)
							mrand.Read(b)
							buf.Write(b)
							if _, err := conn.WriteTo(buf.Bytes(), remoteAddr); err != nil {
								return
							}
							<-ticker.C
						}
					}

					runTest := func(delayCb quicproxy.DelayCallback) {
						startServerAndProxy(delayCb, nil)
						raddr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("localhost:%d", proxy.LocalPort()))
						Expect(err).ToNot(HaveOccurred())
						sess, err := quic.Dial(
							clientConn,
							raddr,
							fmt.Sprintf("localhost:%d", proxy.LocalPort()),
							getTLSClientConfig(),
							&quic.Config{
								Versions:           []protocol.VersionNumber{version},
								ConnectionIDLength: connIDLen,
							},
						)
						Expect(err).ToNot(HaveOccurred())
						str, err := sess.AcceptUniStream(context.Background())
						Expect(err).ToNot(HaveOccurred())
						data, err := ioutil.ReadAll(str)
						Expect(err).ToNot(HaveOccurred())
						Expect(data).To(Equal(testserver.PRData))
						Expect(sess.Close()).To(Succeed())
					}

					It("downloads a message when the packets are injected towards the server", func() {
						delayCb := func(dir quicproxy.Direction, raw []byte) time.Duration {
							if dir == quicproxy.DirectionIncoming {
								defer GinkgoRecover()
								go sendRandomPacketsOfSameType(clientConn, serverConn.LocalAddr(), raw)
							}
							return rtt / 2
						}
						runTest(delayCb)
					})

					It("downloads a message when the packets are injected towards the client", func() {
						delayCb := func(dir quicproxy.Direction, raw []byte) time.Duration {
							if dir == quicproxy.DirectionOutgoing {
								defer GinkgoRecover()
								go sendRandomPacketsOfSameType(serverConn, clientConn.LocalAddr(), raw)
							}
							return rtt / 2
						}
						runTest(delayCb)
					})
				})

				runTest := func(dropCb quicproxy.DropCallback) {
					startServerAndProxy(nil, dropCb)
					raddr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("localhost:%d", proxy.LocalPort()))
					Expect(err).ToNot(HaveOccurred())
					sess, err := quic.Dial(
						clientConn,
						raddr,
						fmt.Sprintf("localhost:%d", proxy.LocalPort()),
						getTLSClientConfig(),
						&quic.Config{
							Versions:           []protocol.VersionNumber{version},
							ConnectionIDLength: connIDLen,
						},
					)
					Expect(err).ToNot(HaveOccurred())
					str, err := sess.AcceptUniStream(context.Background())
					Expect(err).ToNot(HaveOccurred())
					data, err := ioutil.ReadAll(str)
					Expect(err).ToNot(HaveOccurred())
					Expect(data).To(Equal(testserver.PRData))
					Expect(sess.Close()).To(Succeed())
				}

				Context("duplicating packets", func() {
					It("downloads a message when packets are duplicated towards the server", func() {
						dropCb := func(dir quicproxy.Direction, raw []byte) bool {
							defer GinkgoRecover()
							if dir == quicproxy.DirectionIncoming {
								_, err := clientConn.WriteTo(raw, serverConn.LocalAddr())
								Expect(err).ToNot(HaveOccurred())
							}
							return false
						}
						runTest(dropCb)
					})

					It("downloads a message when packets are duplicated towards the client", func() {
						dropCb := func(dir quicproxy.Direction, raw []byte) bool {
							defer GinkgoRecover()
							if dir == quicproxy.DirectionOutgoing {
								_, err := serverConn.WriteTo(raw, clientConn.LocalAddr())
								Expect(err).ToNot(HaveOccurred())
							}
							return false
						}
						runTest(dropCb)
					})
				})

				Context("corrupting packets", func() {
					const interval = 10 // corrupt every 10th packet (stochastically)
					const idleTimeout = time.Second

					var numCorrupted int32

					BeforeEach(func() {
						numCorrupted = 0
						serverConfig.IdleTimeout = idleTimeout
					})

					AfterEach(func() {
						num := atomic.LoadInt32(&numCorrupted)
						fmt.Fprintf(GinkgoWriter, "Corrupted %d packets.", num)
						Expect(num).To(BeNumerically(">=", 1))
						// If the packet containing the CONNECTION_CLOSE is corrupted,
						// we have to wait for the session to time out.
						Eventually(serverSess.Context().Done(), 3*idleTimeout).Should(BeClosed())
					})

					It("downloads a message when packet are corrupted towards the server", func() {
						dropCb := func(dir quicproxy.Direction, raw []byte) bool {
							defer GinkgoRecover()
							if dir == quicproxy.DirectionIncoming && mrand.Intn(interval) == 0 {
								pos := mrand.Intn(len(raw))
								raw[pos] = byte(mrand.Intn(256))
								_, err := clientConn.WriteTo(raw, serverConn.LocalAddr())
								Expect(err).ToNot(HaveOccurred())
								atomic.AddInt32(&numCorrupted, 1)
								return true
							}
							return false
						}
						runTest(dropCb)
					})

					It("downloads a message when packet are corrupted towards the client", func() {
						dropCb := func(dir quicproxy.Direction, raw []byte) bool {
							defer GinkgoRecover()
							isRetry := raw[0]&0xc0 == 0xc0 // don't corrupt Retry packets
							if dir == quicproxy.DirectionOutgoing && mrand.Intn(interval) == 0 && !isRetry {
								pos := mrand.Intn(len(raw))
								raw[pos] = byte(mrand.Intn(256))
								_, err := serverConn.WriteTo(raw, clientConn.LocalAddr())
								Expect(err).ToNot(HaveOccurred())
								atomic.AddInt32(&numCorrupted, 1)
								return true
							}
							return false
						}
						runTest(dropCb)
					})
				})
			})

			Context("successful injection attacks", func() {
				// TODO(tatianab): add successful injection attacks
			})
		})
	}
})
<|endoftext|>


<commit_before>package database

import (
	"database/sql"

	"github.com/nsec/askgod/api"
)

// GetScoreboard generates the current scoreboard
func (db *DB) GetScoreboard(team *api.AdminTeam) ([]api.ScoreboardEntry, error) {
	// Return a list of score entries
	resp := []api.ScoreboardEntry{}

	// Query all the scores from the database
	var rows *sql.Rows
	var err error

	if team == nil {
		rows, err = db.Query("SELECT team.id, team.country, team.name, team.website, COALESCE(SUM(score.value), 0) AS points, MAX(score.submit_time) FROM score RIGHT JOIN team ON team.id=score.teamid WHERE team.name != '' AND team.country != '' GROUP BY team.id ORDER BY points DESC, team.id ASC;")
		if err != nil {
			return nil, err
		}
	} else {
		rows, err = db.Query("SELECT team.id, team.country, team.name, team.website, COALESCE(SUM(score.value), 0) AS points, MAX(score.submit_time) FROM score RIGHT JOIN team ON team.id=score.teamid WHERE team.id=$1 AND team.name != '' AND team.country != '' GROUP BY team.id ORDER BY points DESC, team.id ASC;", team.ID)
		if err != nil {
			return nil, err
		}
	}
	defer rows.Close()

	// Iterate through the results
	for rows.Next() {
		row := api.ScoreboardEntry{}

		err := rows.Scan(&row.Team.ID, &row.Team.Country, &row.Team.Name, &row.Team.Website, &row.Value, &row.LastSubmitTime)
		if err != nil {
			return nil, err
		}

		resp = append(resp, row)
	}

	// Check for any error that might have happened
	err = rows.Err()
	if err != nil {
		return nil, err
	}

	return resp, nil
}
<commit_msg>askgod-server: Fix handling of NULL time<commit_after>package database

import (
	"database/sql"

	"github.com/lib/pq"

	"github.com/nsec/askgod/api"
)

// GetScoreboard generates the current scoreboard
func (db *DB) GetScoreboard(team *api.AdminTeam) ([]api.ScoreboardEntry, error) {
	// Return a list of score entries
	resp := []api.ScoreboardEntry{}

	// Query all the scores from the database
	var rows *sql.Rows
	var err error

	if team == nil {
		rows, err = db.Query("SELECT team.id, team.country, team.name, team.website, COALESCE(SUM(score.value), 0) AS points, MAX(score.submit_time) FROM score RIGHT JOIN team ON team.id=score.teamid WHERE team.name != '' AND team.country != '' GROUP BY team.id ORDER BY points DESC, team.id ASC;")
		if err != nil {
			return nil, err
		}
	} else {
		rows, err = db.Query("SELECT team.id, team.country, team.name, team.website, COALESCE(SUM(score.value), 0) AS points, MAX(score.submit_time) FROM score RIGHT JOIN team ON team.id=score.teamid WHERE team.id=$1 AND team.name != '' AND team.country != '' GROUP BY team.id ORDER BY points DESC, team.id ASC;", team.ID)
		if err != nil {
			return nil, err
		}
	}
	defer rows.Close()

	// Iterate through the results
	for rows.Next() {
		row := api.ScoreboardEntry{}

		submitTime := pq.NullTime{}
		err := rows.Scan(&row.Team.ID, &row.Team.Country, &row.Team.Name, &row.Team.Website, &row.Value, &submitTime)
		if err != nil {
			return nil, err
		}

		row.LastSubmitTime = submitTime.Time

		resp = append(resp, row)
	}

	// Check for any error that might have happened
	err = rows.Err()
	if err != nil {
		return nil, err
	}

	return resp, nil
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcloud

import (
	"crypto/rand"
	"encoding/hex"
	"github.com/GoogleCloudPlatform/serverless-sample-tester/internal/util"
	"strings"
	"unicode"
)

// Maximum length of a Cloud Run Service name.
const maxCloudRunServiceNameLen = 53

// Length of the random suffix at the end of the Cloud Run Service name.
const cloudRunServiceNameRandSuffixLen = 10

// CloudRunService represents a Cloud Run service and stores its parameters.
type CloudRunService struct {
	Name string
	url  string
}

// Delete calls the external gcloud SDK and deletes the Cloud Run Service associated with the current cloudRunService.
func (s *CloudRunService) Delete() (err error) {
	_, err = util.ExecCommand(util.GcloudCommandBuild(
		"run",
		"services",
		"delete",
		s.Name,
		"--region=us-east4",
		"--platform=managed",
	))

	return
}

// URL calls the external gcloud SDK and gets the root URL of the Cloud Run Service associated with the current
// CloudRunService.
func (s *CloudRunService) URL() (string, error) {
	if s.url != "" {
		return s.url, nil
	}

	url, err := util.ExecCommand(util.GcloudCommandBuild(
		"run",
		"--platform=managed",
		"--region=us-east4",
		"services",
		"describe",
		s.Name,
		"--format=value(status.url)",
	))

	return url, err
}

// ServiceName generates a Cloud Run service name for the provided sample. It concatenates the sample's name with a
// random 10-character alphanumeric string.
func ServiceName(sampleName string) (string, error) {
	randBytes := make([]byte, cloudRunServiceNameRandSuffixLen/2)

	_, err := rand.Read(randBytes)
	if err != nil {
		return "", err
	}

	randSuffix := hex.EncodeToString(randBytes)

	l := maxCloudRunServiceNameLen - len(randSuffix) - 1
	sampleName = sampleName[len(sampleName)-l:]
	sampleName = strings.TrimFunc(sampleName, func(r rune) bool {
		return !unicode.IsLetter(r)
	})

	return sampleName + "-" + randSuffix, nil
}
<commit_msg>Add missed line to cloudrunservice.go<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcloud

import (
	"crypto/rand"
	"encoding/hex"
	"github.com/GoogleCloudPlatform/serverless-sample-tester/internal/util"
	"strings"
	"unicode"
)

// Maximum length of a Cloud Run Service name.
const maxCloudRunServiceNameLen = 53

// Length of the random suffix at the end of the Cloud Run Service name.
const cloudRunServiceNameRandSuffixLen = 10

// CloudRunService represents a Cloud Run service and stores its parameters.
type CloudRunService struct {
	Name string
	url  string
}

// Delete calls the external gcloud SDK and deletes the Cloud Run Service associated with the current cloudRunService.
func (s CloudRunService) Delete() (err error) {
	_, err = util.ExecCommand(util.GcloudCommandBuild(
		"run",
		"services",
		"delete",
		s.Name,
		"--region=us-east4",
		"--platform=managed",
	))

	return
}

// URL calls the external gcloud SDK and gets the root URL of the Cloud Run Service associated with the current
// CloudRunService.
func (s *CloudRunService) URL() (string, error) {
	if s.url != "" {
		return s.url, nil
	}

	url, err := util.ExecCommand(util.GcloudCommandBuild(
		"run",
		"--platform=managed",
		"--region=us-east4",
		"services",
		"describe",
		s.Name,
		"--format=value(status.url)",
	))
	s.url = url

	return url, err
}

// ServiceName generates a Cloud Run service name for the provided sample. It concatenates the sample's name with a
// random 10-character alphanumeric string.
func ServiceName(sampleName string) (string, error) {
	randBytes := make([]byte, cloudRunServiceNameRandSuffixLen/2)

	_, err := rand.Read(randBytes)
	if err != nil {
		return "", err
	}

	randSuffix := hex.EncodeToString(randBytes)

	l := maxCloudRunServiceNameLen - len(randSuffix) - 1
	sampleName = sampleName[len(sampleName)-l:]
	sampleName = strings.TrimFunc(sampleName, func(r rune) bool {
		return !unicode.IsLetter(r)
	})

	return sampleName + "-" + randSuffix, nil
}
<|endoftext|>


<commit_before>package s3

import (
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
	"github.com/aws/aws-sdk-go/aws/session"
)

type SessionBuilder interface {
	Config(*aws.Config) SessionBuilder
	Profile(string) SessionBuilder
	Options(*session.Options) SessionBuilder
	Endpoint(string) SessionBuilder
	S3ForcePathStyle(bool) SessionBuilder
	Build() *session.Session
}

type sessionBuilder struct {
	awsConfig      *aws.Config
	profile        string
	options        *session.Options
	endpoint       *string
	forcePathStyle *bool
}

func (sb *sessionBuilder) Config(c *aws.Config) SessionBuilder {
	sb.awsConfig = c
	return sb
}

func (sb *sessionBuilder) Profile(p string) SessionBuilder {
	sb.profile = p
	return sb
}

func (sb *sessionBuilder) Endpoint(e string) SessionBuilder {
	sb.endpoint = aws.String(e)
	return sb
}

func (sb *sessionBuilder) S3ForcePathStyle(b bool) SessionBuilder {
	sb.forcePathStyle = aws.Bool(b)
	return sb
}

func (sb *sessionBuilder) Options(o *session.Options) SessionBuilder {
	sb.options = o
	return sb
}

func (sb *sessionBuilder) Build() *session.Session {
	if sb.awsConfig == nil {
		sb.awsConfig = aws.NewConfig()
	}

	if sb.endpoint != nil {
		sb.awsConfig.Endpoint = sb.endpoint
		sb.awsConfig.S3ForcePathStyle = sb.forcePathStyle
	}

	sb.awsConfig.Credentials = credentials.NewChainCredentials([]credentials.Provider{
		&credentials.EnvProvider{},
		&credentials.SharedCredentialsProvider{
			Profile: sb.profile,
		},
	})

	_, err := sb.awsConfig.Credentials.Get()
	if err == nil {
		return session.Must(session.NewSession(sb.awsConfig))
	} else {
		if sb.options == nil {
			sb.options = &session.Options{
				AssumeRoleTokenProvider: stscreds.StdinTokenProvider,
				SharedConfigState:       session.SharedConfigEnable,
				Profile:                 sb.profile,
			}
		}

		return session.Must(session.NewSessionWithOptions(*sb.options))
	}
}

func newSessionBuilder() SessionBuilder {
	return &sessionBuilder{}
}
<commit_msg>chore: lint fixes<commit_after>package s3

import (
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
	"github.com/aws/aws-sdk-go/aws/session"
)

type SessionBuilder interface {
	Config(*aws.Config) SessionBuilder
	Profile(string) SessionBuilder
	Options(*session.Options) SessionBuilder
	Endpoint(string) SessionBuilder
	S3ForcePathStyle(bool) SessionBuilder
	Build() *session.Session
}

type sessionBuilder struct {
	awsConfig      *aws.Config
	profile        string
	options        *session.Options
	endpoint       *string
	forcePathStyle *bool
}

func (sb *sessionBuilder) Config(c *aws.Config) SessionBuilder {
	sb.awsConfig = c
	return sb
}

func (sb *sessionBuilder) Profile(p string) SessionBuilder {
	sb.profile = p
	return sb
}

func (sb *sessionBuilder) Endpoint(e string) SessionBuilder {
	sb.endpoint = aws.String(e)
	return sb
}

func (sb *sessionBuilder) S3ForcePathStyle(b bool) SessionBuilder {
	sb.forcePathStyle = aws.Bool(b)
	return sb
}

func (sb *sessionBuilder) Options(o *session.Options) SessionBuilder {
	sb.options = o
	return sb
}

func (sb *sessionBuilder) Build() *session.Session {
	if sb.awsConfig == nil {
		sb.awsConfig = aws.NewConfig()
	}

	if sb.endpoint != nil {
		sb.awsConfig.Endpoint = sb.endpoint
		sb.awsConfig.S3ForcePathStyle = sb.forcePathStyle
	}

	sb.awsConfig.Credentials = credentials.NewChainCredentials([]credentials.Provider{
		&credentials.EnvProvider{},
		&credentials.SharedCredentialsProvider{
			Profile: sb.profile,
		},
	})

	_, err := sb.awsConfig.Credentials.Get()
	if err == nil {
		return session.Must(session.NewSession(sb.awsConfig))
	}
	if sb.options == nil {
		sb.options = &session.Options{
			AssumeRoleTokenProvider: stscreds.StdinTokenProvider,
			SharedConfigState:       session.SharedConfigEnable,
			Profile:                 sb.profile,
		}
	}

	return session.Must(session.NewSessionWithOptions(*sb.options))
}

func newSessionBuilder() SessionBuilder {
	return &sessionBuilder{}
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package diff

import (
	"testing"

	"github.com/GoogleContainerTools/kpt/internal/testutil"
	"github.com/GoogleContainerTools/kpt/internal/testutil/pkgbuilder"
	"github.com/stretchr/testify/assert"
	"sigs.k8s.io/kustomize/kyaml/sets"
)

func TestPkgDiff(t *testing.T) {
	testCases := []struct {
		name string
		pkg1 *pkgbuilder.RootPkg
		pkg2 *pkgbuilder.RootPkg
		diff sets.String
	}{
		{
			name: "equal packages doesn't have a diff",
			pkg1: pkgbuilder.NewRootPkg().
				WithKptfile(pkgbuilder.NewKptfile()).
				WithResource(pkgbuilder.DeploymentResource),
			pkg2: pkgbuilder.NewRootPkg().
				WithKptfile(pkgbuilder.NewKptfile()).
				WithResource(pkgbuilder.DeploymentResource),
			diff: toStringSet(),
		},
		{
			name: "different files between packages",
			pkg1: pkgbuilder.NewRootPkg().
				WithKptfile().
				WithResource(pkgbuilder.DeploymentResource),
			pkg2: pkgbuilder.NewRootPkg().
				WithKptfile().
				WithResource(pkgbuilder.ConfigMapResource),
			diff: toStringSet("configmap.yaml", "deployment.yaml"),
		},
		{
			name: "different upstream in Kptfile is not a diff",
			pkg1: pkgbuilder.NewRootPkg().
				WithKptfile(pkgbuilder.NewKptfile().
					WithUpstream("github.com/GoogleContainerTools/kpt", "/", "master", "resource-merge")).
				WithResource(pkgbuilder.DeploymentResource),
			pkg2: pkgbuilder.NewRootPkg().
				WithKptfile(pkgbuilder.NewKptfile().
					WithUpstream("github.com/GoogleContainerTools/kpt", "/", "kpt/v1", "resource-merge")).
				WithResource(pkgbuilder.DeploymentResource),
			diff: toStringSet(),
		},
		{
			name: "subpackages are not included",
			pkg1: pkgbuilder.NewRootPkg().
				WithKptfile().
				WithResource(pkgbuilder.DeploymentResource).
				WithSubPackages(
					pkgbuilder.NewSubPkg("subpackage").
						WithKptfile(pkgbuilder.NewKptfile()).
						WithResource(pkgbuilder.DeploymentResource),
				),
			pkg2: pkgbuilder.NewRootPkg().
				WithKptfile().
				WithResource(pkgbuilder.DeploymentResource).
				WithSubPackages(
					pkgbuilder.NewSubPkg("subpackage").
						WithKptfile().
						WithResource(pkgbuilder.ConfigMapResource),
				),
			diff: toStringSet(),
		},
	}

	for i := range testCases {
		test := testCases[i]
		t.Run(test.name, func(t *testing.T) {
			pkg1Dir := test.pkg1.ExpandPkg(t, testutil.EmptyReposInfo)
			pkg2Dir := test.pkg2.ExpandPkg(t, testutil.EmptyReposInfo)
			diff, err := PkgDiff(pkg1Dir, pkg2Dir)
			if !assert.NoError(t, err) {
				t.FailNow()
			}

			assert.Equal(t, 0, len(diff.SymmetricDifference(test.diff)))
		})
	}
}

func toStringSet(files ...string) sets.String {
	s := sets.String{}
	for _, f := range files {
		s.Insert(f)
	}
	return s
}
<commit_msg>Pipeline changes cause diff (#1777)<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package diff

import (
	"testing"

	"github.com/GoogleContainerTools/kpt/internal/testutil"
	"github.com/GoogleContainerTools/kpt/internal/testutil/pkgbuilder"
	"github.com/stretchr/testify/assert"
	"sigs.k8s.io/kustomize/kyaml/sets"
)

func TestPkgDiff(t *testing.T) {
	testCases := []struct {
		name string
		pkg1 *pkgbuilder.RootPkg
		pkg2 *pkgbuilder.RootPkg
		diff sets.String
	}{
		{
			name: "equal packages doesn't have a diff",
			pkg1: pkgbuilder.NewRootPkg().
				WithKptfile(pkgbuilder.NewKptfile()).
				WithResource(pkgbuilder.DeploymentResource),
			pkg2: pkgbuilder.NewRootPkg().
				WithKptfile(pkgbuilder.NewKptfile()).
				WithResource(pkgbuilder.DeploymentResource),
			diff: toStringSet(),
		},
		{
			name: "different files between packages",
			pkg1: pkgbuilder.NewRootPkg().
				WithKptfile().
				WithResource(pkgbuilder.DeploymentResource),
			pkg2: pkgbuilder.NewRootPkg().
				WithKptfile().
				WithResource(pkgbuilder.ConfigMapResource),
			diff: toStringSet("configmap.yaml", "deployment.yaml"),
		},
		{
			name: "different upstream in Kptfile is not a diff",
			pkg1: pkgbuilder.NewRootPkg().
				WithKptfile(pkgbuilder.NewKptfile().
					WithUpstream("github.com/GoogleContainerTools/kpt", "/", "master", "resource-merge")).
				WithResource(pkgbuilder.DeploymentResource),
			pkg2: pkgbuilder.NewRootPkg().
				WithKptfile(pkgbuilder.NewKptfile().
					WithUpstream("github.com/GoogleContainerTools/kpt", "/", "kpt/v1", "resource-merge")).
				WithResource(pkgbuilder.DeploymentResource),
			diff: toStringSet(),
		},
		{
			name: "different pipelines in Kptfile is a diff",
			pkg1: pkgbuilder.NewRootPkg().
				WithKptfile(pkgbuilder.NewKptfile().
					WithPipeline(pkgbuilder.NewFunction("gcr.io/kpt-dev/foo:latest"))).
				WithResource(pkgbuilder.DeploymentResource),
			pkg2: pkgbuilder.NewRootPkg().
				WithKptfile(pkgbuilder.NewKptfile().
					WithPipeline(pkgbuilder.NewFunction("gcr.io/kpt-dev/buzz:latest"))).
				WithResource(pkgbuilder.DeploymentResource),
			diff: toStringSet("Kptfile"),
		},
		{
			name: "subpackages are not included",
			pkg1: pkgbuilder.NewRootPkg().
				WithKptfile().
				WithResource(pkgbuilder.DeploymentResource).
				WithSubPackages(
					pkgbuilder.NewSubPkg("subpackage").
						WithKptfile(pkgbuilder.NewKptfile()).
						WithResource(pkgbuilder.DeploymentResource),
				),
			pkg2: pkgbuilder.NewRootPkg().
				WithKptfile().
				WithResource(pkgbuilder.DeploymentResource).
				WithSubPackages(
					pkgbuilder.NewSubPkg("subpackage").
						WithKptfile().
						WithResource(pkgbuilder.ConfigMapResource),
				),
			diff: toStringSet(),
		},
	}

	for i := range testCases {
		test := testCases[i]
		t.Run(test.name, func(t *testing.T) {
			pkg1Dir := test.pkg1.ExpandPkg(t, testutil.EmptyReposInfo)
			pkg2Dir := test.pkg2.ExpandPkg(t, testutil.EmptyReposInfo)
			diff, err := PkgDiff(pkg1Dir, pkg2Dir)
			if !assert.NoError(t, err) {
				t.FailNow()
			}

			assert.Equal(t, 0, len(diff.SymmetricDifference(test.diff)))
		})
	}
}

func toStringSet(files ...string) sets.String {
	s := sets.String{}
	for _, f := range files {
		s.Insert(f)
	}
	return s
}
<|endoftext|>


<commit_before>package main_test

import (
	"os"
	"path/filepath"
	"strings"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/markelog/eclectica/io"
	"github.com/markelog/eclectica/plugins"
)

var _ = Describe("python", func() {
	var (
		pipBin = filepath.Join(bins, "pip")
		eIBin  = filepath.Join(bins, "easy_install")
	)

	Describe("2.x", func() {
		Describe("old", func() {
			if shouldRun("python2-old") == false {
				return
			}

			It(`should install "old" 2.6.9 version`, func() {
				Execute("go", "run", path, "python@2.6.9")

				command, _ := Command("go", "run", path, "ls", "python").Output()

				Expect(strings.Contains(string(command), "♥ 2.6.9")).To(Equal(true))
			})

			It(`should install "old" 2.7.0 version`, func() {
				Execute("go", "run", path, "python@2.7.0")

				command, _ := Command("go", "run", path, "ls", "python").Output()

				Expect(strings.Contains(string(command), "♥ 2.7.0")).To(Equal(true))
			})
		})

		Describe("2.7.x versions", func() {
			if shouldRun("python2.7") == false {
				return
			}

			BeforeEach(func() {
				Execute("go", "run", path, "python@2.7.10")
				Execute("go", "run", path, "python@2.7.12")
			})

			It("should list installed versions", func() {
				command, _ := Command("go", "run", path, "ls", "python").Output()

				Expect(strings.Contains(string(command), "♥ 2.7.12")).To(Equal(true))
			})

			It("should use local version", func() {
				pwd, _ := os.Getwd()
				versionFile := filepath.Join(filepath.Dir(pwd), ".python-version")

				io.WriteFile(versionFile, "2.7.10")

				command, _ := Command("go", "run", path, "ls", "python").Output()
				Expect(strings.Contains(string(command), "♥ 2.7.10")).To(Equal(true))

				err := os.RemoveAll(versionFile)
				Expect(err).To(BeNil())
			})

			It("should list remote versions", func() {
				Expect(checkRemoteList("python", "2.x", 50)).To(Equal(true))
			})

			It("should remove version", func() {
				result := true

				Command("go", "run", path, "rm", "python@2.7.12").Output()

				plugin := plugins.New(&plugins.Args{
					Language: "python",
				})
				versions := plugin.List()

				for _, version := range versions {
					if version == "2.7.12" {
						result = false
					}
				}

				Expect(result).To(Equal(true))

				Execute("go", "run", path, "python@2.7.12")
			})

			It("should have pip installed when it delivered with binaries", func() {
				command, err := Command(pipBin).CombinedOutput()

				Expect(strings.Contains(string(command), "has not been established")).To(Equal(false))
				Expect(err).To(BeNil())
			})

			It("should have easy_install installed when it delivered with binaries", func() {
				command, _ := Command(eIBin).CombinedOutput()

				expected := "error: No urls, filenames, or requirements specified (see --help)"
				actual := string(command)

				Expect(actual).ToNot(ContainSubstring("has not been established"))
				Expect(actual).To(ContainSubstring(expected))
			})

			Describe("2.7.8 version", func() {
				BeforeEach(func() {
					Execute("go", "run", path, "python@2.7.8")
				})

				It("should have pip installed when downloaded", func() {
					command, err := Command(pipBin).CombinedOutput()

					Expect(strings.Contains(string(command), "has not been established")).To(Equal(false))
					Expect(err).To(BeNil())
				})

				It("should have easy_install installed when downloaded", func() {
					command, _ := Command(eIBin).CombinedOutput()

					expected := "error: No urls, filenames, or requirements specified (see --help)"
					actual := string(command)

					Expect(actual).ToNot(ContainSubstring("has not been established"))
					Expect(actual).To(ContainSubstring(expected))
				})
			})
		})
	})

	Describe("3.x", func() {
		if shouldRun("python3") == false {
			return
		}

		BeforeEach(func() {
			Execute("go", "run", path, "python@3.5.1")
			Execute("go", "run", path, "python@3.5.2")
		})

		It("should list installed versions", func() {
			command, _ := Command("go", "run", path, "ls", "python").Output()

			Expect(strings.Contains(string(command), "♥ 3.5.2")).To(Equal(true))
		})

		It("should use local version", func() {
			pwd, _ := os.Getwd()
			versionFile := filepath.Join(filepath.Dir(pwd), ".python-version")

			io.WriteFile(versionFile, "3.5.1")

			command, _ := Command("go", "run", path, "ls", "python").Output()

			Expect(strings.Contains(string(command), "♥ 3.5.1")).To(Equal(true))

			err := os.RemoveAll(versionFile)

			Expect(err).To(BeNil())
		})

		It("should list remote versions", func() {
			Expect(checkRemoteList("python", "3.x", 50)).To(Equal(true))
		})

		It("should remove version", func() {
			result := true

			Command("go", "run", path, "rm", "python@3.5.2").Output()

			plugin := plugins.New(&plugins.Args{
				Language: "python",
			})
			versions := plugin.List()

			for _, version := range versions {
				if version == "3.5.2" {
					result = false
				}
			}

			Expect(result).To(Equal(true))
			Execute("go", "run", path, "python@3.5.2")
		})

		It("should have pip installed when it delivered with binaries", func() {
			command, err := Command(pipBin).CombinedOutput()

			Expect(strings.Contains(string(command), "has not been established")).To(Equal(false))
			Expect(err).To(BeNil())
		})

		It("should have easy_install installed when it delivered with binaries", func() {
			command, _ := Command(eIBin).CombinedOutput()

			expected := "error: No urls, filenames, or requirements specified (see --help)"
			actual := string(command)

			Expect(actual).ToNot(ContainSubstring("has not been established"))
			Expect(actual).To(ContainSubstring(expected))
		})
	})
})
<commit_msg>Increase timeout for python remote list test<commit_after>package main_test

import (
	"os"
	"path/filepath"
	"strings"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/markelog/eclectica/io"
	"github.com/markelog/eclectica/plugins"
)

var _ = Describe("python", func() {
	var (
		pipBin = filepath.Join(bins, "pip")
		eIBin  = filepath.Join(bins, "easy_install")
	)

	Describe("2.x", func() {
		Describe("old", func() {
			if shouldRun("python2-old") == false {
				return
			}

			It(`should install "old" 2.6.9 version`, func() {
				Execute("go", "run", path, "python@2.6.9")

				command, _ := Command("go", "run", path, "ls", "python").Output()

				Expect(strings.Contains(string(command), "♥ 2.6.9")).To(Equal(true))
			})

			It(`should install "old" 2.7.0 version`, func() {
				Execute("go", "run", path, "python@2.7.0")

				command, _ := Command("go", "run", path, "ls", "python").Output()

				Expect(strings.Contains(string(command), "♥ 2.7.0")).To(Equal(true))
			})
		})

		Describe("2.7.x versions", func() {
			if shouldRun("python2.7") == false {
				return
			}

			BeforeEach(func() {
				Execute("go", "run", path, "python@2.7.10")
				Execute("go", "run", path, "python@2.7.12")
			})

			It("should list installed versions", func() {
				command, _ := Command("go", "run", path, "ls", "python").Output()

				Expect(strings.Contains(string(command), "♥ 2.7.12")).To(Equal(true))
			})

			It("should use local version", func() {
				pwd, _ := os.Getwd()
				versionFile := filepath.Join(filepath.Dir(pwd), ".python-version")

				io.WriteFile(versionFile, "2.7.10")

				command, _ := Command("go", "run", path, "ls", "python").Output()
				Expect(strings.Contains(string(command), "♥ 2.7.10")).To(Equal(true))

				err := os.RemoveAll(versionFile)
				Expect(err).To(BeNil())
			})

			It("should list remote versions", func() {
				Expect(checkRemoteList("python", "2.x", 150)).To(Equal(true))
			})

			It("should remove version", func() {
				result := true

				Command("go", "run", path, "rm", "python@2.7.12").Output()

				plugin := plugins.New(&plugins.Args{
					Language: "python",
				})
				versions := plugin.List()

				for _, version := range versions {
					if version == "2.7.12" {
						result = false
					}
				}

				Expect(result).To(Equal(true))

				Execute("go", "run", path, "python@2.7.12")
			})

			It("should have pip installed when it delivered with binaries", func() {
				command, err := Command(pipBin).CombinedOutput()

				Expect(strings.Contains(string(command), "has not been established")).To(Equal(false))
				Expect(err).To(BeNil())
			})

			It("should have easy_install installed when it delivered with binaries", func() {
				command, _ := Command(eIBin).CombinedOutput()

				expected := "error: No urls, filenames, or requirements specified (see --help)"
				actual := string(command)

				Expect(actual).ToNot(ContainSubstring("has not been established"))
				Expect(actual).To(ContainSubstring(expected))
			})

			Describe("2.7.8 version", func() {
				BeforeEach(func() {
					Execute("go", "run", path, "python@2.7.8")
				})

				It("should have pip installed when downloaded", func() {
					command, err := Command(pipBin).CombinedOutput()

					Expect(strings.Contains(string(command), "has not been established")).To(Equal(false))
					Expect(err).To(BeNil())
				})

				It("should have easy_install installed when downloaded", func() {
					command, _ := Command(eIBin).CombinedOutput()

					expected := "error: No urls, filenames, or requirements specified (see --help)"
					actual := string(command)

					Expect(actual).ToNot(ContainSubstring("has not been established"))
					Expect(actual).To(ContainSubstring(expected))
				})
			})
		})
	})

	Describe("3.x", func() {
		if shouldRun("python3") == false {
			return
		}

		BeforeEach(func() {
			Execute("go", "run", path, "python@3.5.1")
			Execute("go", "run", path, "python@3.5.2")
		})

		It("should list installed versions", func() {
			command, _ := Command("go", "run", path, "ls", "python").Output()

			Expect(strings.Contains(string(command), "♥ 3.5.2")).To(Equal(true))
		})

		It("should use local version", func() {
			pwd, _ := os.Getwd()
			versionFile := filepath.Join(filepath.Dir(pwd), ".python-version")

			io.WriteFile(versionFile, "3.5.1")

			command, _ := Command("go", "run", path, "ls", "python").Output()

			Expect(strings.Contains(string(command), "♥ 3.5.1")).To(Equal(true))

			err := os.RemoveAll(versionFile)

			Expect(err).To(BeNil())
		})

		It("should list remote versions", func() {
			Expect(checkRemoteList("python", "3.x", 50)).To(Equal(true))
		})

		It("should remove version", func() {
			result := true

			Command("go", "run", path, "rm", "python@3.5.2").Output()

			plugin := plugins.New(&plugins.Args{
				Language: "python",
			})
			versions := plugin.List()

			for _, version := range versions {
				if version == "3.5.2" {
					result = false
				}
			}

			Expect(result).To(Equal(true))
			Execute("go", "run", path, "python@3.5.2")
		})

		It("should have pip installed when it delivered with binaries", func() {
			command, err := Command(pipBin).CombinedOutput()

			Expect(strings.Contains(string(command), "has not been established")).To(Equal(false))
			Expect(err).To(BeNil())
		})

		It("should have easy_install installed when it delivered with binaries", func() {
			command, _ := Command(eIBin).CombinedOutput()

			expected := "error: No urls, filenames, or requirements specified (see --help)"
			actual := string(command)

			Expect(actual).ToNot(ContainSubstring("has not been established"))
			Expect(actual).To(ContainSubstring(expected))
		})
	})
})
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"math"
	"os"
	"os/signal"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	"v.io/x/devtools/lib/collect"
	"v.io/x/devtools/lib/testutil"
	"v.io/x/devtools/lib/util"
	"v.io/x/devtools/lib/xunit"
	"v.io/x/lib/cmdline"
)

var (
	reviewTargetRefsFlag string
	testFlag             string
)

func init() {
	cmdTest.Flags.StringVar(&projectsFlag, "projects", "", "The base names of the remote projects containing the CLs pointed by the refs, separated by ':'.")
	cmdTest.Flags.StringVar(&reviewTargetRefsFlag, "refs", "", "The review references separated by ':'.")
	cmdTest.Flags.StringVar(&manifestFlag, "manifest", "", "Name of the project manifest.")
	cmdTest.Flags.IntVar(&jenkinsBuildNumberFlag, "build_number", -1, "The number of the Jenkins build.")
	cmdTest.Flags.StringVar(&testFlag, "test", "", "The name of a single test to run.")
}

// cmdTest represents the 'test' command of the presubmit tool.
var cmdTest = &cmdline.Command{
	Name:  "test",
	Short: "Run tests for a CL",
	Long: `
This subcommand pulls the open CLs from Gerrit, runs tests specified in a config
file, and posts test results back to the corresponding Gerrit review thread.
`,
	Run: runTest,
}

const (
	mergeConflictTestClass    = "merge conflict"
	mergeConflictMessageTmpl  = "Possible merge conflict detected in %s.\nPresubmit tests will be executed after a new patchset that resolves the conflicts is submitted."
	nanoToMiliSeconds         = 1000000
	prepareTestBranchAttempts = 3
	unknownStatus             = "UNKNOWN"
)

type cl struct {
	clNumber int
	patchset int
	ref      string
	project  string
}

func (c cl) String() string {
	return fmt.Sprintf("http://go/vcl/%d/%d", c.clNumber, c.patchset)
}

// runTest implements the 'test' subcommand.
func runTest(command *cmdline.Command, args []string) (e error) {
	ctx := util.NewContextFromCommand(command, !noColorFlag, dryRunFlag, verboseFlag)

	// Basic sanity checks.
	if err := sanityChecks(command); err != nil {
		return err
	}

	// Record the current timestamp so we can get the correct postsubmit build
	// when processing the results.
	curTimestamp := time.Now().UnixNano() / nanoToMiliSeconds

	// Generate cls from the refs and projects flags.
	cls, err := parseCLs()
	if err != nil {
		return err
	}

	projects, tools, err := util.ReadManifest(ctx, manifestFlag)
	if err != nil {
		return err
	}

	// tmpBinDir is where developer tools are built after changes are
	// pulled from the target CLs.
	tmpBinDir := filepath.Join(vroot, "tmpBin")

	// Setup cleanup function for cleaning up presubmit test branch.
	cleanupFn := func() error {
		os.RemoveAll(tmpBinDir)
		return cleanupAllPresubmitTestBranches(ctx, projects)
	}
	defer collect.Error(func() error { return cleanupFn() }, &e)

	// Trap SIGTERM and SIGINT signal when the program is aborted
	// on Jenkins.
	go func() {
		sigchan := make(chan os.Signal, 1)
		signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM)
		<-sigchan
		if err := cleanupFn(); err != nil {
			fmt.Fprintf(os.Stderr, "%v\n", err)
		}
		// Linux convention is to use 128+signal as the exit
		// code. We use 0 here to let Jenkins properly mark a
		// run as "Aborted" instead of "Failed".
		os.Exit(0)
	}()

	// Prepare presubmit test branch.
	for i := 1; i <= prepareTestBranchAttempts; i++ {
		if failedCL, err := preparePresubmitTestBranch(ctx, cls, projects); err != nil {
			if i > 1 {
				fmt.Fprintf(ctx.Stdout(), "Attempt #%d:\n", i)
			}
			if failedCL != nil {
				fmt.Fprintf(ctx.Stderr(), "%s: %v\n", failedCL.String(), err)
			}
			if strings.Contains(err.Error(), "unable to access") {
				// Cannot access googlesource.com, try again.
				continue
			}
			if strings.Contains(err.Error(), "git pull") {
				// Possible merge conflict.
				if err := recordMergeConflict(ctx, failedCL); err != nil {
					return err
				}
				return nil
			}
			return err
		}
		break
	}

	// Rebuild developer tools and override VANADIUM_ROOT/bin.
	env, errs := rebuildDeveloperTools(ctx, projects, tools, tmpBinDir)
	if len(errs) > 0 {
		// Don't fail on errors.
		for _, err := range errs {
			printf(ctx.Stderr(), "%v\n", err)
		}
	}

	// Run the tests.
	printf(ctx.Stdout(), "### Running the presubmit test\n")
	prefix := fmt.Sprintf("presubmit/%d/%s", jenkinsBuildNumberFlag, os.Getenv("L"))
	opts := []testutil.TestOpt{testutil.ShortOpt(true), testutil.PrefixOpt(prefix)}
	if results, err := testutil.RunTests(ctx, env, []string{testFlag}, opts...); err == nil {
		result, ok := results[testFlag]
		if !ok {
			return fmt.Errorf("No test result found for %q", testFlag)
		}
		return writeTestStatusFile(ctx, *result, curTimestamp)
	} else {
		return err
	}
}

// sanityChecks performs basic sanity checks for various flags.
func sanityChecks(command *cmdline.Command) error {
	manifestFilePath, err := util.RemoteManifestFile(manifestFlag)
	if err != nil {
		return err
	}
	if _, err := os.Stat(manifestFilePath); err != nil {
		return fmt.Errorf("Stat(%q) failed: %v", manifestFilePath, err)
	}
	if projectsFlag == "" {
		return command.UsageErrorf("-projects flag is required")
	}
	if reviewTargetRefsFlag == "" {
		return command.UsageErrorf("-refs flag is required")
	}
	return nil
}

// parseCLs parses cl info from refs and projects flag, and returns a
// slice of "cl" objects.
func parseCLs() ([]cl, error) {
	refs := strings.Split(reviewTargetRefsFlag, ":")
	projects := strings.Split(projectsFlag, ":")
	if got, want := len(refs), len(projects); got != want {
		return nil, fmt.Errorf("Mismatching lengths of %v and %v: %v vs. %v", refs, projects, len(refs), len(projects))
	}
	cls := []cl{}
	for i, ref := range refs {
		project := projects[i]
		clNumber, patchset, err := parseRefString(ref)
		if err != nil {
			return nil, err
		}
		cls = append(cls, cl{
			clNumber: clNumber,
			patchset: patchset,
			ref:      ref,
			project:  project,
		})
	}
	return cls, nil
}

// presubmitTestBranchName returns the name of the branch where the cl
// content is pulled.
func presubmitTestBranchName(ref string) string {
	return "presubmit_" + ref
}

// preparePresubmitTestBranch creates and checks out the presubmit
// test branch and pulls the CL there.
func preparePresubmitTestBranch(ctx *util.Context, cls []cl, projects map[string]util.Project) (_ *cl, e error) {
	strCLs := []string{}
	for _, cl := range cls {
		strCLs = append(strCLs, cl.String())
	}
	wd, err := os.Getwd()
	if err != nil {
		return nil, fmt.Errorf("Getwd() failed: %v", err)
	}
	defer collect.Error(func() error { return ctx.Run().Chdir(wd) }, &e)
	if err := cleanupAllPresubmitTestBranches(ctx, projects); err != nil {
		return nil, fmt.Errorf("%v\n", err)
	}
	// Pull changes for each cl.
	printf(ctx.Stdout(), "### Preparing to test %s\n", strings.Join(strCLs, ", "))
	prepareFn := func(curCL cl) error {
		localRepo, ok := projects[curCL.project]
		if !ok {
			return fmt.Errorf("project %q not found", curCL.project)
		}
		localRepoDir := localRepo.Path
		if err := ctx.Run().Chdir(localRepoDir); err != nil {
			return fmt.Errorf("Chdir(%v) failed: %v", localRepoDir, err)
		}
		branchName := presubmitTestBranchName(curCL.ref)
		if err := ctx.Git().CreateAndCheckoutBranch(branchName); err != nil {
			return err
		}
		if err := ctx.Git().Pull(util.VanadiumGitRepoHost()+localRepo.Name, curCL.ref); err != nil {
			return err
		}
		return nil
	}
	for _, cl := range cls {
		if err := prepareFn(cl); err != nil {
			testutil.Fail(ctx, "pull changes from %s\n", cl.String())
			return &cl, err
		}
		testutil.Pass(ctx, "pull changes from %s\n", cl.String())
	}
	return nil, nil
}

// recordMergeConflict records possible merge conflict in the test status file
// and xUnit report.
func recordMergeConflict(ctx *util.Context, failedCL *cl) error {
	message := fmt.Sprintf(mergeConflictMessageTmpl, failedCL.String())
	if err := xunit.CreateFailureReport(ctx, testFlag, testFlag, "MergeConflict", message, message); err != nil {
		return nil
	}
	result := testutil.TestResult{
		Status:          testutil.TestFailedMergeConflict,
		MergeConflictCL: failedCL.String(),
	}
	// We use math.MaxInt64 here so that the logic that tries to find the newest
	// build before the given timestamp terminates after the first iteration.
	if err := writeTestStatusFile(ctx, result, math.MaxInt64); err != nil {
		return err
	}
	return nil
}

// rebuildDeveloperTools rebuilds developer tools (e.g. v23, vdl..) in a
// temporary directory, which is used to replace VANADIUM_ROOT/bin in the PATH.
func rebuildDeveloperTools(ctx *util.Context, projects util.Projects, tools util.Tools, tmpBinDir string) (map[string]string, []error) {
	errs := []error{}
	toolsProject, ok := projects["release.go.tools"]
	env := map[string]string{}
	if !ok {
		errs = append(errs, fmt.Errorf("tools project not found, not rebuilding tools."))
	} else {
		// Find target Tools.
		targetTools := []util.Tool{}
		for name, tool := range tools {
			if name == "v23" || name == "vdl" || name == "go-depcop" {
				targetTools = append(targetTools, tool)
			}
		}
		// Rebuild.
		for _, tool := range targetTools {
			if err := util.BuildTool(ctx, tmpBinDir, tool.Name, tool.Package, toolsProject); err != nil {
				errs = append(errs, err)
			}
		}
		// Create a new PATH that replaces VANADIUM_ROOT/bin
		// with the temporary directory in which the tools
		// were rebuilt.
		env["PATH"] = strings.Replace(os.Getenv("PATH"), filepath.Join(vroot, "bin"), tmpBinDir, -1)
	}
	return env, errs
}

// cleanupPresubmitTestBranch removes the presubmit test branch.
func cleanupAllPresubmitTestBranches(ctx *util.Context, projects util.Projects) (e error) {
	printf(ctx.Stdout(), "### Cleaning up\n")
	if err := util.CleanupProjects(ctx, projects, true); err != nil {
		return err
	}
	return nil
}

// writeTestStatusFile writes the given TestResult and timestamp to a JSON file.
// This file will be collected (along with the test report xUnit file) by the
// "master" presubmit project for generating final test results message.
//
// For more details, see comments in result.go.
func writeTestStatusFile(ctx *util.Context, result testutil.TestResult, curTimestamp int64) error {
	// Get the file path.
	workspace, fileName := os.Getenv("WORKSPACE"), fmt.Sprintf("status_%s.json", strings.Replace(testFlag, "-", "_", -1))
	statusFilePath := ""
	if workspace == "" {
		statusFilePath = filepath.Join(os.Getenv("HOME"), "tmp", testFlag, fileName)
	} else {
		statusFilePath = filepath.Join(workspace, fileName)
	}

	// Write to file.
	r := testResultInfo{
		Result:    result,
		TestName:  testFlag,
		Timestamp: curTimestamp,
		AxisValues: axisValuesInfo{
			Arch: os.Getenv("ARCH"), // Architecture is stored in environment variable "ARCH"
			OS:   os.Getenv("OS"),   // OS is stored in environment variable "OS"
		},
	}
	bytes, err := json.Marshal(r)
	if err != nil {
		return fmt.Errorf("Marshal(%v) failed: %v", r, err)
	}
	if err := ctx.Run().WriteFile(statusFilePath, bytes, os.FileMode(0644)); err != nil {
		return fmt.Errorf("WriteFile(%v) failed: %v", statusFilePath, err)
	}
	return nil
}
<commit_msg>devtools/presubmit: update Google storage prefix for presubmit results.<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"math"
	"os"
	"os/signal"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	"v.io/x/devtools/lib/collect"
	"v.io/x/devtools/lib/testutil"
	"v.io/x/devtools/lib/util"
	"v.io/x/devtools/lib/xunit"
	"v.io/x/lib/cmdline"
)

var (
	reviewTargetRefsFlag string
	testFlag             string
)

func init() {
	cmdTest.Flags.StringVar(&projectsFlag, "projects", "", "The base names of the remote projects containing the CLs pointed by the refs, separated by ':'.")
	cmdTest.Flags.StringVar(&reviewTargetRefsFlag, "refs", "", "The review references separated by ':'.")
	cmdTest.Flags.StringVar(&manifestFlag, "manifest", "", "Name of the project manifest.")
	cmdTest.Flags.IntVar(&jenkinsBuildNumberFlag, "build_number", -1, "The number of the Jenkins build.")
	cmdTest.Flags.StringVar(&testFlag, "test", "", "The name of a single test to run.")
}

// cmdTest represents the 'test' command of the presubmit tool.
var cmdTest = &cmdline.Command{
	Name:  "test",
	Short: "Run tests for a CL",
	Long: `
This subcommand pulls the open CLs from Gerrit, runs tests specified in a config
file, and posts test results back to the corresponding Gerrit review thread.
`,
	Run: runTest,
}

const (
	mergeConflictTestClass    = "merge conflict"
	mergeConflictMessageTmpl  = "Possible merge conflict detected in %s.\nPresubmit tests will be executed after a new patchset that resolves the conflicts is submitted."
	nanoToMiliSeconds         = 1000000
	prepareTestBranchAttempts = 3
	unknownStatus             = "UNKNOWN"
)

type cl struct {
	clNumber int
	patchset int
	ref      string
	project  string
}

func (c cl) String() string {
	return fmt.Sprintf("http://go/vcl/%d/%d", c.clNumber, c.patchset)
}

// runTest implements the 'test' subcommand.
func runTest(command *cmdline.Command, args []string) (e error) {
	ctx := util.NewContextFromCommand(command, !noColorFlag, dryRunFlag, verboseFlag)

	// Basic sanity checks.
	if err := sanityChecks(command); err != nil {
		return err
	}

	// Record the current timestamp so we can get the correct postsubmit build
	// when processing the results.
	curTimestamp := time.Now().UnixNano() / nanoToMiliSeconds

	// Generate cls from the refs and projects flags.
	cls, err := parseCLs()
	if err != nil {
		return err
	}

	projects, tools, err := util.ReadManifest(ctx, manifestFlag)
	if err != nil {
		return err
	}

	// tmpBinDir is where developer tools are built after changes are
	// pulled from the target CLs.
	tmpBinDir := filepath.Join(vroot, "tmpBin")

	// Setup cleanup function for cleaning up presubmit test branch.
	cleanupFn := func() error {
		os.RemoveAll(tmpBinDir)
		return cleanupAllPresubmitTestBranches(ctx, projects)
	}
	defer collect.Error(func() error { return cleanupFn() }, &e)

	// Trap SIGTERM and SIGINT signal when the program is aborted
	// on Jenkins.
	go func() {
		sigchan := make(chan os.Signal, 1)
		signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM)
		<-sigchan
		if err := cleanupFn(); err != nil {
			fmt.Fprintf(os.Stderr, "%v\n", err)
		}
		// Linux convention is to use 128+signal as the exit
		// code. We use 0 here to let Jenkins properly mark a
		// run as "Aborted" instead of "Failed".
		os.Exit(0)
	}()

	// Prepare presubmit test branch.
	for i := 1; i <= prepareTestBranchAttempts; i++ {
		if failedCL, err := preparePresubmitTestBranch(ctx, cls, projects); err != nil {
			if i > 1 {
				fmt.Fprintf(ctx.Stdout(), "Attempt #%d:\n", i)
			}
			if failedCL != nil {
				fmt.Fprintf(ctx.Stderr(), "%s: %v\n", failedCL.String(), err)
			}
			if strings.Contains(err.Error(), "unable to access") {
				// Cannot access googlesource.com, try again.
				continue
			}
			if strings.Contains(err.Error(), "git pull") {
				// Possible merge conflict.
				if err := recordMergeConflict(ctx, failedCL); err != nil {
					return err
				}
				return nil
			}
			return err
		}
		break
	}

	// Rebuild developer tools and override VANADIUM_ROOT/bin.
	env, errs := rebuildDeveloperTools(ctx, projects, tools, tmpBinDir)
	if len(errs) > 0 {
		// Don't fail on errors.
		for _, err := range errs {
			printf(ctx.Stderr(), "%v\n", err)
		}
	}

	// Run the tests.
	printf(ctx.Stdout(), "### Running the presubmit test\n")
	prefix := fmt.Sprintf("presubmit/%d/%s/%s", jenkinsBuildNumberFlag, os.Getenv("OS"), os.Getenv("ARCH"))
	opts := []testutil.TestOpt{testutil.ShortOpt(true), testutil.PrefixOpt(prefix)}
	if results, err := testutil.RunTests(ctx, env, []string{testFlag}, opts...); err == nil {
		result, ok := results[testFlag]
		if !ok {
			return fmt.Errorf("No test result found for %q", testFlag)
		}
		return writeTestStatusFile(ctx, *result, curTimestamp)
	} else {
		return err
	}
}

// sanityChecks performs basic sanity checks for various flags.
func sanityChecks(command *cmdline.Command) error {
	manifestFilePath, err := util.RemoteManifestFile(manifestFlag)
	if err != nil {
		return err
	}
	if _, err := os.Stat(manifestFilePath); err != nil {
		return fmt.Errorf("Stat(%q) failed: %v", manifestFilePath, err)
	}
	if projectsFlag == "" {
		return command.UsageErrorf("-projects flag is required")
	}
	if reviewTargetRefsFlag == "" {
		return command.UsageErrorf("-refs flag is required")
	}
	return nil
}

// parseCLs parses cl info from refs and projects flag, and returns a
// slice of "cl" objects.
func parseCLs() ([]cl, error) {
	refs := strings.Split(reviewTargetRefsFlag, ":")
	projects := strings.Split(projectsFlag, ":")
	if got, want := len(refs), len(projects); got != want {
		return nil, fmt.Errorf("Mismatching lengths of %v and %v: %v vs. %v", refs, projects, len(refs), len(projects))
	}
	cls := []cl{}
	for i, ref := range refs {
		project := projects[i]
		clNumber, patchset, err := parseRefString(ref)
		if err != nil {
			return nil, err
		}
		cls = append(cls, cl{
			clNumber: clNumber,
			patchset: patchset,
			ref:      ref,
			project:  project,
		})
	}
	return cls, nil
}

// presubmitTestBranchName returns the name of the branch where the cl
// content is pulled.
func presubmitTestBranchName(ref string) string {
	return "presubmit_" + ref
}

// preparePresubmitTestBranch creates and checks out the presubmit
// test branch and pulls the CL there.
func preparePresubmitTestBranch(ctx *util.Context, cls []cl, projects map[string]util.Project) (_ *cl, e error) {
	strCLs := []string{}
	for _, cl := range cls {
		strCLs = append(strCLs, cl.String())
	}
	wd, err := os.Getwd()
	if err != nil {
		return nil, fmt.Errorf("Getwd() failed: %v", err)
	}
	defer collect.Error(func() error { return ctx.Run().Chdir(wd) }, &e)
	if err := cleanupAllPresubmitTestBranches(ctx, projects); err != nil {
		return nil, fmt.Errorf("%v\n", err)
	}
	// Pull changes for each cl.
	printf(ctx.Stdout(), "### Preparing to test %s\n", strings.Join(strCLs, ", "))
	prepareFn := func(curCL cl) error {
		localRepo, ok := projects[curCL.project]
		if !ok {
			return fmt.Errorf("project %q not found", curCL.project)
		}
		localRepoDir := localRepo.Path
		if err := ctx.Run().Chdir(localRepoDir); err != nil {
			return fmt.Errorf("Chdir(%v) failed: %v", localRepoDir, err)
		}
		branchName := presubmitTestBranchName(curCL.ref)
		if err := ctx.Git().CreateAndCheckoutBranch(branchName); err != nil {
			return err
		}
		if err := ctx.Git().Pull(util.VanadiumGitRepoHost()+localRepo.Name, curCL.ref); err != nil {
			return err
		}
		return nil
	}
	for _, cl := range cls {
		if err := prepareFn(cl); err != nil {
			testutil.Fail(ctx, "pull changes from %s\n", cl.String())
			return &cl, err
		}
		testutil.Pass(ctx, "pull changes from %s\n", cl.String())
	}
	return nil, nil
}

// recordMergeConflict records possible merge conflict in the test status file
// and xUnit report.
func recordMergeConflict(ctx *util.Context, failedCL *cl) error {
	message := fmt.Sprintf(mergeConflictMessageTmpl, failedCL.String())
	if err := xunit.CreateFailureReport(ctx, testFlag, testFlag, "MergeConflict", message, message); err != nil {
		return nil
	}
	result := testutil.TestResult{
		Status:          testutil.TestFailedMergeConflict,
		MergeConflictCL: failedCL.String(),
	}
	// We use math.MaxInt64 here so that the logic that tries to find the newest
	// build before the given timestamp terminates after the first iteration.
	if err := writeTestStatusFile(ctx, result, math.MaxInt64); err != nil {
		return err
	}
	return nil
}

// rebuildDeveloperTools rebuilds developer tools (e.g. v23, vdl..) in a
// temporary directory, which is used to replace VANADIUM_ROOT/bin in the PATH.
func rebuildDeveloperTools(ctx *util.Context, projects util.Projects, tools util.Tools, tmpBinDir string) (map[string]string, []error) {
	errs := []error{}
	toolsProject, ok := projects["release.go.tools"]
	env := map[string]string{}
	if !ok {
		errs = append(errs, fmt.Errorf("tools project not found, not rebuilding tools."))
	} else {
		// Find target Tools.
		targetTools := []util.Tool{}
		for name, tool := range tools {
			if name == "v23" || name == "vdl" || name == "go-depcop" {
				targetTools = append(targetTools, tool)
			}
		}
		// Rebuild.
		for _, tool := range targetTools {
			if err := util.BuildTool(ctx, tmpBinDir, tool.Name, tool.Package, toolsProject); err != nil {
				errs = append(errs, err)
			}
		}
		// Create a new PATH that replaces VANADIUM_ROOT/bin
		// with the temporary directory in which the tools
		// were rebuilt.
		env["PATH"] = strings.Replace(os.Getenv("PATH"), filepath.Join(vroot, "bin"), tmpBinDir, -1)
	}
	return env, errs
}

// cleanupPresubmitTestBranch removes the presubmit test branch.
func cleanupAllPresubmitTestBranches(ctx *util.Context, projects util.Projects) (e error) {
	printf(ctx.Stdout(), "### Cleaning up\n")
	if err := util.CleanupProjects(ctx, projects, true); err != nil {
		return err
	}
	return nil
}

// writeTestStatusFile writes the given TestResult and timestamp to a JSON file.
// This file will be collected (along with the test report xUnit file) by the
// "master" presubmit project for generating final test results message.
//
// For more details, see comments in result.go.
func writeTestStatusFile(ctx *util.Context, result testutil.TestResult, curTimestamp int64) error {
	// Get the file path.
	workspace, fileName := os.Getenv("WORKSPACE"), fmt.Sprintf("status_%s.json", strings.Replace(testFlag, "-", "_", -1))
	statusFilePath := ""
	if workspace == "" {
		statusFilePath = filepath.Join(os.Getenv("HOME"), "tmp", testFlag, fileName)
	} else {
		statusFilePath = filepath.Join(workspace, fileName)
	}

	// Write to file.
	r := testResultInfo{
		Result:    result,
		TestName:  testFlag,
		Timestamp: curTimestamp,
		AxisValues: axisValuesInfo{
			Arch: os.Getenv("ARCH"), // Architecture is stored in environment variable "ARCH"
			OS:   os.Getenv("OS"),   // OS is stored in environment variable "OS"
		},
	}
	bytes, err := json.Marshal(r)
	if err != nil {
		return fmt.Errorf("Marshal(%v) failed: %v", r, err)
	}
	if err := ctx.Run().WriteFile(statusFilePath, bytes, os.FileMode(0644)); err != nil {
		return fmt.Errorf("WriteFile(%v) failed: %v", statusFilePath, err)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"../../api"
	"fmt"
	"github.com/crowdmob/goamz/aws"
	"github.com/crowdmob/goamz/cloudwatch"
	"log"
	"time"
)

type Handler struct {
	accessKey string
	secretKey string
	region    string
	namespace string
}

// Configure the provider.
func (h *Handler) Configure(config api.Config) (err error) {
	if config == nil {
		err = fmt.Errorf("provider requires configuration")
		return
	}

	get := func(name string) (string, error) {
		if val, found := config[name]; found {
			if strval, valid := val.(string); valid && strval != "" {
				return strval, nil
			} else {
				return "", fmt.Errorf("invalid config value %s", name)
			}
		} else {
			return "", fmt.Errorf("missing config value %s", name)
		}
	}

	if h.accessKey, err = get("access-key"); err != nil {
		return
	}
	if h.secretKey, err = get("secret-key"); err != nil {
		return
	}
	if h.region, err = get("region"); err != nil {
		return
	}
	h.namespace, err = get("namespace")
	return
}

// Return an error.
func (h *Handler) Get() (metrics api.Metrics, err error) {
	return api.Metrics{}, fmt.Errorf("sink only provider")
}

// Put metrics into AWS CloudWatch.
func (h *Handler) Put(metrics api.Metrics) (err error) {
	var auth aws.Auth
	var cw *cloudwatch.CloudWatch
	region := aws.Regions[h.region]
	expire := time.Now().UTC().Add(10 * time.Minute)

	if auth, err = aws.GetAuth(h.accessKey, h.secretKey, "", expire); err != nil {
		return
	}
	if cw, err = cloudwatch.NewCloudWatch(auth, region.CloudWatchServicepoint); err != nil {
		return
	}

	metricItems := metrics.Items()
	cwMetrics := make([]cloudwatch.MetricDatum, 0, len(metricItems))
	for _, metric := range metricItems {
		dims := make([]cloudwatch.Dimension, 0, len(metric.Metadata))
		for key, value := range metric.Metadata {
			dims = append(dims, cloudwatch.Dimension{Name: key, Value: value})
		}

		var unit string
		switch metric.Unit {
		case api.UNIT_BYTES:
			unit = "Bytes"
		case api.UNIT_KILOBYTES:
			unit = "Kilobytes"
		case api.UNIT_MEGABYTES:
			unit = "Megabytes"
		case api.UNIT_GIGABYTES:
			unit = "Gigabytes"
		case api.UNIT_TERABYTES:
			unit = "Terabytes"
		case api.UNIT_BYTES_PER_SECOND:
			unit = "Bytes/Second"
		case api.UNIT_KILOBYTES_PER_SECOND:
			unit = "Kilobytes/Second"
		case api.UNIT_MEGABYTES_PER_SECOND:
			unit = "Megabytes/Second"
		case api.UNIT_GIGABYTES_PER_SECOND:
			unit = "Gigabytes/Second"
		case api.UNIT_TERABYTES_PER_SECOND:
			unit = "Terabytes/Second"
		case api.UNIT_BITS:
			unit = "Bits"
		case api.UNIT_KILOBITS:
			unit = "Kilobits"
		case api.UNIT_MEGABITS:
			unit = "Megabits"
		case api.UNIT_GIGABITS:
			unit = "Gigabits"
		case api.UNIT_TERABITS:
			unit = "Terabits"
		case api.UNIT_BITS_PER_SECOND:
			unit = "Bits/Second"
		case api.UNIT_KILOBITS_PER_SECOND:
			unit = "Kilobits/Second"
		case api.UNIT_MEGABITS_PER_SECOND:
			unit = "Megabits/Second"
		case api.UNIT_GIGABITS_PER_SECOND:
			unit = "Gigabits/Second"
		case api.UNIT_TERABITS_PER_SECOND:
			unit = "Terabits/Second"
		case api.UNIT_SECONDS:
			unit = "Seconds"
		case api.UNIT_PERCENT:
			unit = "Percent"
		default:
			fallthrough
		case api.UNIT_COUNT:
			unit = "Count"
		case api.UNIT_COUNT_PER_SECOND:
			unit = "Count/Second"
		}

		cwMetric := cloudwatch.MetricDatum{
			Dimensions: dims,
			MetricName: metric.Name,
			Timestamp:  metric.Timestamp,
			Unit:       unit,
			Value:      metric.Value,
		}
		cwMetrics = append(cwMetrics, cwMetric)
	}

	_, err = cw.PutMetricDataNamespace(cwMetrics, h.namespace)
	return
}

// Run the provider.
func main() {
	api.RunProvider(&Handler{})
}
<commit_msg>Remove unused import.<commit_after>package main

import (
	"../../api"
	"fmt"
	"github.com/crowdmob/goamz/aws"
	"github.com/crowdmob/goamz/cloudwatch"
	"time"
)

type Handler struct {
	accessKey string
	secretKey string
	region    string
	namespace string
}

// Configure the provider.
func (h *Handler) Configure(config api.Config) (err error) {
	if config == nil {
		err = fmt.Errorf("provider requires configuration")
		return
	}

	get := func(name string) (string, error) {
		if val, found := config[name]; found {
			if strval, valid := val.(string); valid && strval != "" {
				return strval, nil
			} else {
				return "", fmt.Errorf("invalid config value %s", name)
			}
		} else {
			return "", fmt.Errorf("missing config value %s", name)
		}
	}

	if h.accessKey, err = get("access-key"); err != nil {
		return
	}
	if h.secretKey, err = get("secret-key"); err != nil {
		return
	}
	if h.region, err = get("region"); err != nil {
		return
	}
	h.namespace, err = get("namespace")
	return
}

// Return an error.
func (h *Handler) Get() (metrics api.Metrics, err error) {
	return api.Metrics{}, fmt.Errorf("sink only provider")
}

// Put metrics into AWS CloudWatch.
func (h *Handler) Put(metrics api.Metrics) (err error) {
	var auth aws.Auth
	var cw *cloudwatch.CloudWatch
	region := aws.Regions[h.region]
	expire := time.Now().UTC().Add(10 * time.Minute)

	if auth, err = aws.GetAuth(h.accessKey, h.secretKey, "", expire); err != nil {
		return
	}
	if cw, err = cloudwatch.NewCloudWatch(auth, region.CloudWatchServicepoint); err != nil {
		return
	}

	metricItems := metrics.Items()
	cwMetrics := make([]cloudwatch.MetricDatum, 0, len(metricItems))
	for _, metric := range metricItems {
		dims := make([]cloudwatch.Dimension, 0, len(metric.Metadata))
		for key, value := range metric.Metadata {
			dims = append(dims, cloudwatch.Dimension{Name: key, Value: value})
		}

		var unit string
		switch metric.Unit {
		case api.UNIT_BYTES:
			unit = "Bytes"
		case api.UNIT_KILOBYTES:
			unit = "Kilobytes"
		case api.UNIT_MEGABYTES:
			unit = "Megabytes"
		case api.UNIT_GIGABYTES:
			unit = "Gigabytes"
		case api.UNIT_TERABYTES:
			unit = "Terabytes"
		case api.UNIT_BYTES_PER_SECOND:
			unit = "Bytes/Second"
		case api.UNIT_KILOBYTES_PER_SECOND:
			unit = "Kilobytes/Second"
		case api.UNIT_MEGABYTES_PER_SECOND:
			unit = "Megabytes/Second"
		case api.UNIT_GIGABYTES_PER_SECOND:
			unit = "Gigabytes/Second"
		case api.UNIT_TERABYTES_PER_SECOND:
			unit = "Terabytes/Second"
		case api.UNIT_BITS:
			unit = "Bits"
		case api.UNIT_KILOBITS:
			unit = "Kilobits"
		case api.UNIT_MEGABITS:
			unit = "Megabits"
		case api.UNIT_GIGABITS:
			unit = "Gigabits"
		case api.UNIT_TERABITS:
			unit = "Terabits"
		case api.UNIT_BITS_PER_SECOND:
			unit = "Bits/Second"
		case api.UNIT_KILOBITS_PER_SECOND:
			unit = "Kilobits/Second"
		case api.UNIT_MEGABITS_PER_SECOND:
			unit = "Megabits/Second"
		case api.UNIT_GIGABITS_PER_SECOND:
			unit = "Gigabits/Second"
		case api.UNIT_TERABITS_PER_SECOND:
			unit = "Terabits/Second"
		case api.UNIT_SECONDS:
			unit = "Seconds"
		case api.UNIT_PERCENT:
			unit = "Percent"
		default:
			fallthrough
		case api.UNIT_COUNT:
			unit = "Count"
		case api.UNIT_COUNT_PER_SECOND:
			unit = "Count/Second"
		}

		cwMetric := cloudwatch.MetricDatum{
			Dimensions: dims,
			MetricName: metric.Name,
			Timestamp:  metric.Timestamp,
			Unit:       unit,
			Value:      metric.Value,
		}
		cwMetrics = append(cwMetrics, cwMetric)
	}

	_, err = cw.PutMetricDataNamespace(cwMetrics, h.namespace)
	return
}

// Run the provider.
func main() {
	api.RunProvider(&Handler{})
}
<|endoftext|>


<commit_before>// Copyright 2013 Ooyala, Inc.

/*
Package dogstatsd provides a Go DogStatsD client. DogStatsD extends StatsD - adding tags and
histograms. Refer to http://docs.datadoghq.com/guides/dogstatsd/ for information about DogStatsD.

Example Usage:
		// Create the client
		c, err := statsd.New("127.0.0.1:8125")
		if err != nil {
			log.Fatal(err)
		}
		// Prefix every metric with the app name
		c.Namespace = "flubber."
		// Send the EC2 availability zone as a tag with every metric
		append(c.Tags, "us-east-1a")
		err = c.Gauge("request.duration", 1.2, nil, 1)

statsd is based on go-statsd-client.
*/

package statsd

import (
	"bytes"
	"fmt"
	"math/rand"
	"net"
	"strconv"
	"strings"
	"sync"
	"time"
)

// A Client is a handle for sending udp messages to dogstatsd.  It is safe to
// use one Client from multiple goroutines simultaneously.
type Client struct {
	conn net.Conn
	// Namespace to prepend to all statsd calls
	Namespace string
	// Tags are global tags to be added to every statsd call
	Tags []string
	// BufferLength is the length of the buffer in commands.
	bufferLength int
	flushTime    time.Duration
	commands     []string
	stop         bool
	sync.Mutex
}

// New returns a pointer to a new Client given an addr in the format "hostname:port".
func New(addr string) (*Client, error) {
	conn, err := net.Dial("udp", addr)
	if err != nil {
		return nil, err
	}
	client := &Client{conn: conn}
	return client, nil
}

// NewBuffered returns a Client that buffers its output and sends it in chunks.
// Buflen is the length of the buffer in number of commands.
func NewBuffered(addr string, buflen int) (*Client, error) {
	client, err := New(addr)
	if err != nil {
		return nil, err
	}
	client.bufferLength = buflen
	client.commands = make([]string, 0, buflen)
	client.flushTime = time.Millisecond * 100
	go client.watch()
	return client, nil
}

// format a message from its name, value, tags and rate.  Also adds global
// namespace and tags.
func (c *Client) format(name, value string, tags []string, rate float64) string {
	var buf bytes.Buffer
	if c.Namespace != "" {
		buf.WriteString(c.Namespace)
	}
	buf.WriteString(name)
	buf.WriteString(":")
	buf.WriteString(value)
	if rate < 1 {
		buf.WriteString(`|@`)
		buf.WriteString(strconv.FormatFloat(rate, 'f', -1, 64))
	}

	tags = append(c.Tags, tags...)
	if len(tags) > 0 {
		buf.WriteString("|#")
		buf.WriteString(tags[0])
		for _, tag := range tags[1:] {
			buf.WriteString(",")
			buf.WriteString(tag)
		}
	}
	return buf.String()
}

func (c *Client) watch() {
	for _ = range time.Tick(c.flushTime) {
		if c.stop {
			return
		}
		c.Lock()
		if len(c.commands) > 0 {
			// FIXME: eating error here
			c.flush()
		}
		c.Unlock()
	}
}

func (c *Client) append(cmd string) error {
	c.Lock()
	c.commands = append(c.commands, cmd)
	// if we should flush, lets do it
	if len(c.commands) == c.bufferLength {
		if err := c.flush(); err != nil {
			c.Unlock()
			return err
		}
	}
	c.Unlock()
	return nil
}

// flush the commands in the buffer.  Lock must be held by caller.
func (c *Client) flush() error {
	data := strings.Join(c.commands, "\n")
	_, err := c.conn.Write([]byte(data))
	// clear the slice with a slice op, doesn't realloc
	c.commands = c.commands[:0]
	return err
}

func (c *Client) sendMsg(msg string) error {
	// if this client is buffered, then we'll just append this
	if c.bufferLength > 0 {
		return c.append(msg)
	}
	c.Lock()
	_, err := c.conn.Write([]byte(msg))
	c.Unlock()
	return err
}

// send handles sampling and sends the message over UDP. It also adds global namespace prefixes and tags.
func (c *Client) send(name, value string, tags []string, rate float64) error {
	if c == nil {
		return nil
	}
	if rate < 1 && rand.Float64() > rate {
		return nil
	}
	data := c.format(name, value, tags, rate)
	return c.sendMsg(data)
}

// Gauge measures the value of a metric at a particular time.
func (c *Client) Gauge(name string, value float64, tags []string, rate float64) error {
	stat := fmt.Sprintf("%f|g", value)
	return c.send(name, stat, tags, rate)
}

// Count tracks how many times something happened per second.
func (c *Client) Count(name string, value int64, tags []string, rate float64) error {
	stat := fmt.Sprintf("%d|c", value)
	return c.send(name, stat, tags, rate)
}

// Histogram tracks the statistical distribution of a set of values.
func (c *Client) Histogram(name string, value float64, tags []string, rate float64) error {
	stat := fmt.Sprintf("%f|h", value)
	return c.send(name, stat, tags, rate)
}

// Set counts the number of unique elements in a group.
func (c *Client) Set(name string, value string, tags []string, rate float64) error {
	stat := fmt.Sprintf("%s|s", value)
	return c.send(name, stat, tags, rate)
}

// Event sends the provided Event
func (c *Client) Event(e *Event) error {
	stat, err := e.Encode(c.Tags...)
	if err != nil {
		return err
	}
	return c.sendMsg(stat)
}

// SimpleEvent sends an event with the provided title and text
func (c *Client) SimpleEvent(title, text string) error {
	e := NewEvent(title, text)
	return c.Event(e)
}

// Close the client connection
func (c *Client) Close() error {
	if c == nil {
		return nil
	}
	c.stop = true
	return c.conn.Close()
}

// Events support

type eventAlertType string

const (
	// Info is the "info" AlertType for events
	Info eventAlertType = "info"
	// Error is the "error" AlertType for events
	Error eventAlertType = "error"
	// Warning is the "warning" AlertType for events
	Warning eventAlertType = "warning"
	// Success is the "success" AlertType for events
	Success eventAlertType = "success"
)

type eventPriority string

const (
	// Normal is the "normal" Priority for events
	Normal eventPriority = "normal"
	// Low is the "low" Priority for events
	Low eventPriority = "low"
)

// An Event is an object that can be posted to your DataDog event stream.
type Event struct {
	// Title of the event.  Required.
	Title string
	// Text is the description of the event.  Required.
	Text string
	// Timestamp is a timestamp for the event.  If not provided, the dogstatsd
	// server will set this to the current time.
	Timestamp time.Time
	// Hostname for the event.
	Hostname string
	// AggregationKey groups this event with others of the same key.
	AggregationKey string
	// Priority of the event.  Can be statsd.Low or statsd.Normal.
	Priority eventPriority
	// SourceTypeName is a source type for the event.
	SourceTypeName string
	// AlertType can be statsd.Info, statsd.Error, statsd.Warning, or statsd.Success.
	// If absent, the default value applied by the dogstatsd server is Info.
	AlertType eventAlertType
	// Tags for the event.
	Tags []string
}

// NewEvent creates a new event with the given title and text.  Error checking
// against these values is done at send-time, or upon running e.Check.
func NewEvent(title, text string) *Event {
	return &Event{
		Title: title,
		Text:  text,
	}
}

// Check verifies that an event is valid.
func (e Event) Check() error {
	if len(e.Title) == 0 {
		return fmt.Errorf("statsd.Event title is required")
	}
	if len(e.Text) == 0 {
		return fmt.Errorf("statsd.Event text is required")
	}
	return nil
}

// Encode returns the dogstatsd wire protocol representation for an event.
// Tags may be passed which will be added to the encoded output but not to
// the Event's list of tags, eg. for default tags.
func (e Event) Encode(tags ...string) (string, error) {
	err := e.Check()
	if err != nil {
		return "", err
	}
	var buffer bytes.Buffer
	buffer.WriteString("_e{")
	buffer.WriteString(strconv.FormatInt(int64(len(e.Title)), 10))
	buffer.WriteRune(',')
	buffer.WriteString(strconv.FormatInt(int64(len(e.Text)), 10))
	buffer.WriteString("}:")
	buffer.WriteString(e.Title)
	buffer.WriteRune('|')
	buffer.WriteString(e.Text)

	if !e.Timestamp.IsZero() {
		buffer.WriteString("|d:")
		buffer.WriteString(strconv.FormatInt(int64(e.Timestamp.Unix()), 10))
	}

	if len(e.Hostname) != 0 {
		buffer.WriteString("|h:")
		buffer.WriteString(e.Hostname)
	}

	if len(e.AggregationKey) != 0 {
		buffer.WriteString("|k:")
		buffer.WriteString(e.AggregationKey)

	}

	if len(e.Priority) != 0 {
		buffer.WriteString("|p:")
		buffer.WriteString(string(e.Priority))
	}

	if len(e.SourceTypeName) != 0 {
		buffer.WriteString("|s:")
		buffer.WriteString(e.SourceTypeName)
	}

	if len(e.AlertType) != 0 {
		buffer.WriteString("|t:")
		buffer.WriteString(string(e.AlertType))
	}

	if len(tags)+len(e.Tags) > 0 {
		all := make([]string, 0, len(tags)+len(e.Tags))
		all = append(all, tags...)
		all = append(all, e.Tags...)
		buffer.WriteString("|#")
		buffer.WriteString(all[0])
		for _, tag := range all[1:] {
			buffer.WriteString(",")
			buffer.WriteString(tag)
		}
	}

	return buffer.String(), nil
}
<commit_msg>gddo fix<commit_after>// Copyright 2013 Ooyala, Inc.

/*
Package dogstatsd provides a Go DogStatsD client. DogStatsD extends StatsD - adding tags and
histograms. Refer to http://docs.datadoghq.com/guides/dogstatsd/ for information about DogStatsD.

Example Usage:
		// Create the client
		c, err := statsd.New("127.0.0.1:8125")
		if err != nil {
			log.Fatal(err)
		}
		// Prefix every metric with the app name
		c.Namespace = "flubber."
		// Send the EC2 availability zone as a tag with every metric
		append(c.Tags, "us-east-1a")
		err = c.Gauge("request.duration", 1.2, nil, 1)

statsd is based on go-statsd-client.
*/
package statsd

import (
	"bytes"
	"fmt"
	"math/rand"
	"net"
	"strconv"
	"strings"
	"sync"
	"time"
)

// A Client is a handle for sending udp messages to dogstatsd.  It is safe to
// use one Client from multiple goroutines simultaneously.
type Client struct {
	conn net.Conn
	// Namespace to prepend to all statsd calls
	Namespace string
	// Tags are global tags to be added to every statsd call
	Tags []string
	// BufferLength is the length of the buffer in commands.
	bufferLength int
	flushTime    time.Duration
	commands     []string
	stop         bool
	sync.Mutex
}

// New returns a pointer to a new Client given an addr in the format "hostname:port".
func New(addr string) (*Client, error) {
	conn, err := net.Dial("udp", addr)
	if err != nil {
		return nil, err
	}
	client := &Client{conn: conn}
	return client, nil
}

// NewBuffered returns a Client that buffers its output and sends it in chunks.
// Buflen is the length of the buffer in number of commands.
func NewBuffered(addr string, buflen int) (*Client, error) {
	client, err := New(addr)
	if err != nil {
		return nil, err
	}
	client.bufferLength = buflen
	client.commands = make([]string, 0, buflen)
	client.flushTime = time.Millisecond * 100
	go client.watch()
	return client, nil
}

// format a message from its name, value, tags and rate.  Also adds global
// namespace and tags.
func (c *Client) format(name, value string, tags []string, rate float64) string {
	var buf bytes.Buffer
	if c.Namespace != "" {
		buf.WriteString(c.Namespace)
	}
	buf.WriteString(name)
	buf.WriteString(":")
	buf.WriteString(value)
	if rate < 1 {
		buf.WriteString(`|@`)
		buf.WriteString(strconv.FormatFloat(rate, 'f', -1, 64))
	}

	tags = append(c.Tags, tags...)
	if len(tags) > 0 {
		buf.WriteString("|#")
		buf.WriteString(tags[0])
		for _, tag := range tags[1:] {
			buf.WriteString(",")
			buf.WriteString(tag)
		}
	}
	return buf.String()
}

func (c *Client) watch() {
	for _ = range time.Tick(c.flushTime) {
		if c.stop {
			return
		}
		c.Lock()
		if len(c.commands) > 0 {
			// FIXME: eating error here
			c.flush()
		}
		c.Unlock()
	}
}

func (c *Client) append(cmd string) error {
	c.Lock()
	c.commands = append(c.commands, cmd)
	// if we should flush, lets do it
	if len(c.commands) == c.bufferLength {
		if err := c.flush(); err != nil {
			c.Unlock()
			return err
		}
	}
	c.Unlock()
	return nil
}

// flush the commands in the buffer.  Lock must be held by caller.
func (c *Client) flush() error {
	data := strings.Join(c.commands, "\n")
	_, err := c.conn.Write([]byte(data))
	// clear the slice with a slice op, doesn't realloc
	c.commands = c.commands[:0]
	return err
}

func (c *Client) sendMsg(msg string) error {
	// if this client is buffered, then we'll just append this
	if c.bufferLength > 0 {
		return c.append(msg)
	}
	c.Lock()
	_, err := c.conn.Write([]byte(msg))
	c.Unlock()
	return err
}

// send handles sampling and sends the message over UDP. It also adds global namespace prefixes and tags.
func (c *Client) send(name, value string, tags []string, rate float64) error {
	if c == nil {
		return nil
	}
	if rate < 1 && rand.Float64() > rate {
		return nil
	}
	data := c.format(name, value, tags, rate)
	return c.sendMsg(data)
}

// Gauge measures the value of a metric at a particular time.
func (c *Client) Gauge(name string, value float64, tags []string, rate float64) error {
	stat := fmt.Sprintf("%f|g", value)
	return c.send(name, stat, tags, rate)
}

// Count tracks how many times something happened per second.
func (c *Client) Count(name string, value int64, tags []string, rate float64) error {
	stat := fmt.Sprintf("%d|c", value)
	return c.send(name, stat, tags, rate)
}

// Histogram tracks the statistical distribution of a set of values.
func (c *Client) Histogram(name string, value float64, tags []string, rate float64) error {
	stat := fmt.Sprintf("%f|h", value)
	return c.send(name, stat, tags, rate)
}

// Set counts the number of unique elements in a group.
func (c *Client) Set(name string, value string, tags []string, rate float64) error {
	stat := fmt.Sprintf("%s|s", value)
	return c.send(name, stat, tags, rate)
}

// Event sends the provided Event.
func (c *Client) Event(e *Event) error {
	stat, err := e.Encode(c.Tags...)
	if err != nil {
		return err
	}
	return c.sendMsg(stat)
}

// SimpleEvent sends an event with the provided title and text.
func (c *Client) SimpleEvent(title, text string) error {
	e := NewEvent(title, text)
	return c.Event(e)
}

// Close the client connection.
func (c *Client) Close() error {
	if c == nil {
		return nil
	}
	c.stop = true
	return c.conn.Close()
}

// Events support

type eventAlertType string

const (
	// Info is the "info" AlertType for events
	Info eventAlertType = "info"
	// Error is the "error" AlertType for events
	Error eventAlertType = "error"
	// Warning is the "warning" AlertType for events
	Warning eventAlertType = "warning"
	// Success is the "success" AlertType for events
	Success eventAlertType = "success"
)

type eventPriority string

const (
	// Normal is the "normal" Priority for events
	Normal eventPriority = "normal"
	// Low is the "low" Priority for events
	Low eventPriority = "low"
)

// An Event is an object that can be posted to your DataDog event stream.
type Event struct {
	// Title of the event.  Required.
	Title string
	// Text is the description of the event.  Required.
	Text string
	// Timestamp is a timestamp for the event.  If not provided, the dogstatsd
	// server will set this to the current time.
	Timestamp time.Time
	// Hostname for the event.
	Hostname string
	// AggregationKey groups this event with others of the same key.
	AggregationKey string
	// Priority of the event.  Can be statsd.Low or statsd.Normal.
	Priority eventPriority
	// SourceTypeName is a source type for the event.
	SourceTypeName string
	// AlertType can be statsd.Info, statsd.Error, statsd.Warning, or statsd.Success.
	// If absent, the default value applied by the dogstatsd server is Info.
	AlertType eventAlertType
	// Tags for the event.
	Tags []string
}

// NewEvent creates a new event with the given title and text.  Error checking
// against these values is done at send-time, or upon running e.Check.
func NewEvent(title, text string) *Event {
	return &Event{
		Title: title,
		Text:  text,
	}
}

// Check verifies that an event is valid.
func (e Event) Check() error {
	if len(e.Title) == 0 {
		return fmt.Errorf("statsd.Event title is required")
	}
	if len(e.Text) == 0 {
		return fmt.Errorf("statsd.Event text is required")
	}
	return nil
}

// Encode returns the dogstatsd wire protocol representation for an event.
// Tags may be passed which will be added to the encoded output but not to
// the Event's list of tags, eg. for default tags.
func (e Event) Encode(tags ...string) (string, error) {
	err := e.Check()
	if err != nil {
		return "", err
	}
	var buffer bytes.Buffer
	buffer.WriteString("_e{")
	buffer.WriteString(strconv.FormatInt(int64(len(e.Title)), 10))
	buffer.WriteRune(',')
	buffer.WriteString(strconv.FormatInt(int64(len(e.Text)), 10))
	buffer.WriteString("}:")
	buffer.WriteString(e.Title)
	buffer.WriteRune('|')
	buffer.WriteString(e.Text)

	if !e.Timestamp.IsZero() {
		buffer.WriteString("|d:")
		buffer.WriteString(strconv.FormatInt(int64(e.Timestamp.Unix()), 10))
	}

	if len(e.Hostname) != 0 {
		buffer.WriteString("|h:")
		buffer.WriteString(e.Hostname)
	}

	if len(e.AggregationKey) != 0 {
		buffer.WriteString("|k:")
		buffer.WriteString(e.AggregationKey)

	}

	if len(e.Priority) != 0 {
		buffer.WriteString("|p:")
		buffer.WriteString(string(e.Priority))
	}

	if len(e.SourceTypeName) != 0 {
		buffer.WriteString("|s:")
		buffer.WriteString(e.SourceTypeName)
	}

	if len(e.AlertType) != 0 {
		buffer.WriteString("|t:")
		buffer.WriteString(string(e.AlertType))
	}

	if len(tags)+len(e.Tags) > 0 {
		all := make([]string, 0, len(tags)+len(e.Tags))
		all = append(all, tags...)
		all = append(all, e.Tags...)
		buffer.WriteString("|#")
		buffer.WriteString(all[0])
		for _, tag := range all[1:] {
			buffer.WriteString(",")
			buffer.WriteString(tag)
		}
	}

	return buffer.String(), nil
}
<|endoftext|>