<commit_before>package service

import (
	"errors"
	"fmt"
	"os"
	"strings"
)

type Flag struct {
	Key   string
	Value *string
	Usage string
}

// Command represents a command-line keyword for the app.
// This is then typically invoked as follows:
//   ./myapp <keyword>
type Command struct {
	Keyword    string
	Run        func(*CommandContext)
	ShortUsage string
	Usage      string
	Flags      []*Flag
}

// AddCommand adds a command to the service via its Config.
func (c *Config) AddCommand(cmd *Command) {
	c.service.registerCommand(cmd)
}

// CommandContext is passed to the command when it is run,
// containing an array of parsed arguments.
type CommandContext struct {
	cmd   *Command
	Args  []string
	Flags map[string]*Flag
}

// UsageExit prints the usage for the executed command and exits.
func (c *CommandContext) UsageExit() {
	fmt.Println(c.cmd.Keyword)
	fmt.Println(c.cmd.Usage)
	os.Exit(0)
}

// RequireAtLeastNArgs is a helper function to ensure we have at least n args.
func (c *CommandContext) RequireAtLeastNArgs(n int) {
	if len(c.Args) < n {
		c.UsageExit()
	}
}

// RequireExactlyNArgs is a helper function to ensure we have at exactly n args.
func (c *CommandContext) RequireExactlyNArgs(n int) {
	if len(c.Args) != n {
		c.UsageExit()
	}
}

func parseArgs(flags []*Flag, args []string) (map[string]*Flag, []string, error) {
	flagMap := map[string]*Flag{}
	for _, f := range flags {
		ks := strings.Split(f.Key, ",")
		for _, k := range ks {
			k = strings.TrimSpace(k)
			k = strings.TrimLeft(k, "-")
			flagMap[k] = f
		}
	}

	updatedArgs := []string{}
	for i := 0; i < len(args); i++ {
		k := args[i]
		if !isFlag(k) {
			updatedArgs = append(updatedArgs, k)
			continue
		}
		k = strings.TrimLeft(k, "-")
		f, ok := flagMap[k]
		if !ok {
			return nil, nil, errors.New("no flag found: " + k)
		}
		if i+1 == len(args) || isFlag(args[i+1]) {
			f.Value = ptr("")
		} else {
			f.Value = ptr(args[i+1])
			i++
		}
	}
	return flagMap, updatedArgs, nil
}

func isFlag(f string) bool {
	return strings.HasPrefix(f, "-")
}

func ptr(s string) *string {
	return &s
}
<commit_msg>Fatal helper for commands.<commit_after>package service

import (
	"errors"
	"fmt"
	"os"
	"strings"
)

type Flag struct {
	Key   string
	Value *string
	Usage string
}

// Command represents a command-line keyword for the app.
// This is then typically invoked as follows:
//   ./myapp <keyword>
type Command struct {
	Keyword    string
	Run        func(*CommandContext)
	ShortUsage string
	Usage      string
	Flags      []*Flag
}

// AddCommand adds a command to the service via its Config.
func (c *Config) AddCommand(cmd *Command) {
	c.service.registerCommand(cmd)
}

// CommandContext is passed to the command when it is run,
// containing an array of parsed arguments.
type CommandContext struct {
	cmd   *Command
	Args  []string
	Flags map[string]*Flag
}

// UsageExit prints the usage for the executed command and exits.
func (c *CommandContext) UsageExit() {
	fmt.Println(c.cmd.Keyword)
	fmt.Println(c.cmd.Usage)
	os.Exit(0)
}

func (c *CommandContext) Fatal(format string, args ...interface{}) {
	fmt.Printf(format+"\n", args...)
	os.Exit(1)
}

// RequireAtLeastNArgs is a helper function to ensure we have at least n args.
func (c *CommandContext) RequireAtLeastNArgs(n int) {
	if len(c.Args) < n {
		c.UsageExit()
	}
}

// RequireExactlyNArgs is a helper function to ensure we have at exactly n args.
func (c *CommandContext) RequireExactlyNArgs(n int) {
	if len(c.Args) != n {
		c.UsageExit()
	}
}

func parseArgs(flags []*Flag, args []string) (map[string]*Flag, []string, error) {
	flagMap := map[string]*Flag{}
	for _, f := range flags {
		ks := strings.Split(f.Key, ",")
		for _, k := range ks {
			k = strings.TrimSpace(k)
			k = strings.TrimLeft(k, "-")
			flagMap[k] = f
		}
	}

	updatedArgs := []string{}
	for i := 0; i < len(args); i++ {
		k := args[i]
		if !isFlag(k) {
			updatedArgs = append(updatedArgs, k)
			continue
		}
		k = strings.TrimLeft(k, "-")
		f, ok := flagMap[k]
		if !ok {
			return nil, nil, errors.New("no flag found: " + k)
		}
		if i+1 == len(args) || isFlag(args[i+1]) {
			f.Value = ptr("")
		} else {
			f.Value = ptr(args[i+1])
			i++
		}
	}
	return flagMap, updatedArgs, nil
}

func isFlag(f string) bool {
	return strings.HasPrefix(f, "-")
}

func ptr(s string) *string {
	return &s
}
<|endoftext|>


<commit_before>// Copyright 2014, The Serviced Authors. All rights reserved.
// Use of this source code is governed by a
// license that can be found in the LICENSE file.

// Package stats collects serviced metrics and posts them to the TSDB.

package stats

import (
	"bytes"
	"encoding/json"
	"github.com/rcrowley/go-metrics"
	"github.com/zenoss/glog"
	"github.com/zenoss/serviced/dao"
	"github.com/zenoss/serviced/stats/cgroup"
	"github.com/zenoss/serviced/utils"
	"github.com/zenoss/serviced/zzk"
	"net/http"
	"strconv"
	"strings"
	"time"
)

// StatsReporter collects and posts serviced stats to the TSDB.
type StatsReporter struct {
	destination  string
	closeChannel chan bool
	zkDAO        *zzk.ZkDao
}

type containerStat struct {
	Metric    string            `json:"metric"`
	Value     string            `json:"value"`
	Timestamp int64             `json:"timestamp"`
	Tags      map[string]string `json:"tags"`
}

var registries map[string]metrics.Registry
var hostID string
var hostRegistry metrics.Registry

// getOrCreateRegistry returns a registry for a given service id or creates it
// if it doesn't exist.
func getOrCreateRegistry(serviceid string) metrics.Registry {
	if registry, ok := registries[serviceid]; ok {
		return registry
	}
	registries[serviceid] = metrics.NewRegistry()
	return registries[serviceid]
}

// NewStatsReporter creates a new StatsReporter and kicks off the reporting goroutine.
func NewStatsReporter(destination string, interval time.Duration, zkDAO *zzk.ZkDao) (*StatsReporter, error) {
	registries = make(map[string]metrics.Registry)
	var err error
	hostID, err = utils.HostID()
	if err != nil {
		glog.Errorf("Could not determine host ID.")
		return nil, err
	}
	hostRegistry = getOrCreateRegistry(hostID)
	sr := StatsReporter{destination, make(chan bool), zkDAO}
	go sr.report(interval)
	return &sr, nil
}

// Close shuts down the reporting goroutine. Blocks waiting for the goroutine to signal that it
// is indeed shutting down.
func (sr StatsReporter) Close() {
	sr.closeChannel <- true
	_ = <-sr.closeChannel
}

// Updates the default registry, fills out the metric consumer format, and posts
// the data to the TSDB. Stops when close signal is received on closeChannel.
func (sr StatsReporter) report(d time.Duration) {
	tc := time.Tick(d)
	for {
		select {
		case _ = <-sr.closeChannel:
			glog.V(3).Info("Ceasing stat reporting.")
			sr.closeChannel <- true
			return
		case t := <-tc:
			glog.V(3).Info("Reporting container stats at:", t)
			sr.updateStats()
			stats := sr.gatherStats(t)
			err := sr.post(stats)
			if err != nil {
				glog.Errorf("Error reporting container stats.")
			}
		}
	}
}

// Updates the default registry.
func (sr StatsReporter) updateStats() {
	// Stats for host.
	if cpuacctStat, err := cgroup.ReadCpuacctStat(""); err != nil {
		glog.V(3).Info("Couldn't read CpuacctStat:", err)
	} else {
		metrics.GetOrRegisterGauge("CpuacctStat.system", hostRegistry).Update(cpuacctStat.System)
		metrics.GetOrRegisterGauge("CpuacctStat.user", hostRegistry).Update(cpuacctStat.User)
	}

	if memoryStat, err := cgroup.ReadMemoryStat(""); err != nil {
		glog.V(3).Info("Couldn't read MemoryStat:", err)
	} else {
		metrics.GetOrRegisterGauge("MemoryStat.pgfault", hostRegistry).Update(memoryStat.Pgfault)
		metrics.GetOrRegisterGauge("MemoryStat.rss", hostRegistry).Update(memoryStat.Rss)
	}

	if openFileDescriptorCount, err := GetOpenFileDescriptorCount(); err != nil {
		glog.V(3).Info("Couldn't get open file descriptor count", err)
	} else {
		metrics.GetOrRegisterGauge("Serviced.OpenFileDescriptors", hostRegistry).Update(openFileDescriptorCount)
	}
	// Stats for the containers.
	var running []*dao.RunningService
	sr.zkDAO.GetRunningServicesForHost(hostID, &running)
	for _, rs := range running {
		containerRegistry := getOrCreateRegistry(rs.ServiceID)
		if cpuacctStat, err := cgroup.ReadCpuacctStat("/sys/fs/cgroup/cpuacct/docker/" + rs.DockerID + "/cpuacct.stat"); err != nil {
			glog.V(3).Info("Couldn't read CpuacctStat:", err)
		} else {
			metrics.GetOrRegisterGauge("CpuacctStat.system", containerRegistry).Update(cpuacctStat.System)
			metrics.GetOrRegisterGauge("CpuacctStat.user", containerRegistry).Update(cpuacctStat.User)
		}
		if memoryStat, err := cgroup.ReadMemoryStat("/sys/fs/cgroup/memory/docker/" + rs.DockerID + "/memory.stat"); err != nil {
			glog.V(3).Info("Couldn't read MemoryStat:", err)
		} else {
			metrics.GetOrRegisterGauge("MemoryStat.pgfault", containerRegistry).Update(memoryStat.Pgfault)
			metrics.GetOrRegisterGauge("MemoryStat.rss", containerRegistry).Update(memoryStat.Rss)
		}
	}
}

// Fills out the metric consumer format.
func (sr StatsReporter) gatherStats(t time.Time) []containerStat {
	stats := []containerStat{}
	for serviceID, registry := range registries {
		reg, _ := registry.(*metrics.StandardRegistry)
		reg.Each(func(name string, i interface{}) {
			metric := i.(metrics.Gauge)
			tagmap := make(map[string]string)
			if serviceID != hostID {
				tagmap["controlplane_service_id"] = serviceID
			}
			tagmap["controlplane_host_id"] = hostID
			stats = append(stats, containerStat{name, strconv.FormatInt(metric.Value(), 10), t.Unix(), tagmap})
		})
	}
	return stats
}

// Send the list of stats to the TSDB.
func (sr StatsReporter) post(stats []containerStat) error {
	payload := map[string][]containerStat{"metrics": stats}
	data, err := json.Marshal(payload)
	if err != nil {
		glog.V(3).Info("Couldn't marshal stats: ", err)
		return err
	}
	statsreq, err := http.NewRequest("POST", sr.destination, bytes.NewBuffer(data))
	if err != nil {
		glog.V(3).Info("Couldn't create stats request: ", err)
		return err
	}
	statsreq.Header["User-Agent"] = []string{"Zenoss Metric Publisher"}
	statsreq.Header["Content-Type"] = []string{"application/json"}
	resp, reqerr := http.DefaultClient.Do(statsreq)
	if reqerr != nil {
		glog.V(3).Info("Couldn't post stats: ", reqerr)
		return reqerr
	}
	if strings.Contains(resp.Status, "200 OK") == false {
		glog.Warningf("couldn't post stats: ", resp.Status)
		return nil
	}
	resp.Body.Close()
	return nil
}
<commit_msg>compiles. we're done here.<commit_after>// Copyright 2014, The Serviced Authors. All rights reserved.
// Use of this source code is governed by a
// license that can be found in the LICENSE file.

// Package stats collects serviced metrics and posts them to the TSDB.

package stats

import (
	"bytes"
	"encoding/json"
	"github.com/rcrowley/go-metrics"
	"github.com/zenoss/glog"
	"github.com/zenoss/serviced/dao"
	"github.com/zenoss/serviced/stats/cgroup"
	"github.com/zenoss/serviced/utils"
	"github.com/zenoss/serviced/zzk"
	"net/http"
	"strconv"
	"strings"
	"time"
)

// StatsReporter collects and posts serviced stats to the TSDB.
type StatsReporter struct {
	destination  string
	closeChannel chan bool
	zkDAO        *zzk.ZkDao
}

type containerStat struct {
	Metric    string            `json:"metric"`
	Value     string            `json:"value"`
	Timestamp int64             `json:"timestamp"`
	Tags      map[string]string `json:"tags"`
}

type registryKey struct {
	serviceID string
	instanceID int
}

var registries map[registryKey]metrics.Registry
var hostID string
var hostRegistry metrics.Registry

// getOrCreateRegistry returns a registry for a given service id or creates it
// if it doesn't exist.
func getOrCreateRegistry(serviceID string, instanceID int) metrics.Registry {
	key := registryKey{serviceID, instanceID}
	if registry, ok := registries[key]; ok {
		return registry
	}
	registries[key] = metrics.NewRegistry()
	return registries[key]
}

// NewStatsReporter creates a new StatsReporter and kicks off the reporting goroutine.
func NewStatsReporter(destination string, interval time.Duration, zkDAO *zzk.ZkDao) (*StatsReporter, error) {
	registries = make(map[registryKey]metrics.Registry)
	var err error
	hostID, err = utils.HostID()
	if err != nil {
		glog.Errorf("Could not determine host ID.")
		return nil, err
	}
	hostRegistry = getOrCreateRegistry("", 0)
	sr := StatsReporter{destination, make(chan bool), zkDAO}
	go sr.report(interval)
	return &sr, nil
}

// Close shuts down the reporting goroutine. Blocks waiting for the goroutine to signal that it
// is indeed shutting down.
func (sr StatsReporter) Close() {
	sr.closeChannel <- true
	_ = <-sr.closeChannel
}

// Updates the default registry, fills out the metric consumer format, and posts
// the data to the TSDB. Stops when close signal is received on closeChannel.
func (sr StatsReporter) report(d time.Duration) {
	tc := time.Tick(d)
	for {
		select {
		case _ = <-sr.closeChannel:
			glog.V(3).Info("Ceasing stat reporting.")
			sr.closeChannel <- true
			return
		case t := <-tc:
			glog.V(3).Info("Reporting container stats at:", t)
			sr.updateStats()
			stats := sr.gatherStats(t)
			err := sr.post(stats)
			if err != nil {
				glog.Errorf("Error reporting container stats.")
			}
		}
	}
}

// Updates the default registry.
func (sr StatsReporter) updateStats() {
	// Stats for host.
	if cpuacctStat, err := cgroup.ReadCpuacctStat(""); err != nil {
		glog.V(3).Info("Couldn't read CpuacctStat:", err)
	} else {
		metrics.GetOrRegisterGauge("CpuacctStat.system", hostRegistry).Update(cpuacctStat.System)
		metrics.GetOrRegisterGauge("CpuacctStat.user", hostRegistry).Update(cpuacctStat.User)
	}

	if memoryStat, err := cgroup.ReadMemoryStat(""); err != nil {
		glog.V(3).Info("Couldn't read MemoryStat:", err)
	} else {
		metrics.GetOrRegisterGauge("MemoryStat.pgfault", hostRegistry).Update(memoryStat.Pgfault)
		metrics.GetOrRegisterGauge("MemoryStat.rss", hostRegistry).Update(memoryStat.Rss)
	}

	if openFileDescriptorCount, err := GetOpenFileDescriptorCount(); err != nil {
		glog.V(3).Info("Couldn't get open file descriptor count", err)
	} else {
		metrics.GetOrRegisterGauge("Serviced.OpenFileDescriptors", hostRegistry).Update(openFileDescriptorCount)
	}
	// Stats for the containers.
	var running []*dao.RunningService
	sr.zkDAO.GetRunningServicesForHost(hostID, &running)
	for _, rs := range running {
		containerRegistry := getOrCreateRegistry(rs.ServiceID, rs.InstanceID)
		if cpuacctStat, err := cgroup.ReadCpuacctStat("/sys/fs/cgroup/cpuacct/docker/" + rs.DockerID + "/cpuacct.stat"); err != nil {
			glog.V(3).Info("Couldn't read CpuacctStat:", err)
		} else {
			metrics.GetOrRegisterGauge("CpuacctStat.system", containerRegistry).Update(cpuacctStat.System)
			metrics.GetOrRegisterGauge("CpuacctStat.user", containerRegistry).Update(cpuacctStat.User)
		}
		if memoryStat, err := cgroup.ReadMemoryStat("/sys/fs/cgroup/memory/docker/" + rs.DockerID + "/memory.stat"); err != nil {
			glog.V(3).Info("Couldn't read MemoryStat:", err)
		} else {
			metrics.GetOrRegisterGauge("MemoryStat.pgfault", containerRegistry).Update(memoryStat.Pgfault)
			metrics.GetOrRegisterGauge("MemoryStat.rss", containerRegistry).Update(memoryStat.Rss)
		}
	}
}

// Fills out the metric consumer format.
func (sr StatsReporter) gatherStats(t time.Time) []containerStat {
	stats := []containerStat{}
	for key, registry := range registries {
		reg, _ := registry.(*metrics.StandardRegistry)
		reg.Each(func(name string, i interface{}) {
			metric := i.(metrics.Gauge)
			tagmap := make(map[string]string)
			if key.serviceID != "" {
				tagmap["controlplane_service_id"] = key.serviceID
				tagmap["controlplane_instance_id"] = string(key.instanceID)
			}
			tagmap["controlplane_host_id"] = hostID
			stats = append(stats, containerStat{name, strconv.FormatInt(metric.Value(), 10), t.Unix(), tagmap})
		})
	}
	return stats
}

// Send the list of stats to the TSDB.
func (sr StatsReporter) post(stats []containerStat) error {
	payload := map[string][]containerStat{"metrics": stats}
	data, err := json.Marshal(payload)
	if err != nil {
		glog.V(3).Info("Couldn't marshal stats: ", err)
		return err
	}
	statsreq, err := http.NewRequest("POST", sr.destination, bytes.NewBuffer(data))
	if err != nil {
		glog.V(3).Info("Couldn't create stats request: ", err)
		return err
	}
	statsreq.Header["User-Agent"] = []string{"Zenoss Metric Publisher"}
	statsreq.Header["Content-Type"] = []string{"application/json"}
	resp, reqerr := http.DefaultClient.Do(statsreq)
	if reqerr != nil {
		glog.V(3).Info("Couldn't post stats: ", reqerr)
		return reqerr
	}
	if strings.Contains(resp.Status, "200 OK") == false {
		glog.Warningf("couldn't post stats: ", resp.Status)
		return nil
	}
	resp.Body.Close()
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"log"
	"math"
	"net"
	"os"
	"os/signal"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"syscall"
	"time"
)

const (
	MAX_UNPROCESSED_PACKETS = 1000
	MAX_UDP_PACKET_SIZE     = 512
)

var signalchan chan os.Signal

type Packet struct {
	Bucket   string
	Value    interface{}
	Modifier string
	Sampling float32
}

type GaugeData struct {
	Relative bool
	Negative bool
	Value    uint64
}

type Uint64Slice []uint64

func (s Uint64Slice) Len() int           { return len(s) }
func (s Uint64Slice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s Uint64Slice) Less(i, j int) bool { return s[i] < s[j] }

type Percentiles []*Percentile
type Percentile struct {
	float float64
	str   string
}

func (a *Percentiles) Set(s string) error {
	f, err := strconv.ParseFloat(s, 64)
	if err != nil {
		return err
	}
	*a = append(*a, &Percentile{f, strings.Replace(s, ".", "_", -1)})
	return nil
}
func (p *Percentile) String() string {
	return p.str
}
func (a *Percentiles) String() string {
	return fmt.Sprintf("%v", *a)
}

var (
	serviceAddress   = flag.String("address", ":8125", "UDP service address")
	graphiteAddress  = flag.String("graphite", "127.0.0.1:2003", "Graphite service address (or - to disable)")
	flushInterval    = flag.Int64("flush-interval", 10, "Flush interval (seconds)")
	debug            = flag.Bool("debug", false, "print statistics sent to graphite")
	showVersion      = flag.Bool("version", false, "print version string")
	persistCountKeys = flag.Int64("persist-count-keys", 60, "number of flush-intervals to persist count keys")
	receiveCounter   = flag.String("receive-counter", "", "Metric name for total metrics received per interval")
	percentThreshold = Percentiles{}
	prefix           = flag.String("prefix", "", "Prefix for all stats")
)

func init() {
	flag.Var(&percentThreshold, "percent-threshold",
		"percentile calculation for timers (0-100, may be given multiple times)")
}

var (
	In              = make(chan *Packet, MAX_UNPROCESSED_PACKETS)
	counters        = make(map[string]int64)
	gauges          = make(map[string]uint64)
	trackedGauges   = make(map[string]uint64)
	timers          = make(map[string]Uint64Slice)
	countInactivity = make(map[string]int64)
	sets            = make(map[string][]string)
)

func monitor() {
	period := time.Duration(*flushInterval) * time.Second
	ticker := time.NewTicker(period)
	for {
		select {
		case sig := <-signalchan:
			fmt.Printf("!! Caught signal %d... shutting down\n", sig)
			if err := submit(time.Now().Add(period)); err != nil {
				log.Printf("ERROR: %s", err)
			}
			return
		case <-ticker.C:
			if err := submit(time.Now().Add(period)); err != nil {
				log.Printf("ERROR: %s", err)
			}
		case s := <-In:
			packetHandler(s)
		}
	}
}

func packetHandler(s *Packet) {
	if *receiveCounter != "" {
		v, ok := counters[*receiveCounter]
		if !ok || v < 0 {
			counters[*receiveCounter] = 0
		}
		counters[*receiveCounter] += 1
	}

	switch s.Modifier {
	case "ms":
		_, ok := timers[s.Bucket]
		if !ok {
			var t Uint64Slice
			timers[s.Bucket] = t
		}
		timers[s.Bucket] = append(timers[s.Bucket], s.Value.(uint64))
	case "g":
		gaugeValue, _ := gauges[s.Bucket]

		gaugeData := s.Value.(GaugeData)
		if gaugeData.Relative {
			if gaugeData.Negative {
				// subtract checking for -ve numbers
				if gaugeData.Value > gaugeValue {
					gaugeValue = 0
				} else {
					gaugeValue -= gaugeData.Value
				}
			} else {
				// watch out for overflows
				if gaugeData.Value > (math.MaxUint64 - gaugeValue) {
					gaugeValue = math.MaxUint64
				} else {
					gaugeValue += gaugeData.Value
				}
			}
		} else {
			gaugeValue = gaugeData.Value
		}

		gauges[s.Bucket] = gaugeValue
	case "c":
		_, ok := counters[s.Bucket]
		if !ok {
			counters[s.Bucket] = 0
		}
		counters[s.Bucket] += int64(float64(s.Value.(int64)) * float64(1/s.Sampling))
	case "s":
		_, ok := sets[s.Bucket]
		if !ok {
			sets[s.Bucket] = make([]string, 0)
		}
		sets[s.Bucket] = append(sets[s.Bucket], s.Value.(string))
	}
}

func submit(deadline time.Time) error {
	var buffer bytes.Buffer
	var num int64

	now := time.Now().Unix()

	if *graphiteAddress == "-" {
		return nil
	}

	client, err := net.Dial("tcp", *graphiteAddress)
	if err != nil {
		if *debug {
			log.Printf("WARNING: resetting counters when in debug mode")
			processCounters(&buffer, now)
			processGauges(&buffer, now)
			processTimers(&buffer, now, percentThreshold)
			processSets(&buffer, now)
		}
		errmsg := fmt.Sprintf("dialing %s failed - %s", *graphiteAddress, err)
		return errors.New(errmsg)
	}
	defer client.Close()

	err = client.SetDeadline(deadline)
	if err != nil {
		errmsg := fmt.Sprintf("could not set deadline:", err)
		return errors.New(errmsg)
	}

	num += processCounters(&buffer, now)
	num += processGauges(&buffer, now)
	num += processTimers(&buffer, now, percentThreshold)
	num += processSets(&buffer, now)
	if num == 0 {
		return nil
	}

	if *debug {
		for _, line := range bytes.Split(buffer.Bytes(), []byte("\n")) {
			if len(line) == 0 {
				continue
			}
			log.Printf("DEBUG: %s", line)
		}
	}

	_, err = client.Write(buffer.Bytes())
	if err != nil {
		errmsg := fmt.Sprintf("failed to write stats - %s", err)
		return errors.New(errmsg)
	}

	log.Printf("sent %d stats to %s", num, *graphiteAddress)

	return nil
}

func processCounters(buffer *bytes.Buffer, now int64) int64 {
	var num int64
	// continue sending zeros for counters for a short period of time even if we have no new data
	for bucket, value := range counters {
		fmt.Fprintf(buffer, "%s %d %d\n", bucket, value, now)
		delete(counters, bucket)
		countInactivity[bucket] = 0
		num++
	}
	for bucket, purgeCount := range countInactivity {
		if purgeCount > 0 {
			fmt.Fprintf(buffer, "%s %d %d\n", bucket, 0, now)
			num++
		}
		countInactivity[bucket] += 1
		if countInactivity[bucket] > *persistCountKeys {
			delete(countInactivity, bucket)
		}
	}
	return num
}

func processGauges(buffer *bytes.Buffer, now int64) int64 {
	var num int64

	for g, c := range gauges {
		lastValue, ok := trackedGauges[g]

		if ok && c == lastValue {
			continue
		}
		fmt.Fprintf(buffer, "%s %d %d\n", g, c, now)
		trackedGauges[g] = c
		num++
	}
	return num
}

func processSets(buffer *bytes.Buffer, now int64) int64 {
	num := int64(len(sets))
	for bucket, set := range sets {

		uniqueSet := map[string]bool{}
		for _, str := range set {
			uniqueSet[str] = true
		}

		fmt.Fprintf(buffer, "%s %d %d\n", bucket, len(uniqueSet), now)
		delete(sets, bucket)
	}
	return num
}

func processTimers(buffer *bytes.Buffer, now int64, pctls Percentiles) int64 {
	var num int64
	for u, t := range timers {
		num++

		sort.Sort(t)
		min := t[0]
		max := t[len(t)-1]
		maxAtThreshold := max
		count := len(t)

		sum := uint64(0)
		for _, value := range t {
			sum += value
		}
		mean := float64(sum) / float64(len(t))

		for _, pct := range pctls {
			if len(t) > 1 {
				var abs float64
				if pct.float >= 0 {
					abs = pct.float
				} else {
					abs = 100 + pct.float
				}
				// poor man's math.Round(x):
				// math.Floor(x + 0.5)
				indexOfPerc := int(math.Floor(((abs / 100.0) * float64(count)) + 0.5))
				if pct.float >= 0 {
					indexOfPerc -= 1 // index offset=0
				}
				maxAtThreshold = t[indexOfPerc]
			}

			var tmpl string
			var pctstr string
			if pct.float >= 0 {
				tmpl = "%s.upper_%s %d %d\n"
				pctstr = pct.str
			} else {
				tmpl = "%s.lower_%s %d %d\n"
				pctstr = pct.str[1:]
			}
			fmt.Fprintf(buffer, tmpl, u, pctstr, maxAtThreshold, now)
		}

		fmt.Fprintf(buffer, "%s.mean %f %d\n", u, mean, now)
		fmt.Fprintf(buffer, "%s.upper %d %d\n", u, max, now)
		fmt.Fprintf(buffer, "%s.lower %d %d\n", u, min, now)
		fmt.Fprintf(buffer, "%s.count %d %d\n", u, count, now)

		delete(timers, u)
	}
	return num
}

func parseMessage(data []byte) []*Packet {
	var (
		output []*Packet
		input  []byte
	)

	for _, line := range bytes.Split(data, []byte("\n")) {
		if len(line) == 0 {
			continue
		}
		input = line

		index := bytes.IndexByte(input, ':')
		if index < 0 || index == len(input) - 1 {
			if *debug {
				log.Printf("ERROR: failed to parse line: %s\n", string(line))
			}
			continue
		}

		name := input[:index]

		index++
		input = input[index:]

		index = bytes.IndexByte(input, '|')
		if index < 0 || index == len(input) - 1 {
			if *debug {
				log.Printf("ERROR: failed to parse line: %s\n", string(line))
			}
			continue
		}

		val := input[:index]
		index++

		var mtypeStr string

		if input[index] == 'm' {
			index++
			if index >= len(input) || input[index] != 's' {
				if *debug {
					log.Printf("ERROR: failed to parse line: %s\n", string(line))
				}
				continue
			}
			mtypeStr = "ms"
		} else {
			mtypeStr = string(input[index])
		}

		index++
		input = input[index:]

		var (
			value interface{}
			err   error
		)

		if mtypeStr[0] == 'c' {
			value, err = strconv.ParseInt(string(val), 10, 64)
			if err != nil {
				log.Printf("ERROR: failed to ParseInt %s - %s", string(val), err)
				continue
			}
		} else if mtypeStr[0] == 'g' {
			var relative, negative bool
			var stringToParse string

			switch val[0] {
			case '+', '-':
				relative = true
				negative = val[0] == '-'
				stringToParse = string(val[1:])
			default:
				relative = false
				negative = false
				stringToParse = string(val)
			}

			gaugeValue, err := strconv.ParseUint(stringToParse, 10, 64)
			if err != nil {
				log.Printf("ERROR: failed to ParseUint %s - %s", string(val), err)
				continue
			}

			value = GaugeData{relative, negative, gaugeValue}
		} else if mtypeStr[0] == 's' {
			value = string(val)
		} else {
			value, err = strconv.ParseUint(string(val), 10, 64)
			if err != nil {
				log.Printf("ERROR: failed to ParseUint %s - %s", string(val), err)
				continue
			}
		}

		var sampleRate float32 = 1

		if len(input) > 0 && bytes.HasPrefix(input, []byte("|@")) {
			input = input[2:]
			rate, err := strconv.ParseFloat(string(input), 32)
			if err == nil {
				sampleRate = float32(rate)
			}
		}

		packet := &Packet{
			Bucket:   *prefix + string(name),
			Value:    value,
			Modifier: mtypeStr,
			Sampling: sampleRate,
		}
		output = append(output, packet)
	}
	return output
}

func udpListener() {
	address, _ := net.ResolveUDPAddr("udp", *serviceAddress)
	log.Printf("listening on %s", address)
	listener, err := net.ListenUDP("udp", address)
	if err != nil {
		log.Fatalf("ERROR: ListenUDP - %s", err)
	}
	defer listener.Close()

	message := make([]byte, MAX_UDP_PACKET_SIZE)
	for {
		n, remaddr, err := listener.ReadFromUDP(message)
		if err != nil {
			log.Printf("ERROR: reading UDP packet from %+v - %s", remaddr, err)
			continue
		}

		for _, p := range parseMessage(message[:n]) {
			In <- p
		}
	}
}

func main() {
	flag.Parse()

	if *showVersion {
		fmt.Printf("statsdaemon v%s (built w/%s)\n", VERSION, runtime.Version())
		return
	}

	signalchan = make(chan os.Signal, 1)
	signal.Notify(signalchan, syscall.SIGTERM)

	go udpListener()
	monitor()
}
<commit_msg>make gauges a float value<commit_after>package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"log"
	"math"
	"net"
	"os"
	"os/signal"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"syscall"
	"time"
)

const (
	MAX_UNPROCESSED_PACKETS = 1000
	MAX_UDP_PACKET_SIZE     = 512
)

var signalchan chan os.Signal

type Packet struct {
	Bucket   string
	Value    interface{}
	Modifier string
	Sampling float32
}

type GaugeData struct {
	Relative bool
	Negative bool
	Value    float64
}

type Uint64Slice []uint64

func (s Uint64Slice) Len() int           { return len(s) }
func (s Uint64Slice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s Uint64Slice) Less(i, j int) bool { return s[i] < s[j] }

type Percentiles []*Percentile
type Percentile struct {
	float float64
	str   string
}

func (a *Percentiles) Set(s string) error {
	f, err := strconv.ParseFloat(s, 64)
	if err != nil {
		return err
	}
	*a = append(*a, &Percentile{f, strings.Replace(s, ".", "_", -1)})
	return nil
}
func (p *Percentile) String() string {
	return p.str
}
func (a *Percentiles) String() string {
	return fmt.Sprintf("%v", *a)
}

var (
	serviceAddress   = flag.String("address", ":8125", "UDP service address")
	graphiteAddress  = flag.String("graphite", "127.0.0.1:2003", "Graphite service address (or - to disable)")
	flushInterval    = flag.Int64("flush-interval", 10, "Flush interval (seconds)")
	debug            = flag.Bool("debug", false, "print statistics sent to graphite")
	showVersion      = flag.Bool("version", false, "print version string")
	persistCountKeys = flag.Int64("persist-count-keys", 60, "number of flush-intervals to persist count keys")
	receiveCounter   = flag.String("receive-counter", "", "Metric name for total metrics received per interval")
	percentThreshold = Percentiles{}
	prefix           = flag.String("prefix", "", "Prefix for all stats")
)

func init() {
	flag.Var(&percentThreshold, "percent-threshold",
		"percentile calculation for timers (0-100, may be given multiple times)")
}

var (
	In              = make(chan *Packet, MAX_UNPROCESSED_PACKETS)
	counters        = make(map[string]int64)
	gauges          = make(map[string]float64)
	trackedGauges   = make(map[string]float64)
	timers          = make(map[string]Uint64Slice)
	countInactivity = make(map[string]int64)
	sets            = make(map[string][]string)
)

func monitor() {
	period := time.Duration(*flushInterval) * time.Second
	ticker := time.NewTicker(period)
	for {
		select {
		case sig := <-signalchan:
			fmt.Printf("!! Caught signal %d... shutting down\n", sig)
			if err := submit(time.Now().Add(period)); err != nil {
				log.Printf("ERROR: %s", err)
			}
			return
		case <-ticker.C:
			if err := submit(time.Now().Add(period)); err != nil {
				log.Printf("ERROR: %s", err)
			}
		case s := <-In:
			packetHandler(s)
		}
	}
}

func packetHandler(s *Packet) {
	if *receiveCounter != "" {
		v, ok := counters[*receiveCounter]
		if !ok || v < 0 {
			counters[*receiveCounter] = 0
		}
		counters[*receiveCounter] += 1
	}

	switch s.Modifier {
	case "ms":
		_, ok := timers[s.Bucket]
		if !ok {
			var t Uint64Slice
			timers[s.Bucket] = t
		}
		timers[s.Bucket] = append(timers[s.Bucket], s.Value.(uint64))
	case "g":
		gaugeValue, _ := gauges[s.Bucket]

		gaugeData := s.Value.(GaugeData)
		if gaugeData.Relative {
			if gaugeData.Negative {
				// subtract checking for -ve numbers
				if gaugeData.Value > gaugeValue {
					gaugeValue = 0
				} else {
					gaugeValue -= gaugeData.Value
				}
			} else {
				// watch out for overflows
				if gaugeData.Value > (math.MaxUint64 - gaugeValue) {
					gaugeValue = math.MaxUint64
				} else {
					gaugeValue += gaugeData.Value
				}
			}
		} else {
			gaugeValue = gaugeData.Value
		}

		gauges[s.Bucket] = gaugeValue
	case "c":
		_, ok := counters[s.Bucket]
		if !ok {
			counters[s.Bucket] = 0
		}
		counters[s.Bucket] += int64(float64(s.Value.(int64)) * float64(1/s.Sampling))
	case "s":
		_, ok := sets[s.Bucket]
		if !ok {
			sets[s.Bucket] = make([]string, 0)
		}
		sets[s.Bucket] = append(sets[s.Bucket], s.Value.(string))
	}
}

func submit(deadline time.Time) error {
	var buffer bytes.Buffer
	var num int64

	now := time.Now().Unix()

	if *graphiteAddress == "-" {
		return nil
	}

	client, err := net.Dial("tcp", *graphiteAddress)
	if err != nil {
		if *debug {
			log.Printf("WARNING: resetting counters when in debug mode")
			processCounters(&buffer, now)
			processGauges(&buffer, now)
			processTimers(&buffer, now, percentThreshold)
			processSets(&buffer, now)
		}
		errmsg := fmt.Sprintf("dialing %s failed - %s", *graphiteAddress, err)
		return errors.New(errmsg)
	}
	defer client.Close()

	err = client.SetDeadline(deadline)
	if err != nil {
		errmsg := fmt.Sprintf("could not set deadline:", err)
		return errors.New(errmsg)
	}

	num += processCounters(&buffer, now)
	num += processGauges(&buffer, now)
	num += processTimers(&buffer, now, percentThreshold)
	num += processSets(&buffer, now)
	if num == 0 {
		return nil
	}

	if *debug {
		for _, line := range bytes.Split(buffer.Bytes(), []byte("\n")) {
			if len(line) == 0 {
				continue
			}
			log.Printf("DEBUG: %s", line)
		}
	}

	_, err = client.Write(buffer.Bytes())
	if err != nil {
		errmsg := fmt.Sprintf("failed to write stats - %s", err)
		return errors.New(errmsg)
	}

	log.Printf("sent %d stats to %s", num, *graphiteAddress)

	return nil
}

func processCounters(buffer *bytes.Buffer, now int64) int64 {
	var num int64
	// continue sending zeros for counters for a short period of time even if we have no new data
	for bucket, value := range counters {
		fmt.Fprintf(buffer, "%s %d %d\n", bucket, value, now)
		delete(counters, bucket)
		countInactivity[bucket] = 0
		num++
	}
	for bucket, purgeCount := range countInactivity {
		if purgeCount > 0 {
			fmt.Fprintf(buffer, "%s %d %d\n", bucket, 0, now)
			num++
		}
		countInactivity[bucket] += 1
		if countInactivity[bucket] > *persistCountKeys {
			delete(countInactivity, bucket)
		}
	}
	return num
}

func processGauges(buffer *bytes.Buffer, now int64) int64 {
	var num int64

	for g, c := range gauges {
		lastValue, ok := trackedGauges[g]

		if ok && c == lastValue {
			continue
		}
		fmt.Fprintf(buffer, "%s %d %d\n", g, c, now)
		trackedGauges[g] = c
		num++
	}
	return num
}

func processSets(buffer *bytes.Buffer, now int64) int64 {
	num := int64(len(sets))
	for bucket, set := range sets {

		uniqueSet := map[string]bool{}
		for _, str := range set {
			uniqueSet[str] = true
		}

		fmt.Fprintf(buffer, "%s %d %d\n", bucket, len(uniqueSet), now)
		delete(sets, bucket)
	}
	return num
}

func processTimers(buffer *bytes.Buffer, now int64, pctls Percentiles) int64 {
	var num int64
	for u, t := range timers {
		num++

		sort.Sort(t)
		min := t[0]
		max := t[len(t)-1]
		maxAtThreshold := max
		count := len(t)

		sum := uint64(0)
		for _, value := range t {
			sum += value
		}
		mean := float64(sum) / float64(len(t))

		for _, pct := range pctls {
			if len(t) > 1 {
				var abs float64
				if pct.float >= 0 {
					abs = pct.float
				} else {
					abs = 100 + pct.float
				}
				// poor man's math.Round(x):
				// math.Floor(x + 0.5)
				indexOfPerc := int(math.Floor(((abs / 100.0) * float64(count)) + 0.5))
				if pct.float >= 0 {
					indexOfPerc -= 1 // index offset=0
				}
				maxAtThreshold = t[indexOfPerc]
			}

			var tmpl string
			var pctstr string
			if pct.float >= 0 {
				tmpl = "%s.upper_%s %d %d\n"
				pctstr = pct.str
			} else {
				tmpl = "%s.lower_%s %d %d\n"
				pctstr = pct.str[1:]
			}
			fmt.Fprintf(buffer, tmpl, u, pctstr, maxAtThreshold, now)
		}

		fmt.Fprintf(buffer, "%s.mean %f %d\n", u, mean, now)
		fmt.Fprintf(buffer, "%s.upper %d %d\n", u, max, now)
		fmt.Fprintf(buffer, "%s.lower %d %d\n", u, min, now)
		fmt.Fprintf(buffer, "%s.count %d %d\n", u, count, now)

		delete(timers, u)
	}
	return num
}

func parseMessage(data []byte) []*Packet {
	var (
		output []*Packet
		input  []byte
	)

	for _, line := range bytes.Split(data, []byte("\n")) {
		if len(line) == 0 {
			continue
		}
		input = line

		index := bytes.IndexByte(input, ':')
		if index < 0 || index == len(input) - 1 {
			if *debug {
				log.Printf("ERROR: failed to parse line: %s\n", string(line))
			}
			continue
		}

		name := input[:index]

		index++
		input = input[index:]

		index = bytes.IndexByte(input, '|')
		if index < 0 || index == len(input) - 1 {
			if *debug {
				log.Printf("ERROR: failed to parse line: %s\n", string(line))
			}
			continue
		}

		val := input[:index]
		index++

		var mtypeStr string

		if input[index] == 'm' {
			index++
			if index >= len(input) || input[index] != 's' {
				if *debug {
					log.Printf("ERROR: failed to parse line: %s\n", string(line))
				}
				continue
			}
			mtypeStr = "ms"
		} else {
			mtypeStr = string(input[index])
		}

		index++
		input = input[index:]

		var (
			value interface{}
			err   error
		)

		if mtypeStr[0] == 'c' {
			value, err = strconv.ParseInt(string(val), 10, 64)
			if err != nil {
				log.Printf("ERROR: failed to ParseInt %s - %s", string(val), err)
				continue
			}
		} else if mtypeStr[0] == 'g' {
			var relative, negative bool
			var stringToParse string

			switch val[0] {
			case '+', '-':
				relative = true
				negative = val[0] == '-'
				stringToParse = string(val[1:])
			default:
				relative = false
				negative = false
				stringToParse = string(val)
			}

			gaugeValue, err := strconv.ParseFloat(stringToParse, 64)
			if err != nil {
				log.Printf("ERROR: failed to ParseUint %s - %s", string(val), err)
				continue
			}

			value = GaugeData{relative, negative, gaugeValue}
		} else if mtypeStr[0] == 's' {
			value = string(val)
		} else {
			value, err = strconv.ParseUint(string(val), 10, 64)
			if err != nil {
				log.Printf("ERROR: failed to ParseUint %s - %s", string(val), err)
				continue
			}
		}

		var sampleRate float32 = 1

		if len(input) > 0 && bytes.HasPrefix(input, []byte("|@")) {
			input = input[2:]
			rate, err := strconv.ParseFloat(string(input), 32)
			if err == nil {
				sampleRate = float32(rate)
			}
		}

		packet := &Packet{
			Bucket:   *prefix + string(name),
			Value:    value,
			Modifier: mtypeStr,
			Sampling: sampleRate,
		}
		output = append(output, packet)
	}
	return output
}

func udpListener() {
	address, _ := net.ResolveUDPAddr("udp", *serviceAddress)
	log.Printf("listening on %s", address)
	listener, err := net.ListenUDP("udp", address)
	if err != nil {
		log.Fatalf("ERROR: ListenUDP - %s", err)
	}
	defer listener.Close()

	message := make([]byte, MAX_UDP_PACKET_SIZE)
	for {
		n, remaddr, err := listener.ReadFromUDP(message)
		if err != nil {
			log.Printf("ERROR: reading UDP packet from %+v - %s", remaddr, err)
			continue
		}

		for _, p := range parseMessage(message[:n]) {
			In <- p
		}
	}
}

func main() {
	flag.Parse()

	if *showVersion {
		fmt.Printf("statsdaemon v%s (built w/%s)\n", VERSION, runtime.Version())
		return
	}

	signalchan = make(chan os.Signal, 1)
	signal.Notify(signalchan, syscall.SIGTERM)

	go udpListener()
	monitor()
}
<|endoftext|>


<commit_before>package main

import (
	"math/big"
	"testing"
	"time"
)

func TestStatusString(t *testing.T) {
	s := new(big.Int)
	s.SetString("17015245701990644280577643802745589798", 10)

	tt := time.Date(2017, 12, 20, 23, 59, 59, 0, time.UTC)
	st := &Status{
		SerialNumber: s,
		Status:       "Revoked",
		RevokedAt:    tt,
	}

	got := st.String()

	expected := "Serial number: 17015245701990644280577643802745589798\n\n" +
		"Status: Revoked\n" +
		"Revoked at: 2017-12-20 23:59:59 +0000 UTC\n"

	if got != expected {
		t.Errorf("expected %q, got %q", expected, got)
	}
}

func TestStatusWithReasonString(t *testing.T) {
	s := new(big.Int)
	s.SetString("17015245701990644280577643802745589799", 10)

	tt := time.Date(2017, 12, 24, 23, 59, 59, 0, time.UTC)
	st := &Status{
		SerialNumber: s,
		Status:       "Revoked",
		Reason:       "Key compromise",
		RevokedAt:    tt,
	}

	got := st.String()

	expected := "Serial number: 17015245701990644280577643802745589799\n\n" +
		"Status: Revoked\n" +
		"Reason: Key compromise\n" +
		"Revoked at: 2017-12-24 23:59:59 +0000 UTC\n"

	if got != expected {
		t.Errorf("expected %q, got %q", expected, got)
	}
}
<commit_msg>Updated status test.<commit_after>package main

import (
	"math/big"
	"testing"
	"time"
)

func TestStatusString(t *testing.T) {
	s := new(big.Int)
	s.SetString("17015245701990644280577643802745589798", 10)

	st := &Status{
		SerialNumber: s,
		Status:       "Good",
	}

	got := st.String()

	expected := "Serial number: 17015245701990644280577643802745589798\n\n" +
		"Status: Good\n"

	if got != expected {
		t.Errorf("expected %q, got %q", expected, got)
	}
}

func TestStatusWithReasonString(t *testing.T) {
	s := new(big.Int)
	s.SetString("17015245701990644280577643802745589799", 10)

	tt := time.Date(2017, 12, 24, 23, 59, 59, 0, time.UTC)
	st := &Status{
		SerialNumber: s,
		Status:       "Revoked",
		Reason:       "Key compromise",
		RevokedAt:    tt,
	}

	got := st.String()

	expected := "Serial number: 17015245701990644280577643802745589799\n\n" +
		"Status: Revoked\n" +
		"Reason: Key compromise\n" +
		"Revoked at: 2017-12-24 23:59:59 +0000 UTC\n"

	if got != expected {
		t.Errorf("expected %q, got %q", expected, got)
	}
}
<|endoftext|>


<commit_before>package people

import (
	"encoding/json"
	"errors"
	"fmt"

	"github.com/Financial-Times/go-fthealth/v1a"
	"github.com/Financial-Times/neo-cypher-runner-go"
	"github.com/jmcvetta/neoism"
)

type CypherDriver struct {
	cypherRunner neocypherrunner.CypherRunner
}

func NewCypherDriver(cypherRunner neocypherrunner.CypherRunner) CypherDriver {
	return CypherDriver{cypherRunner}
}

func (pcd CypherDriver) Read(uuid string) (interface{}, bool, error) {
	results := []struct {
		UUID              string `json:"uuid"`
		Name              string `json:"name"`
		BirthYear         int    `json:"birthYear"`
		Salutation        string `json:"salutation"`
		FactsetIdentifier string `json:"factsetIdentifier"`
	}{}

	query := &neoism.CypherQuery{
		Statement: `MATCH (n:Person {uuid:{uuid}}) return n.uuid
		as uuid, n.name as name,
		n.factsetIdentifier as factsetIdentifier,
		n.birthYear as birthYear,
		n.salutation as salutation`,
		Parameters: map[string]interface{}{
			"uuid": uuid,
		},
		Result: &results,
	}

	err := pcd.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})

	if err != nil {
		return person{}, false, err
	}

	if len(results) == 0 {
		return person{}, false, nil
	}

	result := results[0]

	p := person{
		UUID:       result.UUID,
		Name:       result.Name,
		BirthYear:  result.BirthYear,
		Salutation: result.Salutation,
	}

	if result.FactsetIdentifier != "" {
		p.Identifiers = append(p.Identifiers, identifier{fsAuthority, result.FactsetIdentifier})
	}

	return p, true, nil

}

func (pcd CypherDriver) Write(thing interface{}) error {

	p := thing.(person)

	params := map[string]interface{}{
		"uuid": p.UUID,
	}

	if p.Name != "" {
		params["name"] = p.Name
	}

	if p.BirthYear != 0 {
		params["birthYear"] = p.BirthYear
	}

	if p.Salutation != "" {
		params["salutation"] = p.Salutation
	}

	for _, identifier := range p.Identifiers {
		if identifier.Authority == fsAuthority {
			params["factsetIdentifier"] = identifier.IdentifierValue
		}
	}

	query := &neoism.CypherQuery{
		Statement: `MERGE (n:Thing {uuid: {uuid}})
					set n={allprops}
					set n :Concept
					set n :Person
		`,
		Parameters: map[string]interface{}{
			"uuid":     p.UUID,
			"allprops": params,
		},
	}

	return pcd.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})

}

func (pcd CypherDriver) Delete(uuid string) (bool, error) {
	clearNode := &neoism.CypherQuery{
		Statement: `
			MATCH (p:Thing {uuid: {uuid}})
			REMOVE p:Concept
			REMOVE p:Person
			SET p={props}
		`,
		Parameters: map[string]interface{}{
			"uuid": uuid,
			"props": map[string]interface{}{
				"uuid": uuid,
			},
		},
		IncludeStats: true,
	}

	removeNodeIfUnused := &neoism.CypherQuery{
		Statement: `
			MATCH (p:Thing {uuid: {uuid}})
			OPTIONAL MATCH (p)-[a]-(x)
			WITH p, count(a) AS relCount
			WHERE relCount = 0
			DELETE p
		`,
		Parameters: map[string]interface{}{
			"uuid": uuid,
		},
	}

	err := pcd.cypherRunner.CypherBatch([]*neoism.CypherQuery{clearNode, removeNodeIfUnused})

	s1, err := clearNode.Stats()
	if err != nil {
		return false, err
	}

	var deleted bool
	if s1.ContainsUpdates && s1.LabelsRemoved > 0 {
		deleted = true
	}

	return deleted, err
}

func (pcd CypherDriver) DecodeJSON(dec *json.Decoder) (interface{}, string, error) {
	p := person{}
	err := dec.Decode(&p)
	return p, p.UUID, err

}

func (pcd CypherDriver) Check() (check v1a.Check) {
	type hcUUIDResult struct {
		UUID string `json:"uuid"`
	}

	checker := func() (string, error) {
		var result []hcUUIDResult

		query := &neoism.CypherQuery{
			Statement: `MATCH (n:Person)
					return  n.uuid as uuid
					limit 1`,
			Result: &result,
		}

		err := pcd.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})

		if err != nil {
			return "", err
		}
		if len(result) == 0 {
			return "", errors.New("No Person found")
		}
		if result[0].UUID == "" {
			return "", errors.New("UUID not set")
		}
		return fmt.Sprintf("Found a person with a valid uuid = %v", result[0].UUID), nil
	}

	return v1a.Check{
		BusinessImpact:   "Cannot read/write people via this writer",
		Name:             "Check connectivity to Neo4j - neoUrl is a parameter in hieradata for this service",
		PanicGuide:       "TODO - write panic guide",
		Severity:         1,
		TechnicalSummary: fmt.Sprintf("Cannot connect to Neo4j instance %s with at least one person loaded in it", pcd.cypherRunner),
		Checker:          checker,
	}
}

const (
	fsAuthority = "http://api.ft.com/system/FACTSET-PPL"
)
<commit_msg>added Count() method<commit_after>package people

import (
	"encoding/json"
	"errors"
	"fmt"

	"github.com/Financial-Times/go-fthealth/v1a"
	"github.com/Financial-Times/neo-cypher-runner-go"
	"github.com/jmcvetta/neoism"
)

type CypherDriver struct {
	cypherRunner neocypherrunner.CypherRunner
}

func NewCypherDriver(cypherRunner neocypherrunner.CypherRunner) CypherDriver {
	return CypherDriver{cypherRunner}
}

func (pcd CypherDriver) Read(uuid string) (interface{}, bool, error) {
	results := []struct {
		UUID              string `json:"uuid"`
		Name              string `json:"name"`
		BirthYear         int    `json:"birthYear"`
		Salutation        string `json:"salutation"`
		FactsetIdentifier string `json:"factsetIdentifier"`
	}{}

	query := &neoism.CypherQuery{
		Statement: `MATCH (n:Person {uuid:{uuid}}) return n.uuid
		as uuid, n.name as name,
		n.factsetIdentifier as factsetIdentifier,
		n.birthYear as birthYear,
		n.salutation as salutation`,
		Parameters: map[string]interface{}{
			"uuid": uuid,
		},
		Result: &results,
	}

	err := pcd.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})

	if err != nil {
		return person{}, false, err
	}

	if len(results) == 0 {
		return person{}, false, nil
	}

	result := results[0]

	p := person{
		UUID:       result.UUID,
		Name:       result.Name,
		BirthYear:  result.BirthYear,
		Salutation: result.Salutation,
	}

	if result.FactsetIdentifier != "" {
		p.Identifiers = append(p.Identifiers, identifier{fsAuthority, result.FactsetIdentifier})
	}

	return p, true, nil

}

func (pcd CypherDriver) Write(thing interface{}) error {

	p := thing.(person)

	params := map[string]interface{}{
		"uuid": p.UUID,
	}

	if p.Name != "" {
		params["name"] = p.Name
	}

	if p.BirthYear != 0 {
		params["birthYear"] = p.BirthYear
	}

	if p.Salutation != "" {
		params["salutation"] = p.Salutation
	}

	for _, identifier := range p.Identifiers {
		if identifier.Authority == fsAuthority {
			params["factsetIdentifier"] = identifier.IdentifierValue
		}
	}

	query := &neoism.CypherQuery{
		Statement: `MERGE (n:Thing {uuid: {uuid}})
					set n={allprops}
					set n :Concept
					set n :Person
		`,
		Parameters: map[string]interface{}{
			"uuid":     p.UUID,
			"allprops": params,
		},
	}

	return pcd.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})

}

func (pcd CypherDriver) Delete(uuid string) (bool, error) {
	clearNode := &neoism.CypherQuery{
		Statement: `
			MATCH (p:Thing {uuid: {uuid}})
			REMOVE p:Concept
			REMOVE p:Person
			SET p={props}
		`,
		Parameters: map[string]interface{}{
			"uuid": uuid,
			"props": map[string]interface{}{
				"uuid": uuid,
			},
		},
		IncludeStats: true,
	}

	removeNodeIfUnused := &neoism.CypherQuery{
		Statement: `
			MATCH (p:Thing {uuid: {uuid}})
			OPTIONAL MATCH (p)-[a]-(x)
			WITH p, count(a) AS relCount
			WHERE relCount = 0
			DELETE p
		`,
		Parameters: map[string]interface{}{
			"uuid": uuid,
		},
	}

	err := pcd.cypherRunner.CypherBatch([]*neoism.CypherQuery{clearNode, removeNodeIfUnused})

	s1, err := clearNode.Stats()
	if err != nil {
		return false, err
	}

	var deleted bool
	if s1.ContainsUpdates && s1.LabelsRemoved > 0 {
		deleted = true
	}

	return deleted, err
}

func (pcd CypherDriver) DecodeJSON(dec *json.Decoder) (interface{}, string, error) {
	p := person{}
	err := dec.Decode(&p)
	return p, p.UUID, err

}

func (pcd CypherDriver) Check() (check v1a.Check) {
	type hcUUIDResult struct {
		UUID string `json:"uuid"`
	}

	checker := func() (string, error) {
		var result []hcUUIDResult

		query := &neoism.CypherQuery{
			Statement: `MATCH (n:Person)
					return  n.uuid as uuid
					limit 1`,
			Result: &result,
		}

		err := pcd.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})

		if err != nil {
			return "", err
		}
		if len(result) == 0 {
			return "", errors.New("No Person found")
		}
		if result[0].UUID == "" {
			return "", errors.New("UUID not set")
		}
		return fmt.Sprintf("Found a person with a valid uuid = %v", result[0].UUID), nil
	}

	return v1a.Check{
		BusinessImpact:   "Cannot read/write people via this writer",
		Name:             "Check connectivity to Neo4j - neoUrl is a parameter in hieradata for this service",
		PanicGuide:       "TODO - write panic guide",
		Severity:         1,
		TechnicalSummary: fmt.Sprintf("Cannot connect to Neo4j instance %s with at least one person loaded in it", pcd.cypherRunner),
		Checker:          checker,
	}
}

func (pcd CypherDriver) Count() (int, error) {

	results := []struct {
		Count int `json:"c"`
	}{}

	query := &neoism.CypherQuery{
		Statement: `MATCH (n:Person) return count(n) as c`,
		Result:    &results,
	}

	err := pcd.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})

	if err != nil {
		return 0, err
	}

	return results[0].Count, nil
}

const (
	fsAuthority = "http://api.ft.com/system/FACTSET-PPL"
)
<|endoftext|>


<commit_before>package healthcheck_test

import (
	"fmt"
	"net"
	"net/http"
	"runtime"
	"time"

	"code.cloudfoundry.org/healthcheck"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/ghttp"
)

var _ = Describe("HealthCheck", func() {
	itReturnsHealthCheckError := func(healthCheck func() error, code int, reason string) {
		err := healthCheck()
		Expect(err).To(HaveOccurred())
		Expect(err).To(BeAssignableToTypeOf(healthcheck.HealthCheckError{}))
		hErr := err.(healthcheck.HealthCheckError)
		Expect(hErr.Code).To(Equal(code))
		Expect(hErr.Message).To(ContainSubstring(reason))
	}

	var (
		server     *ghttp.Server
		serverAddr string

		ip string

		uri         string
		port        string
		timeout     time.Duration
		serverDelay time.Duration

		hc      healthcheck.HealthCheck
		handler func(http.ResponseWriter, *http.Request)
	)

	BeforeEach(func() {
		ip = getNonLoopbackIP()
		server = ghttp.NewUnstartedServer()

		listener, err := net.Listen("tcp", ip+":0")
		Expect(err).NotTo(HaveOccurred())

		timeout = 100 * time.Millisecond
		serverDelay = 0

		server.HTTPTestServer.Listener = listener
		serverAddr = listener.Addr().String()
		_, port, err = net.SplitHostPort(serverAddr)
		Expect(err).NotTo(HaveOccurred())

		handler = func(http.ResponseWriter, *http.Request) {
			time.Sleep(serverDelay)
		}

	})

	JustBeforeEach(func() {

		server.RouteToHandler("GET", "/api/_ping", handler)
		server.Start()

		hc = healthcheck.NewHealthCheck("tcp", uri, port, timeout)
	})

	AfterEach(func() {
		if server != nil {
			server.CloseClientConnections()
			server.Close()
		}
	})

	Describe("check interfaces", func() {
		It("succeeds when there are healthy interfaces", func() {
			interfaces, err := net.Interfaces()
			Expect(err).NotTo(HaveOccurred())

			err = hc.CheckInterfaces(interfaces)
			Expect(err).NotTo(HaveOccurred())
		})

		Context("when the server is failing", func() {
			AfterEach(func() {
				server = nil
			})

			It("fails appropriately when there are unhealthy interfaces", func() {
				server.Close()

				interfaces, err := net.Interfaces()
				Expect(err).NotTo(HaveOccurred())

				err = hc.CheckInterfaces(interfaces)
				Expect(err).To(HaveOccurred())
				Expect(err).To(BeAssignableToTypeOf(healthcheck.HealthCheckError{}))

				hErr := err.(healthcheck.HealthCheckError)
				// fails with different error codes on Linux (4) or OSX (64)
				// check to see it was not the NO interfaces error (3)
				Expect(hErr.Code).ToNot(Equal(3))
			})
		})

		It("fails appropriately when there are no interfaces", func() {
			err := hc.CheckInterfaces(nil)
			Expect(err).To(HaveOccurred())
			Expect(err).To(BeAssignableToTypeOf(healthcheck.HealthCheckError{}))

			hErr := err.(healthcheck.HealthCheckError)
			Expect(hErr.Code).To(Equal(3))
			Expect(hErr.Message).To(ContainSubstring("failure to find suitable interface"))
		})
	})

	Describe("port healthcheck", func() {
		portHealthCheck := func() error {
			return hc.PortHealthCheck(ip)
		}

		BeforeEach(func() {
			uri = ""
		})

		Context("when the address is listening", func() {
			It("succeeds", func() {
				Expect(portHealthCheck()).To(Succeed())
			})
		})

		Context("when the address is not listening", func() {
			BeforeEach(func() {
				port = "-1"
			})

			It("returns healthcheck error with code 4 with an appropriate message", func() {
				errMsg := fmt.Sprintf(
					"Failed to make TCP connection to port %s: connection refused",
					port,
				)
				itReturnsHealthCheckError(portHealthCheck, 4, errMsg)
			})
		})

		Context("when the server is slow in responding", func() {
			BeforeEach(func() {
				if runtime.GOOS == "windows" {
					Skip("TCP timeout tests are unreliable on Windows")
				}

				timeout = time.Nanosecond
			})

			It("returns healthcheck error with code 64 with an appropriate message", func() {
				errMsg := fmt.Sprintf(
					"Failed to make TCP connection to port %s: timed out after %.2f seconds",
					port,
					timeout.Seconds(),
				)
				itReturnsHealthCheckError(portHealthCheck, 64, errMsg)
			})
		})
	})

	Describe("http healthcheck", func() {
		httpHealthCheck := func() error {
			return hc.HTTPHealthCheck(ip)
		}

		BeforeEach(func() {
			uri = "/api/_ping"
		})

		Context("when the healthcheck is properly invoked", func() {
			Context("when the address is listening", func() {
				It("succeeds", func() {
					Expect(httpHealthCheck()).To(Succeed())
				})
			})

			Context("when the address returns error http code", func() {
				JustBeforeEach(func() {
					server.RouteToHandler("GET", "/api/_ping", ghttp.RespondWith(500, ""))
				})

				It("returns healthcheck error with code 6 with an appropriate message", func() {
					errMsg := fmt.Sprintf(
						"Failed to make HTTP request to '%s' on port %s: received status code 500 in",
						uri,
						port,
					)
					itReturnsHealthCheckError(httpHealthCheck, 6, errMsg)
				})
			})

			Context("when the address is not listening", func() {
				BeforeEach(func() {
					port = "-1"
				})

				It("returns healthcheck error with code 5 with an appropriate message", func() {
					errMsg := fmt.Sprintf(
						"Failed to make HTTP request to '%s' on port %s: connection refused",
						uri,
						port,
					)
					itReturnsHealthCheckError(httpHealthCheck, 5, errMsg)
				})
			})

			Context("when the server is too slow to respond", func() {
				BeforeEach(func() {
					timeout = time.Nanosecond
					serverDelay = time.Second
				})

				It("returns healthcheck error with code 65 with an appropriate message", func() {
					errMsg := fmt.Sprintf(
						"Failed to make HTTP request to '%s' on port %s: timed out after %.2f seconds",
						uri,
						port,
						timeout.Seconds(),
					)
					itReturnsHealthCheckError(httpHealthCheck, 65, errMsg)
				})
			})

			Context("with a tls-aware endpoint", func() {
				var request *http.Request
				BeforeEach(func() {
					uri = "/api/_ping"
					handler = func(resp http.ResponseWriter, req *http.Request) {
						request = req
						time.Sleep(serverDelay)
					}
				})

				It("should set the x-Forwarded-proto header", func() {
					err := hc.HTTPHealthCheck(ip)
					Expect(err).NotTo(HaveOccurred())
					Expect(request.Header.Get("X-Forwarded-Proto")).To(Equal("https"))
				})
			})

		})
	})
})

func getNonLoopbackIP() string {
	interfaces, err := net.Interfaces()
	Expect(err).NotTo(HaveOccurred())
	for _, intf := range interfaces {
		addrs, err := intf.Addrs()
		if err != nil {
			continue
		}

		for _, a := range addrs {
			if ipnet, ok := a.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
				if ipnet.IP.To4() != nil {
					return ipnet.IP.String()
				}
			}
		}
	}
	Fail("no non-loopback address found")
	panic("non-reachable")
}
<commit_msg>Go 1.12.9 is URL validating the port<commit_after>package healthcheck_test

import (
	"fmt"
	"net"
	"net/http"
	"runtime"
	"time"

	"code.cloudfoundry.org/healthcheck"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/ghttp"
)

var _ = Describe("HealthCheck", func() {
	itReturnsHealthCheckError := func(healthCheck func() error, code int, reason string) {
		err := healthCheck()
		Expect(err).To(HaveOccurred())
		Expect(err).To(BeAssignableToTypeOf(healthcheck.HealthCheckError{}))
		hErr := err.(healthcheck.HealthCheckError)
		Expect(hErr.Code).To(Equal(code))
		Expect(hErr.Message).To(ContainSubstring(reason))
	}

	var (
		server     *ghttp.Server
		serverAddr string

		ip string

		uri         string
		port        string
		timeout     time.Duration
		serverDelay time.Duration

		hc      healthcheck.HealthCheck
		handler func(http.ResponseWriter, *http.Request)
	)

	BeforeEach(func() {
		ip = getNonLoopbackIP()
		server = ghttp.NewUnstartedServer()

		listener, err := net.Listen("tcp", ip+":0")
		Expect(err).NotTo(HaveOccurred())

		timeout = 100 * time.Millisecond
		serverDelay = 0

		server.HTTPTestServer.Listener = listener
		serverAddr = listener.Addr().String()
		_, port, err = net.SplitHostPort(serverAddr)
		Expect(err).NotTo(HaveOccurred())

		handler = func(http.ResponseWriter, *http.Request) {
			time.Sleep(serverDelay)
		}

	})

	JustBeforeEach(func() {

		server.RouteToHandler("GET", "/api/_ping", handler)
		server.Start()

		hc = healthcheck.NewHealthCheck("tcp", uri, port, timeout)
	})

	AfterEach(func() {
		if server != nil {
			server.CloseClientConnections()
			server.Close()
		}
	})

	Describe("check interfaces", func() {
		It("succeeds when there are healthy interfaces", func() {
			interfaces, err := net.Interfaces()
			Expect(err).NotTo(HaveOccurred())

			err = hc.CheckInterfaces(interfaces)
			Expect(err).NotTo(HaveOccurred())
		})

		Context("when the server is failing", func() {
			AfterEach(func() {
				server = nil
			})

			It("fails appropriately when there are unhealthy interfaces", func() {
				server.Close()

				interfaces, err := net.Interfaces()
				Expect(err).NotTo(HaveOccurred())

				err = hc.CheckInterfaces(interfaces)
				Expect(err).To(HaveOccurred())
				Expect(err).To(BeAssignableToTypeOf(healthcheck.HealthCheckError{}))

				hErr := err.(healthcheck.HealthCheckError)
				// fails with different error codes on Linux (4) or OSX (64)
				// check to see it was not the NO interfaces error (3)
				Expect(hErr.Code).ToNot(Equal(3))
			})
		})

		It("fails appropriately when there are no interfaces", func() {
			err := hc.CheckInterfaces(nil)
			Expect(err).To(HaveOccurred())
			Expect(err).To(BeAssignableToTypeOf(healthcheck.HealthCheckError{}))

			hErr := err.(healthcheck.HealthCheckError)
			Expect(hErr.Code).To(Equal(3))
			Expect(hErr.Message).To(ContainSubstring("failure to find suitable interface"))
		})
	})

	Describe("port healthcheck", func() {
		portHealthCheck := func() error {
			return hc.PortHealthCheck(ip)
		}

		BeforeEach(func() {
			uri = ""
		})

		Context("when the address is listening", func() {
			It("succeeds", func() {
				Expect(portHealthCheck()).To(Succeed())
			})
		})

		Context("when the address is not listening", func() {
			BeforeEach(func() {
				port = "-1"
			})

			It("returns healthcheck error with code 4 with an appropriate message", func() {
				errMsg := fmt.Sprintf(
					"Failed to make TCP connection to port %s: connection refused",
					port,
				)
				itReturnsHealthCheckError(portHealthCheck, 4, errMsg)
			})
		})

		Context("when the server is slow in responding", func() {
			BeforeEach(func() {
				if runtime.GOOS == "windows" {
					Skip("TCP timeout tests are unreliable on Windows")
				}

				timeout = time.Nanosecond
			})

			It("returns healthcheck error with code 64 with an appropriate message", func() {
				errMsg := fmt.Sprintf(
					"Failed to make TCP connection to port %s: timed out after %.2f seconds",
					port,
					timeout.Seconds(),
				)
				itReturnsHealthCheckError(portHealthCheck, 64, errMsg)
			})
		})
	})

	Describe("http healthcheck", func() {
		httpHealthCheck := func() error {
			return hc.HTTPHealthCheck(ip)
		}

		BeforeEach(func() {
			uri = "/api/_ping"
		})

		Context("when the healthcheck is properly invoked", func() {
			Context("when the address is listening", func() {
				It("succeeds", func() {
					Expect(httpHealthCheck()).To(Succeed())
				})
			})

			Context("when the address returns error http code", func() {
				JustBeforeEach(func() {
					server.RouteToHandler("GET", "/api/_ping", ghttp.RespondWith(500, ""))
				})

				It("returns healthcheck error with code 6 with an appropriate message", func() {
					errMsg := fmt.Sprintf(
						"Failed to make HTTP request to '%s' on port %s: received status code 500 in",
						uri,
						port,
					)
					itReturnsHealthCheckError(httpHealthCheck, 6, errMsg)
				})
			})

			Context("when the address is not listening", func() {
				BeforeEach(func() {
					port = "9876"
				})

				It("returns healthcheck error with code 5 with an appropriate message", func() {
					errMsg := fmt.Sprintf(
						"Failed to make HTTP request to '%s' on port %s: connection refused",
						uri,
						port,
					)
					itReturnsHealthCheckError(httpHealthCheck, 5, errMsg)
				})
			})

			Context("when the server is too slow to respond", func() {
				BeforeEach(func() {
					timeout = time.Nanosecond
					serverDelay = time.Second
				})

				It("returns healthcheck error with code 65 with an appropriate message", func() {
					errMsg := fmt.Sprintf(
						"Failed to make HTTP request to '%s' on port %s: timed out after %.2f seconds",
						uri,
						port,
						timeout.Seconds(),
					)
					itReturnsHealthCheckError(httpHealthCheck, 65, errMsg)
				})
			})

			Context("with a tls-aware endpoint", func() {
				var request *http.Request
				BeforeEach(func() {
					uri = "/api/_ping"
					handler = func(resp http.ResponseWriter, req *http.Request) {
						request = req
						time.Sleep(serverDelay)
					}
				})

				It("should set the x-Forwarded-proto header", func() {
					err := hc.HTTPHealthCheck(ip)
					Expect(err).NotTo(HaveOccurred())
					Expect(request.Header.Get("X-Forwarded-Proto")).To(Equal("https"))
				})
			})

		})
	})
})

func getNonLoopbackIP() string {
	interfaces, err := net.Interfaces()
	Expect(err).NotTo(HaveOccurred())
	for _, intf := range interfaces {
		addrs, err := intf.Addrs()
		if err != nil {
			continue
		}

		for _, a := range addrs {
			if ipnet, ok := a.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
				if ipnet.IP.To4() != nil {
					return ipnet.IP.String()
				}
			}
		}
	}
	Fail("no non-loopback address found")
	panic("non-reachable")
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/gob"
	"fmt"
	"$GO_BOOTSTRAP_REPO_NAME/$GO_BOOTSTRAP_REPO_USER/$GO_BOOTSTRAP_PROJECT_NAME/dal"
	"$GO_BOOTSTRAP_REPO_NAME/$GO_BOOTSTRAP_REPO_USER/$GO_BOOTSTRAP_PROJECT_NAME/handlers"
	"$GO_BOOTSTRAP_REPO_NAME/$GO_BOOTSTRAP_REPO_USER/$GO_BOOTSTRAP_PROJECT_NAME/libenv"
	"$GO_BOOTSTRAP_REPO_NAME/$GO_BOOTSTRAP_REPO_USER/$GO_BOOTSTRAP_PROJECT_NAME/libunix"
	"$GO_BOOTSTRAP_REPO_NAME/$GO_BOOTSTRAP_REPO_USER/$GO_BOOTSTRAP_PROJECT_NAME/middlewares"
	"github.com/carbocation/interpose"
	gorilla_mux "github.com/gorilla/mux"
	"github.com/gorilla/sessions"
	"github.com/jmoiron/sqlx"
	_ "github.com/lib/pq"
	"github.com/Sirupsen/logrus"
	"github.com/tylerb/graceful"
	"net/http"
	"time"
)

func init() {
	gob.Register(&dal.UserRow{})
}

// NewApplication is the constructor for Application struct.
func NewApplication() (*Application, error) {
	u, err := libunix.CurrentUser()
	if err != nil {
		return nil, err
	}

	dsn := libenv.EnvWithDefault("DSN", fmt.Sprintf("postgres://%v@localhost:5432/$GO_BOOTSTRAP_PROJECT_NAME?sslmode=disable", u))

	db, err := sqlx.Connect("postgres", dsn)
	if err != nil {
		return nil, err
	}

	cookieStoreSecret := libenv.EnvWithDefault("COOKIE_SECRET", "$GO_BOOTSTRAP_COOKIE_SECRET")

	app := &Application{}
	app.dsn = dsn
	app.db = db
	app.cookieStore = sessions.NewCookieStore([]byte(cookieStoreSecret))

	return app, err
}

// Application is the application object that runs HTTP server.
type Application struct {
	dsn         string
	db          *sqlx.DB
	cookieStore *sessions.CookieStore
}

func (app *Application) middlewareStruct() (*interpose.Middleware, error) {
	middle := interpose.New()
	middle.Use(middlewares.SetDB(app.db))
	middle.Use(middlewares.SetCookieStore(app.cookieStore))

	middle.UseHandler(app.mux())

	return middle, nil
}

func (app *Application) mux() *gorilla_mux.Router {
	MustLogin := middlewares.MustLogin

	router := gorilla_mux.NewRouter()

	router.Handle("/", MustLogin(http.HandlerFunc(handlers.GetHome))).Methods("GET")

	router.HandleFunc("/signup", handlers.GetSignup).Methods("GET")
	router.HandleFunc("/signup", handlers.PostSignup).Methods("POST")
	router.HandleFunc("/login", handlers.GetLogin).Methods("GET")
	router.HandleFunc("/login", handlers.PostLogin).Methods("POST")
	router.HandleFunc("/logout", handlers.GetLogout).Methods("GET")

	router.Handle("/users/{id:[0-9]+}", MustLogin(http.HandlerFunc(handlers.PostPutDeleteUsersID))).Methods("POST", "PUT", "DELETE")

	// Path of static files must be last!
	router.PathPrefix("/").Handler(http.FileServer(http.Dir("static")))

	return router
}

func main() {
	app, err := NewApplication()
	if err != nil {
		logrus.Fatal(err.Error())
	}

	middle, err := app.middlewareStruct()
	if err != nil {
		logrus.Fatal(err.Error())
	}

	serverAddress := libenv.EnvWithDefault("HTTP_ADDR", ":8888")
	certFile := libenv.EnvWithDefault("HTTP_CERT_FILE", "")
	keyFile := libenv.EnvWithDefault("HTTP_KEY_FILE", "")
	drainIntervalString := libenv.EnvWithDefault("HTTP_DRAIN_INTERVAL", "1s")

	drainInterval, err := time.ParseDuration(drainIntervalString)
	if err != nil {
		logrus.Fatal(err.Error())
	}

	srv := &graceful.Server{
		Timeout: drainInterval,
		Server:  &http.Server{Addr: serverAddress, Handler: middle},
	}

	logrus.Infoln("Running HTTP server on "+ serverAddress)
	if certFile != "" && keyFile != "" {
		srv.ListenAndServeTLS(certFile, keyFile)
	} else {
		srv.ListenAndServe()
	}
}
<commit_msg>check for return error on ListenAndServe()<commit_after>package main

import (
	"encoding/gob"
	"fmt"
	"$GO_BOOTSTRAP_REPO_NAME/$GO_BOOTSTRAP_REPO_USER/$GO_BOOTSTRAP_PROJECT_NAME/dal"
	"$GO_BOOTSTRAP_REPO_NAME/$GO_BOOTSTRAP_REPO_USER/$GO_BOOTSTRAP_PROJECT_NAME/handlers"
	"$GO_BOOTSTRAP_REPO_NAME/$GO_BOOTSTRAP_REPO_USER/$GO_BOOTSTRAP_PROJECT_NAME/libenv"
	"$GO_BOOTSTRAP_REPO_NAME/$GO_BOOTSTRAP_REPO_USER/$GO_BOOTSTRAP_PROJECT_NAME/libunix"
	"$GO_BOOTSTRAP_REPO_NAME/$GO_BOOTSTRAP_REPO_USER/$GO_BOOTSTRAP_PROJECT_NAME/middlewares"
	"github.com/carbocation/interpose"
	gorilla_mux "github.com/gorilla/mux"
	"github.com/gorilla/sessions"
	"github.com/jmoiron/sqlx"
	_ "github.com/lib/pq"
	"github.com/Sirupsen/logrus"
	"github.com/tylerb/graceful"
	"net/http"
	"time"
)

func init() {
	gob.Register(&dal.UserRow{})
}

// NewApplication is the constructor for Application struct.
func NewApplication() (*Application, error) {
	u, err := libunix.CurrentUser()
	if err != nil {
		return nil, err
	}

	dsn := libenv.EnvWithDefault("DSN", fmt.Sprintf("postgres://%v@localhost:5432/$GO_BOOTSTRAP_PROJECT_NAME?sslmode=disable", u))

	db, err := sqlx.Connect("postgres", dsn)
	if err != nil {
		return nil, err
	}

	cookieStoreSecret := libenv.EnvWithDefault("COOKIE_SECRET", "$GO_BOOTSTRAP_COOKIE_SECRET")

	app := &Application{}
	app.dsn = dsn
	app.db = db
	app.cookieStore = sessions.NewCookieStore([]byte(cookieStoreSecret))

	return app, err
}

// Application is the application object that runs HTTP server.
type Application struct {
	dsn         string
	db          *sqlx.DB
	cookieStore *sessions.CookieStore
}

func (app *Application) middlewareStruct() (*interpose.Middleware, error) {
	middle := interpose.New()
	middle.Use(middlewares.SetDB(app.db))
	middle.Use(middlewares.SetCookieStore(app.cookieStore))

	middle.UseHandler(app.mux())

	return middle, nil
}

func (app *Application) mux() *gorilla_mux.Router {
	MustLogin := middlewares.MustLogin

	router := gorilla_mux.NewRouter()

	router.Handle("/", MustLogin(http.HandlerFunc(handlers.GetHome))).Methods("GET")

	router.HandleFunc("/signup", handlers.GetSignup).Methods("GET")
	router.HandleFunc("/signup", handlers.PostSignup).Methods("POST")
	router.HandleFunc("/login", handlers.GetLogin).Methods("GET")
	router.HandleFunc("/login", handlers.PostLogin).Methods("POST")
	router.HandleFunc("/logout", handlers.GetLogout).Methods("GET")

	router.Handle("/users/{id:[0-9]+}", MustLogin(http.HandlerFunc(handlers.PostPutDeleteUsersID))).Methods("POST", "PUT", "DELETE")

	// Path of static files must be last!
	router.PathPrefix("/").Handler(http.FileServer(http.Dir("static")))

	return router
}

func main() {
	app, err := NewApplication()
	if err != nil {
		logrus.Fatal(err.Error())
	}

	middle, err := app.middlewareStruct()
	if err != nil {
		logrus.Fatal(err.Error())
	}

	serverAddress := libenv.EnvWithDefault("HTTP_ADDR", ":8888")
	certFile := libenv.EnvWithDefault("HTTP_CERT_FILE", "")
	keyFile := libenv.EnvWithDefault("HTTP_KEY_FILE", "")
	drainIntervalString := libenv.EnvWithDefault("HTTP_DRAIN_INTERVAL", "1s")

	drainInterval, err := time.ParseDuration(drainIntervalString)
	if err != nil {
		logrus.Fatal(err.Error())
	}

	srv := &graceful.Server{
		Timeout: drainInterval,
		Server:  &http.Server{Addr: serverAddress, Handler: middle},
	}

	logrus.Infoln("Running HTTP server on "+ serverAddress)
	if certFile != "" && keyFile != "" {
		err = srv.ListenAndServeTLS(certFile, keyFile)
	} else {
		err = srv.ListenAndServe()
	}
	if err != nil {
		logrus.Fatal(err.Error())
	}
}
<|endoftext|>


<commit_before>package shed

import (
	"fmt"
	"log"

	"github.com/frozzare/shed/app"
	"github.com/frozzare/shed/config"
	"github.com/frozzare/shed/docker"
	"github.com/frozzare/shed/repository"
	cli "gopkg.in/urfave/cli.v1"
)

var DeployCmd = cli.Command{
	Name:   "deploy",
	Usage:  "deploy",
	Action: deploy,
	Flags: []cli.Flag{
		cli.BoolFlag{
			Name: "debug",
		},
	},
}

func rerr(c *cli.Context, err error) {
	if c.Bool("debug") {
		log.Fatal(err)
	} else {
		fmt.Printf("==> error: %s\n", err.Error())
		return
	}
}

func deploy(c *cli.Context) {
	// Load configuration.
	config, err := config.NewConfig()
	if err != nil {
		rerr(c, err)
		return
	}

	// Load git repository.
	repo, err := repository.NewRepository(config.Git)
	if err != nil {
		rerr(c, err)
		return
	}

	// Create a new application.
	app, err := app.NewApp(&app.Options{
		Config:     config,
		Repository: repo,
	})

	fmt.Printf("==>    shed: deploying %s\n", app.Domain())

	// Connect to docker.
	fmt.Println("==>  docker: connecting to docker remote api")
	dock, err := docker.NewDocker(app)
	if err != nil {
		rerr(c, err)
		return
	}

	// Prune removes all unused containers, volumes, networks and images (both dangling and unreferenced).
	fmt.Println("==>  docker: system pruning")
	if err := dock.Prune(); err != nil {
		fmt.Printf("==>    error: %s\n", err.Error())
	} else {
		fmt.Println("==>  docker: system pruned")
	}

	// Start nginx proxy container if it don't exists.
	if err := dock.StartNginxContainer(); err != nil {
		fmt.Printf("==>    error: %s\n", err.Error())
	} else {
		fmt.Println("==>  docker: nginx proxy container is created or already exists")
	}

	// Run docker-compose command.
	cmd := fmt.Sprintf("docker-compose -H %s up -d --force-recreate", docker.Endpoint())
	if err := docker.ExecCmd(cmd, true); err != nil {
		fmt.Printf("==> %s\n", err.Error())
	}

	fmt.Printf("==>    shed: done, http://%s is now up\n", app.Domain())
}
<commit_msg>Pass right value to NewDocker<commit_after>package shed

import (
	"fmt"
	"log"

	"github.com/frozzare/shed/app"
	"github.com/frozzare/shed/config"
	"github.com/frozzare/shed/docker"
	"github.com/frozzare/shed/repository"
	cli "gopkg.in/urfave/cli.v1"
)

var DeployCmd = cli.Command{
	Name:   "deploy",
	Usage:  "deploy",
	Action: deploy,
	Flags: []cli.Flag{
		cli.BoolFlag{
			Name: "debug",
		},
	},
}

func rerr(c *cli.Context, err error) {
	if c.Bool("debug") {
		log.Fatal(err)
	} else {
		fmt.Printf("==> error: %s\n", err.Error())
		return
	}
}

func deploy(c *cli.Context) {
	// Load configuration.
	config, err := config.NewConfig()
	if err != nil {
		rerr(c, err)
		return
	}

	// Load git repository.
	repo, err := repository.NewRepository(config.Git)
	if err != nil {
		rerr(c, err)
		return
	}

	// Create a new application.
	app, err := app.NewApp(&app.Options{
		Config:     config,
		Repository: repo,
	})

	fmt.Printf("==>    shed: deploying %s\n", app.Domain())

	// Connect to docker.
	fmt.Println("==>  docker: connecting to docker remote api")
	dock, err := docker.NewDocker(app.Config().Docker)
	if err != nil {
		rerr(c, err)
		return
	}

	// Prune removes all unused containers, volumes, networks and images (both dangling and unreferenced).
	fmt.Println("==>  docker: system pruning")
	if err := dock.Prune(); err != nil {
		fmt.Printf("==>    error: %s\n", err.Error())
	} else {
		fmt.Println("==>  docker: system pruned")
	}

	// Start nginx proxy container if it don't exists.
	if err := dock.StartNginxContainer(); err != nil {
		fmt.Printf("==>    error: %s\n", err.Error())
	} else {
		fmt.Println("==>  docker: nginx proxy container is created or already exists")
	}

	// Run docker-compose command.
	cmd := fmt.Sprintf("docker-compose -H %s up -d --force-recreate", docker.Endpoint())
	if err := docker.ExecCmd(cmd, true); err != nil {
		fmt.Printf("==> %s\n", err.Error())
	}

	fmt.Printf("==>    shed: done, http://%s is now up\n", app.Domain())
}
<|endoftext|>


<commit_before>// Copyright 2016 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
Package storage provides an easy way to work with Google Cloud Storage.
Google Cloud Storage stores data in named objects, which are grouped into buckets.

More information about Google Cloud Storage is available at
https://cloud.google.com/storage/docs.

See https://godoc.org/cloud.google.com/go for authentication, timeouts,
connection pooling and similar aspects of this package.

All of the methods of this package use exponential backoff to retry calls that fail
with certain errors, as described in
https://cloud.google.com/storage/docs/exponential-backoff. Retrying continues
indefinitely unless the controlling context is canceled or the client is closed. See
context.WithTimeout and context.WithCancel.

Creating a Client

To start working with this package, create a client:

    ctx := context.Background()
    client, err := storage.NewClient(ctx)
    if err != nil {
        // TODO: Handle error.
    }

The client will use your default application credentials. Clients should be
reused instead of created as needed. The methods of Client are safe for
concurrent use by multiple goroutines.

If you only wish to access public data, you can create
an unauthenticated client with

    client, err := storage.NewClient(ctx, option.WithoutAuthentication())

To use an emulator with this library, you can set the STORAGE_EMULATOR_HOST
environment variable to the address at which your emulator is running. This will
send requests to that address instead of to Cloud Storage. You can then create
and use a client as usual:

    // Set STORAGE_EMULATOR_HOST environment variable.
    err := os.Setenv("STORAGE_EMULATOR_HOST", "localhost:9000")
    if err != nil {
        // TODO: Handle error.
    }

    // Create client as usual.
    client, err := storage.NewClient(ctx)
    if err != nil {
        // TODO: Handle error.
    }

    // This request is now directed to http://localhost:9000/storage/v1/b
    // instead of https://storage.googleapis.com/storage/v1/b
    if err := client.Bucket("my-bucket").Create(ctx, projectID, nil); err != nil {
        // TODO: Handle error.
    }

Please note that there is no official emulator for Cloud Storage.

Buckets

A Google Cloud Storage bucket is a collection of objects. To work with a
bucket, make a bucket handle:

    bkt := client.Bucket(bucketName)

A handle is a reference to a bucket. You can have a handle even if the
bucket doesn't exist yet. To create a bucket in Google Cloud Storage,
call Create on the handle:

    if err := bkt.Create(ctx, projectID, nil); err != nil {
        // TODO: Handle error.
    }

Note that although buckets are associated with projects, bucket names are
global across all projects.

Each bucket has associated metadata, represented in this package by
BucketAttrs. The third argument to BucketHandle.Create allows you to set
the initial BucketAttrs of a bucket. To retrieve a bucket's attributes, use
Attrs:

    attrs, err := bkt.Attrs(ctx)
    if err != nil {
        // TODO: Handle error.
    }
    fmt.Printf("bucket %s, created at %s, is located in %s with storage class %s\n",
        attrs.Name, attrs.Created, attrs.Location, attrs.StorageClass)

Objects

An object holds arbitrary data as a sequence of bytes, like a file. You
refer to objects using a handle, just as with buckets, but unlike buckets
you don't explicitly create an object. Instead, the first time you write
to an object it will be created. You can use the standard Go io.Reader
and io.Writer interfaces to read and write object data:

    obj := bkt.Object("data")
    // Write something to obj.
    // w implements io.Writer.
    w := obj.NewWriter(ctx)
    // Write some text to obj. This will either create the object or overwrite whatever is there already.
    if _, err := fmt.Fprintf(w, "This object contains text.\n"); err != nil {
        // TODO: Handle error.
    }
    // Close, just like writing a file.
    if err := w.Close(); err != nil {
        // TODO: Handle error.
    }

    // Read it back.
    r, err := obj.NewReader(ctx)
    if err != nil {
        // TODO: Handle error.
    }
    defer r.Close()
    if _, err := io.Copy(os.Stdout, r); err != nil {
        // TODO: Handle error.
    }
    // Prints "This object contains text."

Objects also have attributes, which you can fetch with Attrs:

    objAttrs, err := obj.Attrs(ctx)
    if err != nil {
        // TODO: Handle error.
    }
    fmt.Printf("object %s has size %d and can be read using %s\n",
        objAttrs.Name, objAttrs.Size, objAttrs.MediaLink)

Listing objects

Listing objects in a bucket is done with the Bucket.Objects method:

    query := &storage.Query{Prefix: ""}

    var names []string
    it := bkt.Objects(ctx, query)
    for {
        attrs, err := it.Next()
        if err == iterator.Done {
            break
        }
        if err != nil {
            log.Fatal(err)
        }
        names = append(names, attrs.Name)
    }

Objects are listed lexicographically by name. To filter objects
lexicographically, Query.StartOffset and/or Query.EndOffset can be used:

    query := &storage.Query{
        Prefix: "",
        StartOffset: "bar/",  // Only list objects lexicographically >= "bar/"
        EndOffset: "foo/",    // Only list objects lexicographically < "foo/"
    }

    // ... as before

If only a subset of object attributes is needed when listing, specifying this
subset using Query.SetAttrSelection may speed up the listing process:

    query := &storage.Query{Prefix: ""}
    query.SetAttrSelection([]string{"Name"})

    // ... as before

ACLs

Both objects and buckets have ACLs (Access Control Lists). An ACL is a list of
ACLRules, each of which specifies the role of a user, group or project. ACLs
are suitable for fine-grained control, but you may prefer using IAM to control
access at the project level (see
https://cloud.google.com/storage/docs/access-control/iam).

To list the ACLs of a bucket or object, obtain an ACLHandle and call its List method:

    acls, err := obj.ACL().List(ctx)
    if err != nil {
        // TODO: Handle error.
    }
    for _, rule := range acls {
        fmt.Printf("%s has role %s\n", rule.Entity, rule.Role)
    }

You can also set and delete ACLs.

Conditions

Every object has a generation and a metageneration. The generation changes
whenever the content changes, and the metageneration changes whenever the
metadata changes. Conditions let you check these values before an operation;
the operation only executes if the conditions match. You can use conditions to
prevent race conditions in read-modify-write operations.

For example, say you've read an object's metadata into objAttrs. Now
you want to write to that object, but only if its contents haven't changed
since you read it. Here is how to express that:

    w = obj.If(storage.Conditions{GenerationMatch: objAttrs.Generation}).NewWriter(ctx)
    // Proceed with writing as above.

Signed URLs

You can obtain a URL that lets anyone read or write an object for a limited time.
You don't need to create a client to do this. See the documentation of
SignedURL for details.

    url, err := storage.SignedURL(bucketName, "shared-object", opts)
    if err != nil {
        // TODO: Handle error.
    }
    fmt.Println(url)

Post Policy V4 Signed Request

A type of signed request that allows uploads through HTML forms directly to Cloud Storage with
temporary permission. Conditions can be applied to restrict how the HTML form is used and exercised
by a user.

For more information, please see https://cloud.google.com/storage/docs/xml-api/post-object as well
as the documentation of GenerateSignedPostPolicyV4.

    pv4, err := storage.GenerateSignedPostPolicyV4(bucketName, objectName, opts)
    if err != nil {
        // TODO: Handle error.
    }
    fmt.Printf("URL: %s\nFields; %v\n", pv4.URL, pv4.Fields)

Errors

Errors returned by this client are often of the type [`googleapi.Error`](https://godoc.org/google.golang.org/api/googleapi#Error).
These errors can be introspected for more information by using `errors.As` with the richer `googleapi.Error` type. For example:

	var e *googleapi.Error
	if ok := errors.As(err, &e); ok {
		  if e.Code == 409 { ... }
	}
*/
package storage // import "cloud.google.com/go/storage"
<commit_msg>doc(storage): improve retry package docs (#5300)<commit_after>// Copyright 2016 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
Package storage provides an easy way to work with Google Cloud Storage.
Google Cloud Storage stores data in named objects, which are grouped into buckets.

More information about Google Cloud Storage is available at
https://cloud.google.com/storage/docs.

See https://pkg.go.dev/cloud.google.com/go for authentication, timeouts,
connection pooling and similar aspects of this package.

Creating a Client

To start working with this package, create a client:

    ctx := context.Background()
    client, err := storage.NewClient(ctx)
    if err != nil {
        // TODO: Handle error.
    }

The client will use your default application credentials. Clients should be
reused instead of created as needed. The methods of Client are safe for
concurrent use by multiple goroutines.

If you only wish to access public data, you can create
an unauthenticated client with

    client, err := storage.NewClient(ctx, option.WithoutAuthentication())

To use an emulator with this library, you can set the STORAGE_EMULATOR_HOST
environment variable to the address at which your emulator is running. This will
send requests to that address instead of to Cloud Storage. You can then create
and use a client as usual:

    // Set STORAGE_EMULATOR_HOST environment variable.
    err := os.Setenv("STORAGE_EMULATOR_HOST", "localhost:9000")
    if err != nil {
        // TODO: Handle error.
    }

    // Create client as usual.
    client, err := storage.NewClient(ctx)
    if err != nil {
        // TODO: Handle error.
    }

    // This request is now directed to http://localhost:9000/storage/v1/b
    // instead of https://storage.googleapis.com/storage/v1/b
    if err := client.Bucket("my-bucket").Create(ctx, projectID, nil); err != nil {
        // TODO: Handle error.
    }

Please note that there is no official emulator for Cloud Storage.

Buckets

A Google Cloud Storage bucket is a collection of objects. To work with a
bucket, make a bucket handle:

    bkt := client.Bucket(bucketName)

A handle is a reference to a bucket. You can have a handle even if the
bucket doesn't exist yet. To create a bucket in Google Cloud Storage,
call Create on the handle:

    if err := bkt.Create(ctx, projectID, nil); err != nil {
        // TODO: Handle error.
    }

Note that although buckets are associated with projects, bucket names are
global across all projects.

Each bucket has associated metadata, represented in this package by
BucketAttrs. The third argument to BucketHandle.Create allows you to set
the initial BucketAttrs of a bucket. To retrieve a bucket's attributes, use
Attrs:

    attrs, err := bkt.Attrs(ctx)
    if err != nil {
        // TODO: Handle error.
    }
    fmt.Printf("bucket %s, created at %s, is located in %s with storage class %s\n",
        attrs.Name, attrs.Created, attrs.Location, attrs.StorageClass)

Objects

An object holds arbitrary data as a sequence of bytes, like a file. You
refer to objects using a handle, just as with buckets, but unlike buckets
you don't explicitly create an object. Instead, the first time you write
to an object it will be created. You can use the standard Go io.Reader
and io.Writer interfaces to read and write object data:

    obj := bkt.Object("data")
    // Write something to obj.
    // w implements io.Writer.
    w := obj.NewWriter(ctx)
    // Write some text to obj. This will either create the object or overwrite whatever is there already.
    if _, err := fmt.Fprintf(w, "This object contains text.\n"); err != nil {
        // TODO: Handle error.
    }
    // Close, just like writing a file.
    if err := w.Close(); err != nil {
        // TODO: Handle error.
    }

    // Read it back.
    r, err := obj.NewReader(ctx)
    if err != nil {
        // TODO: Handle error.
    }
    defer r.Close()
    if _, err := io.Copy(os.Stdout, r); err != nil {
        // TODO: Handle error.
    }
    // Prints "This object contains text."

Objects also have attributes, which you can fetch with Attrs:

    objAttrs, err := obj.Attrs(ctx)
    if err != nil {
        // TODO: Handle error.
    }
    fmt.Printf("object %s has size %d and can be read using %s\n",
        objAttrs.Name, objAttrs.Size, objAttrs.MediaLink)

Listing objects

Listing objects in a bucket is done with the Bucket.Objects method:

    query := &storage.Query{Prefix: ""}

    var names []string
    it := bkt.Objects(ctx, query)
    for {
        attrs, err := it.Next()
        if err == iterator.Done {
            break
        }
        if err != nil {
            log.Fatal(err)
        }
        names = append(names, attrs.Name)
    }

Objects are listed lexicographically by name. To filter objects
lexicographically, Query.StartOffset and/or Query.EndOffset can be used:

    query := &storage.Query{
        Prefix: "",
        StartOffset: "bar/",  // Only list objects lexicographically >= "bar/"
        EndOffset: "foo/",    // Only list objects lexicographically < "foo/"
    }

    // ... as before

If only a subset of object attributes is needed when listing, specifying this
subset using Query.SetAttrSelection may speed up the listing process:

    query := &storage.Query{Prefix: ""}
    query.SetAttrSelection([]string{"Name"})

    // ... as before

ACLs

Both objects and buckets have ACLs (Access Control Lists). An ACL is a list of
ACLRules, each of which specifies the role of a user, group or project. ACLs
are suitable for fine-grained control, but you may prefer using IAM to control
access at the project level (see
https://cloud.google.com/storage/docs/access-control/iam).

To list the ACLs of a bucket or object, obtain an ACLHandle and call its List method:

    acls, err := obj.ACL().List(ctx)
    if err != nil {
        // TODO: Handle error.
    }
    for _, rule := range acls {
        fmt.Printf("%s has role %s\n", rule.Entity, rule.Role)
    }

You can also set and delete ACLs.

Conditions

Every object has a generation and a metageneration. The generation changes
whenever the content changes, and the metageneration changes whenever the
metadata changes. Conditions let you check these values before an operation;
the operation only executes if the conditions match. You can use conditions to
prevent race conditions in read-modify-write operations.

For example, say you've read an object's metadata into objAttrs. Now
you want to write to that object, but only if its contents haven't changed
since you read it. Here is how to express that:

    w = obj.If(storage.Conditions{GenerationMatch: objAttrs.Generation}).NewWriter(ctx)
    // Proceed with writing as above.

Signed URLs

You can obtain a URL that lets anyone read or write an object for a limited time.
You don't need to create a client to do this. See the documentation of
SignedURL for details.

    url, err := storage.SignedURL(bucketName, "shared-object", opts)
    if err != nil {
        // TODO: Handle error.
    }
    fmt.Println(url)

Post Policy V4 Signed Request

A type of signed request that allows uploads through HTML forms directly to Cloud Storage with
temporary permission. Conditions can be applied to restrict how the HTML form is used and exercised
by a user.

For more information, please see https://cloud.google.com/storage/docs/xml-api/post-object as well
as the documentation of GenerateSignedPostPolicyV4.

    pv4, err := storage.GenerateSignedPostPolicyV4(bucketName, objectName, opts)
    if err != nil {
        // TODO: Handle error.
    }
    fmt.Printf("URL: %s\nFields; %v\n", pv4.URL, pv4.Fields)

Errors

Errors returned by this client are often of the type googleapi.Error.
These errors can be introspected for more information by using errors.As
with the richer googleapi.Error type. For example:

	var e *googleapi.Error
	if ok := errors.As(err, &e); ok {
		  if e.Code == 409 { ... }
	}

See https://pkg.go.dev/google.golang.org/api/googleapi#Error for more information.

Retrying failed requests

Methods of this package may use exponential backoff to retry calls
that fail with transient errors. Retrying continues indefinitely unless the
controlling context is canceled, the client is closed, or a non-transient error
is received. See context.WithTimeout and context.WithCancel.

Retry strategy in this library follows best practices for Cloud Storage. By
default, only idempotent operations are retried, exponential backoff with jitter
is employed, and only transient network errors and response codes defined as
transient by the service and will be retried. See
https://cloud.google.com/storage/docs/retry-strategy for more information.

Users can configure non-default retry behavior for a particular operation (using
BucketHandle.Retryer and ObjectHandle.Retryer) or for all calls made by a
client (using Client.SetRetry). For example:

	o := client.Bucket(bucket).Object(object).Retryer(
		// Use WithBackoff to change the timing of the exponential backoff.
		storage.WithBackoff(gax.Backoff{
			Initial:    2 * time.Second,
		}),
		// Use WithPolicy to configure the idempotency policy. RetryAlways will
		// retry the operation even if it is non-idempotent.
		storage.WithPolicy(storage.RetryAlways),
	)

	// Use context timeouts to set an overall deadline on the call, including all
	// potential retries.
	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
	defer cancel()

	// Delete an object using the specified strategy and timeout.
	if err := o.Delete(ctx); err != nil {
		// Handle err.
	}
*/
package storage // import "cloud.google.com/go/storage"
<|endoftext|>