<commit_before>package ping

import (
	"errors"
	"fmt"
	"math"
	"net"
	"os/exec"
	"runtime"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/go-ping/ping"
	"github.com/influxdata/telegraf"
	"github.com/influxdata/telegraf/internal"
	"github.com/influxdata/telegraf/plugins/inputs"
)

// HostPinger is a function that runs the "ping" function using a list of
// passed arguments. This can be easily switched with a mocked ping function
// for unit test purposes (see ping_test.go)
type HostPinger func(binary string, timeout float64, args ...string) (string, error)

type Ping struct {
	// wg is used to wait for ping with multiple URLs
	wg sync.WaitGroup

	// Pre-calculated interval and timeout
	calcInterval time.Duration
	calcTimeout  time.Duration

	sourceAddress string

	Log telegraf.Logger `toml:"-"`

	// Interval at which to ping (ping -i <INTERVAL>)
	PingInterval float64 `toml:"ping_interval"`

	// Number of pings to send (ping -c <COUNT>)
	Count int

	// Per-ping timeout, in seconds. 0 means no timeout (ping -W <TIMEOUT>)
	Timeout float64

	// Ping deadline, in seconds. 0 means no deadline. (ping -w <DEADLINE>)
	Deadline int

	// Interface or source address to send ping from (ping -I/-S <INTERFACE/SRC_ADDR>)
	Interface string

	// URLs to ping
	Urls []string

	// Method defines how to ping (native or exec)
	Method string

	// Ping executable binary
	Binary string

	// Arguments for ping command. When arguments is not empty, system binary will be used and
	// other options (ping_interval, timeout, etc) will be ignored
	Arguments []string

	// Whether to resolve addresses using ipv6 or not.
	IPv6 bool

	// host ping function
	pingHost HostPinger

	nativePingFunc NativePingFunc

	// Calculate the given percentiles when using native method
	Percentiles []int
}

func (*Ping) Description() string {
	return "Ping given url(s) and return statistics"
}

const sampleConfig = `
  ## Hosts to send ping packets to.
  urls = ["example.org"]

  ## Method used for sending pings, can be either "exec" or "native".  When set
  ## to "exec" the systems ping command will be executed.  When set to "native"
  ## the plugin will send pings directly.
  ##
  ## While the default is "exec" for backwards compatibility, new deployments
  ## are encouraged to use the "native" method for improved compatibility and
  ## performance.
  # method = "exec"

  ## Number of ping packets to send per interval.  Corresponds to the "-c"
  ## option of the ping command.
  # count = 1

  ## Time to wait between sending ping packets in seconds.  Operates like the
  ## "-i" option of the ping command.
  # ping_interval = 1.0

  ## If set, the time to wait for a ping response in seconds.  Operates like
  ## the "-W" option of the ping command.
  # timeout = 1.0

  ## If set, the total ping deadline, in seconds.  Operates like the -w option
  ## of the ping command.
  # deadline = 10

  ## Interface or source address to send ping from.  Operates like the -I or -S
  ## option of the ping command.
  # interface = ""

  ## Percentiles to calculate. This only works with the native method.
  # percentiles = [50, 95, 99]

  ## Specify the ping executable binary.
  # binary = "ping"

  ## Arguments for ping command. When arguments is not empty, the command from
  ## the binary option will be used and other options (ping_interval, timeout,
  ## etc) will be ignored.
  # arguments = ["-c", "3"]

  ## Use only IPv6 addresses when resolving a hostname.
  # ipv6 = false
`

func (*Ping) SampleConfig() string {
	return sampleConfig
}

func (p *Ping) Gather(acc telegraf.Accumulator) error {
	for _, host := range p.Urls {
		p.wg.Add(1)
		go func(host string) {
			defer p.wg.Done()

			switch p.Method {
			case "native":
				p.pingToURLNative(host, acc)
			default:
				p.pingToURL(host, acc)
			}
		}(host)
	}

	p.wg.Wait()

	return nil
}

type pingStats struct {
	ping.Statistics
	ttl int
}

type NativePingFunc func(destination string) (*pingStats, error)

func (p *Ping) nativePing(destination string) (*pingStats, error) {
	ps := &pingStats{}

	pinger, err := ping.NewPinger(destination)
	if err != nil {
		return nil, fmt.Errorf("failed to create new pinger: %w", err)
	}

	// Required for windows. Despite the method name, this should work without the need to elevate privileges and has been tested on Windows 10
	if runtime.GOOS == "windows" {
		pinger.SetPrivileged(true)
	}

	if p.IPv6 {
		pinger.SetNetwork("ip6")
	}

	pinger.Source = p.sourceAddress
	pinger.Interval = p.calcInterval

	if p.Deadline > 0 {
		pinger.Timeout = time.Duration(p.Deadline) * time.Second
	}

	// Get Time to live (TTL) of first response, matching original implementation
	once := &sync.Once{}
	pinger.OnRecv = func(pkt *ping.Packet) {
		once.Do(func() {
			ps.ttl = pkt.Ttl
		})
	}

	pinger.Count = p.Count
	err = pinger.Run()
	if err != nil {
		return nil, fmt.Errorf("failed to run pinger: %w", err)
	}

	ps.Statistics = *pinger.Statistics()

	return ps, nil
}

func (p *Ping) pingToURLNative(destination string, acc telegraf.Accumulator) {

	tags := map[string]string{"url": destination}
	fields := map[string]interface{}{}

	stats, err := p.nativePingFunc(destination)
	if err != nil {
		if strings.Contains(err.Error(), "unknown") {
			fields["result_code"] = 1
		} else {
			fields["result_code"] = 2
		}
		acc.AddFields("ping", fields, tags)
		return
	}

	fields = map[string]interface{}{
		"result_code":         0,
		"packets_transmitted": stats.PacketsSent,
		"packets_received":    stats.PacketsRecv,
	}

	if stats.PacketsSent == 0 {
		fields["result_code"] = 2
		acc.AddFields("ping", fields, tags)
		return
	}

	if stats.PacketsRecv == 0 {
		fields["result_code"] = 1
		fields["percent_packet_loss"] = float64(100)
		acc.AddFields("ping", fields, tags)
		return
	}

	sort.Sort(durationSlice(stats.Rtts))
	for _, perc := range p.Percentiles {
		var value = percentile(durationSlice(stats.Rtts), perc)
		var field = fmt.Sprintf("percentile%v_ms", perc)
		fields[field] = float64(value.Nanoseconds()) / float64(time.Millisecond)
	}

	// Set TTL only on supported platform. See golang.org/x/net/ipv4/payload_cmsg.go
	switch runtime.GOOS {
	case "aix", "darwin", "dragonfly", "freebsd", "linux", "netbsd", "openbsd", "solaris":
		fields["ttl"] = stats.ttl
	}

	fields["percent_packet_loss"] = float64(stats.PacketLoss)
	fields["minimum_response_ms"] = float64(stats.MinRtt) / float64(time.Millisecond)
	fields["average_response_ms"] = float64(stats.AvgRtt) / float64(time.Millisecond)
	fields["maximum_response_ms"] = float64(stats.MaxRtt) / float64(time.Millisecond)
	fields["standard_deviation_ms"] = float64(stats.StdDevRtt) / float64(time.Millisecond)

	acc.AddFields("ping", fields, tags)
}

type durationSlice []time.Duration

func (p durationSlice) Len() int           { return len(p) }
func (p durationSlice) Less(i, j int) bool { return p[i] < p[j] }
func (p durationSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// R7 from Hyndman and Fan (1996), which matches Excel
func percentile(values durationSlice, perc int) time.Duration {
	if len(values) == 0 {
		return 0
	}
	if perc < 0 {
		perc = 0
	}
	if perc > 100 {
		perc = 100
	}
	var percFloat = float64(perc) / 100.0

	var count = len(values)
	var rank = percFloat * float64(count-1)
	var rankInteger = int(rank)
	var rankFraction = rank - math.Floor(rank)

	if rankInteger >= count-1 {
		return values[count-1]
	}

	upper := values[rankInteger+1]
	lower := values[rankInteger]
	return lower + time.Duration(rankFraction*float64(upper-lower))
}

// Init ensures the plugin is configured correctly.
func (p *Ping) Init() error {
	if p.Count < 1 {
		return errors.New("bad number of packets to transmit")
	}

	// The interval cannot be below 0.2 seconds, matching ping implementation: https://linux.die.net/man/8/ping
	if p.PingInterval < 0.2 {
		p.calcInterval = time.Duration(.2 * float64(time.Second))
	} else {
		p.calcInterval = time.Duration(p.PingInterval * float64(time.Second))
	}

	// If no timeout is given default to 5 seconds, matching original implementation
	if p.Timeout == 0 {
		p.calcTimeout = time.Duration(5) * time.Second
	} else {
		p.calcTimeout = time.Duration(p.Timeout) * time.Second
	}

	// Support either an IP address or interface name
	if p.Interface != "" {
		if addr := net.ParseIP(p.Interface); addr != nil {
			p.sourceAddress = p.Interface
		} else {
			i, err := net.InterfaceByName(p.Interface)
			if err != nil {
				return fmt.Errorf("failed to get interface: %w", err)
			}
			addrs, err := i.Addrs()
			if err != nil {
				return fmt.Errorf("failed to get the address of interface: %w", err)
			}
			p.sourceAddress = addrs[0].(*net.IPNet).IP.String()
		}
	}

	return nil
}

func hostPinger(binary string, timeout float64, args ...string) (string, error) {
	bin, err := exec.LookPath(binary)
	if err != nil {
		return "", err
	}
	c := exec.Command(bin, args...)
	out, err := internal.CombinedOutputTimeout(c,
		time.Second*time.Duration(timeout+5))
	return string(out), err
}

func init() {
	inputs.Add("ping", func() telegraf.Input {
		p := &Ping{
			pingHost:     hostPinger,
			PingInterval: 1.0,
			Count:        1,
			Timeout:      1.0,
			Deadline:     10,
			Method:       "exec",
			Binary:       "ping",
			Arguments:    []string{},
			Percentiles:  []int{},
		}
		p.nativePingFunc = p.nativePing
		return p
	})
}
<commit_msg>add more logging to ping plugin<commit_after>package ping

import (
	"errors"
	"fmt"
	"math"
	"net"
	"os/exec"
	"runtime"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/go-ping/ping"
	"github.com/influxdata/telegraf"
	"github.com/influxdata/telegraf/internal"
	"github.com/influxdata/telegraf/plugins/inputs"
)

// HostPinger is a function that runs the "ping" function using a list of
// passed arguments. This can be easily switched with a mocked ping function
// for unit test purposes (see ping_test.go)
type HostPinger func(binary string, timeout float64, args ...string) (string, error)

type Ping struct {
	// wg is used to wait for ping with multiple URLs
	wg sync.WaitGroup

	// Pre-calculated interval and timeout
	calcInterval time.Duration
	calcTimeout  time.Duration

	sourceAddress string

	Log telegraf.Logger `toml:"-"`

	// Interval at which to ping (ping -i <INTERVAL>)
	PingInterval float64 `toml:"ping_interval"`

	// Number of pings to send (ping -c <COUNT>)
	Count int

	// Per-ping timeout, in seconds. 0 means no timeout (ping -W <TIMEOUT>)
	Timeout float64

	// Ping deadline, in seconds. 0 means no deadline. (ping -w <DEADLINE>)
	Deadline int

	// Interface or source address to send ping from (ping -I/-S <INTERFACE/SRC_ADDR>)
	Interface string

	// URLs to ping
	Urls []string

	// Method defines how to ping (native or exec)
	Method string

	// Ping executable binary
	Binary string

	// Arguments for ping command. When arguments is not empty, system binary will be used and
	// other options (ping_interval, timeout, etc) will be ignored
	Arguments []string

	// Whether to resolve addresses using ipv6 or not.
	IPv6 bool

	// host ping function
	pingHost HostPinger

	nativePingFunc NativePingFunc

	// Calculate the given percentiles when using native method
	Percentiles []int
}

func (*Ping) Description() string {
	return "Ping given url(s) and return statistics"
}

const sampleConfig = `
  ## Hosts to send ping packets to.
  urls = ["example.org"]

  ## Method used for sending pings, can be either "exec" or "native".  When set
  ## to "exec" the systems ping command will be executed.  When set to "native"
  ## the plugin will send pings directly.
  ##
  ## While the default is "exec" for backwards compatibility, new deployments
  ## are encouraged to use the "native" method for improved compatibility and
  ## performance.
  # method = "exec"

  ## Number of ping packets to send per interval.  Corresponds to the "-c"
  ## option of the ping command.
  # count = 1

  ## Time to wait between sending ping packets in seconds.  Operates like the
  ## "-i" option of the ping command.
  # ping_interval = 1.0

  ## If set, the time to wait for a ping response in seconds.  Operates like
  ## the "-W" option of the ping command.
  # timeout = 1.0

  ## If set, the total ping deadline, in seconds.  Operates like the -w option
  ## of the ping command.
  # deadline = 10

  ## Interface or source address to send ping from.  Operates like the -I or -S
  ## option of the ping command.
  # interface = ""

  ## Percentiles to calculate. This only works with the native method.
  # percentiles = [50, 95, 99]

  ## Specify the ping executable binary.
  # binary = "ping"

  ## Arguments for ping command. When arguments is not empty, the command from
  ## the binary option will be used and other options (ping_interval, timeout,
  ## etc) will be ignored.
  # arguments = ["-c", "3"]

  ## Use only IPv6 addresses when resolving a hostname.
  # ipv6 = false
`

func (*Ping) SampleConfig() string {
	return sampleConfig
}

func (p *Ping) Gather(acc telegraf.Accumulator) error {
	for _, host := range p.Urls {
		p.wg.Add(1)
		go func(host string) {
			defer p.wg.Done()

			switch p.Method {
			case "native":
				p.pingToURLNative(host, acc)
			default:
				p.pingToURL(host, acc)
			}
		}(host)
	}

	p.wg.Wait()

	return nil
}

type pingStats struct {
	ping.Statistics
	ttl int
}

type NativePingFunc func(destination string) (*pingStats, error)

func (p *Ping) nativePing(destination string) (*pingStats, error) {
	ps := &pingStats{}

	pinger, err := ping.NewPinger(destination)
	if err != nil {
		return nil, fmt.Errorf("failed to create new pinger: %w", err)
	}

	// Required for windows. Despite the method name, this should work without the need to elevate privileges and has been tested on Windows 10
	if runtime.GOOS == "windows" {
		pinger.SetPrivileged(true)
	}

	if p.IPv6 {
		pinger.SetNetwork("ip6")
	}

	pinger.Source = p.sourceAddress
	pinger.Interval = p.calcInterval

	if p.Deadline > 0 {
		pinger.Timeout = time.Duration(p.Deadline) * time.Second
	}

	// Get Time to live (TTL) of first response, matching original implementation
	once := &sync.Once{}
	pinger.OnRecv = func(pkt *ping.Packet) {
		once.Do(func() {
			ps.ttl = pkt.Ttl
		})
	}

	pinger.Count = p.Count
	err = pinger.Run()
	if err != nil {
		return nil, fmt.Errorf("failed to run pinger: %w", err)
	}

	ps.Statistics = *pinger.Statistics()

	return ps, nil
}

func (p *Ping) pingToURLNative(destination string, acc telegraf.Accumulator) {

	tags := map[string]string{"url": destination}
	fields := map[string]interface{}{}

	stats, err := p.nativePingFunc(destination)
	if err != nil {
		p.Log.Errorf("ping failed: %s", err.Error())
		if strings.Contains(err.Error(), "unknown") {
			fields["result_code"] = 1
		} else {
			fields["result_code"] = 2
		}
		acc.AddFields("ping", fields, tags)
		return
	}

	fields = map[string]interface{}{
		"result_code":         0,
		"packets_transmitted": stats.PacketsSent,
		"packets_received":    stats.PacketsRecv,
	}

	if stats.PacketsSent == 0 {
		p.Log.Debug("no packets sent")
		fields["result_code"] = 2
		acc.AddFields("ping", fields, tags)
		return
	}

	if stats.PacketsRecv == 0 {
		p.Log.Debug("no packets received")
		fields["result_code"] = 1
		fields["percent_packet_loss"] = float64(100)
		acc.AddFields("ping", fields, tags)
		return
	}

	sort.Sort(durationSlice(stats.Rtts))
	for _, perc := range p.Percentiles {
		var value = percentile(durationSlice(stats.Rtts), perc)
		var field = fmt.Sprintf("percentile%v_ms", perc)
		fields[field] = float64(value.Nanoseconds()) / float64(time.Millisecond)
	}

	// Set TTL only on supported platform. See golang.org/x/net/ipv4/payload_cmsg.go
	switch runtime.GOOS {
	case "aix", "darwin", "dragonfly", "freebsd", "linux", "netbsd", "openbsd", "solaris":
		fields["ttl"] = stats.ttl
	}

	fields["percent_packet_loss"] = float64(stats.PacketLoss)
	fields["minimum_response_ms"] = float64(stats.MinRtt) / float64(time.Millisecond)
	fields["average_response_ms"] = float64(stats.AvgRtt) / float64(time.Millisecond)
	fields["maximum_response_ms"] = float64(stats.MaxRtt) / float64(time.Millisecond)
	fields["standard_deviation_ms"] = float64(stats.StdDevRtt) / float64(time.Millisecond)

	acc.AddFields("ping", fields, tags)
}

type durationSlice []time.Duration

func (p durationSlice) Len() int           { return len(p) }
func (p durationSlice) Less(i, j int) bool { return p[i] < p[j] }
func (p durationSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// R7 from Hyndman and Fan (1996), which matches Excel
func percentile(values durationSlice, perc int) time.Duration {
	if len(values) == 0 {
		return 0
	}
	if perc < 0 {
		perc = 0
	}
	if perc > 100 {
		perc = 100
	}
	var percFloat = float64(perc) / 100.0

	var count = len(values)
	var rank = percFloat * float64(count-1)
	var rankInteger = int(rank)
	var rankFraction = rank - math.Floor(rank)

	if rankInteger >= count-1 {
		return values[count-1]
	}

	upper := values[rankInteger+1]
	lower := values[rankInteger]
	return lower + time.Duration(rankFraction*float64(upper-lower))
}

// Init ensures the plugin is configured correctly.
func (p *Ping) Init() error {
	if p.Count < 1 {
		return errors.New("bad number of packets to transmit")
	}

	// The interval cannot be below 0.2 seconds, matching ping implementation: https://linux.die.net/man/8/ping
	if p.PingInterval < 0.2 {
		p.calcInterval = time.Duration(.2 * float64(time.Second))
	} else {
		p.calcInterval = time.Duration(p.PingInterval * float64(time.Second))
	}

	// If no timeout is given default to 5 seconds, matching original implementation
	if p.Timeout == 0 {
		p.calcTimeout = time.Duration(5) * time.Second
	} else {
		p.calcTimeout = time.Duration(p.Timeout) * time.Second
	}

	// Support either an IP address or interface name
	if p.Interface != "" {
		if addr := net.ParseIP(p.Interface); addr != nil {
			p.sourceAddress = p.Interface
		} else {
			i, err := net.InterfaceByName(p.Interface)
			if err != nil {
				return fmt.Errorf("failed to get interface: %w", err)
			}
			addrs, err := i.Addrs()
			if err != nil {
				return fmt.Errorf("failed to get the address of interface: %w", err)
			}
			p.sourceAddress = addrs[0].(*net.IPNet).IP.String()
		}
	}

	return nil
}

func hostPinger(binary string, timeout float64, args ...string) (string, error) {
	bin, err := exec.LookPath(binary)
	if err != nil {
		return "", err
	}
	c := exec.Command(bin, args...)
	out, err := internal.CombinedOutputTimeout(c,
		time.Second*time.Duration(timeout+5))
	return string(out), err
}

func init() {
	inputs.Add("ping", func() telegraf.Input {
		p := &Ping{
			pingHost:     hostPinger,
			PingInterval: 1.0,
			Count:        1,
			Timeout:      1.0,
			Deadline:     10,
			Method:       "exec",
			Binary:       "ping",
			Arguments:    []string{},
			Percentiles:  []int{},
		}
		p.nativePingFunc = p.nativePing
		return p
	})
}
<|endoftext|>


<commit_before>package tablestorageproxy

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha256"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httputil"
	"strings"
	"time"
)

type GoHaveStorage interface {
	GetKey() []byte
	GetAccount() string
}

type TableStorageProxy struct {
	goHaveStorage GoHaveStorage
	baseUrl string
}

func New(goHaveStorage GoHaveStorage) *TableStorageProxy {
	var tableStorageProxy TableStorageProxy

	tableStorageProxy.goHaveStorage = goHaveStorage
	tableStorageProxy.baseUrl = "https://"+goHaveStorage.GetAccount()+".table.core.windows.net/"

	return &tableStorageProxy
}

func (tableStorageProxy *TableStorageProxy) QueryTables() {
	tableStorageProxy.executeCommonRequest("GET", "Tables", "", nil, false, true, false)
}

func (tableStorageProxy *TableStorageProxy) QueryEntity(tableName string, partitionKey string, rowKey string, selects string) {
	tableStorageProxy.executeCommonRequest("GET", tableName + "%28PartitionKey=%27" + partitionKey + "%27,RowKey=%27" + rowKey + "%27%29", "?$select="+selects, nil, false, true, false)
}

func (tableStorageProxy *TableStorageProxy) QueryEntities(tableName string, selects string, filter string, top string) {
	tableStorageProxy.executeCommonRequest("GET", tableName, "?$filter="+filter + "&$select=" + selects+"&$top="+top, nil, false, true, false)
}

func (tableStorageProxy *TableStorageProxy) DeleteEntity(tableName string, partitionKey string, rowKey string) {
	tableStorageProxy.executeEntityRequest("DELETE",tableName, partitionKey, rowKey, nil, true)
}

func (tableStorageProxy *TableStorageProxy) UpdateEntity(tableName string, partitionKey string, rowKey string, json []byte) {
	tableStorageProxy.executeEntityRequest("PUT",tableName, partitionKey, rowKey, json, true)
}

func (tableStorageProxy *TableStorageProxy) MergeEntity(tableName string, partitionKey string, rowKey string, json []byte) {
	tableStorageProxy.executeEntityRequest("MERGE",tableName, partitionKey, rowKey, json, true)
}

func (tableStorageProxy *TableStorageProxy) InsertOrMergeEntity(tableName string, partitionKey string, rowKey string, json []byte) {
	tableStorageProxy.executeEntityRequest("MERGE",tableName, partitionKey, rowKey, json, false)
}

func (tableStorageProxy *TableStorageProxy) InsertOrReplaceEntity(tableName string, partitionKey string, rowKey string, json []byte) {
	tableStorageProxy.executeEntityRequest("PUT",tableName, partitionKey, rowKey, json, false)
}

func (tableStorageProxy *TableStorageProxy) DeleteTable(tableName string) {
	tableStorageProxy.executeCommonRequest("DELETE", tableName, "", nil, false, false, true)
}

type CreateTableArgs struct {
	TableName string
}

func (tableStorageProxy *TableStorageProxy) CreateTable(tableName string) {
	json, _ := json.Marshal(CreateTableArgs{TableName: tableName})
	tableStorageProxy.executeCommonRequest("POST", "Tables", "", json, false, false, false)
}

func (tableStorageProxy *TableStorageProxy) InsertEntity(tableName string, json []byte) {
	tableStorageProxy.executeCommonRequest("POST", tableName, "", json, false, false, false)
}

func (tableStorageProxy *TableStorageProxy) executeRequest(request *http.Request, client *http.Client, target string) {
	xmsdate, Authentication := tableStorageProxy.calculateDateAndAuthentication(target)

	request.Header.Set("x-ms-date", xmsdate)
	request.Header.Set("x-ms-version", "2013-08-15")
	request.Header.Set("Authorization", Authentication)

	requestDump, _ := httputil.DumpRequest(request, true)

	fmt.Printf("Request: %s\n", requestDump)

	response, _ := client.Do(request)

	responseDump, _ := httputil.DumpResponse(response, true)
	fmt.Printf("Response: %s\n", responseDump)
}

func (tableStorageProxy *TableStorageProxy)  executeEntityRequest(httpVerb string, tableName string, partitionKey string, rowKey string, json []byte, useIfMatch bool) {
	tableStorageProxy.executeCommonRequest(httpVerb, tableName + "%28PartitionKey=%27" + partitionKey + "%27,RowKey=%27" + rowKey + "%27%29", "", json, useIfMatch, false, false)
}

func (tableStorageProxy *TableStorageProxy)  executeCommonRequest(httpVerb string, target string, query string, json []byte, useIfMatch bool, useAccept bool, useContentTypeXML bool) {
	client := &http.Client{}
	request, _ := http.NewRequest(httpVerb, tableStorageProxy.baseUrl+target+query, bytes.NewBuffer(json))

	if json != nil {
		request.Header.Set("Content-Type", "application/json")
		request.Header.Set("Content-Length", string(len(json)))
	}

	if useContentTypeXML {
		request.Header.Set("Content-Type", "application/atom+xml")
	}

	if useIfMatch {
		request.Header.Set("If-Match", "*")
	}

	if useAccept {
		request.Header.Set("Accept", "application/json;odata=nometadata")
	}

	tableStorageProxy.executeRequest(request, client, target)
}

func (tableStorageProxy *TableStorageProxy) calculateDateAndAuthentication(target string) (string, string) {
	xmsdate := strings.Replace(time.Now().UTC().Add(-time.Minute).Format(time.RFC1123), "UTC", "GMT", -1)
	SignatureString := xmsdate + "\n/" + tableStorageProxy.goHaveStorage.GetAccount() + "/" + target
	Authentication := "SharedKeyLite " + tableStorageProxy.goHaveStorage.GetAccount() + ":" + computeHmac256(SignatureString, tableStorageProxy.goHaveStorage.GetKey())
	return xmsdate, Authentication
}

func computeHmac256(message string, key []byte) string {
	h := hmac.New(sha256.New, key)
	h.Write([]byte(message))
	return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
<commit_msg>Now we only have 1 requester method<commit_after>package tablestorageproxy

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha256"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httputil"
	"strings"
	"time"
)

type GoHaveStorage interface {
	GetKey() []byte
	GetAccount() string
}

type TableStorageProxy struct {
	goHaveStorage GoHaveStorage
	baseUrl string
}

func New(goHaveStorage GoHaveStorage) *TableStorageProxy {
	var tableStorageProxy TableStorageProxy

	tableStorageProxy.goHaveStorage = goHaveStorage
	tableStorageProxy.baseUrl = "https://"+goHaveStorage.GetAccount()+".table.core.windows.net/"

	return &tableStorageProxy
}

func (tableStorageProxy *TableStorageProxy) QueryTables() {
	tableStorageProxy.executeCommonRequest("GET", "Tables", "", nil, false, true, false)
}

func (tableStorageProxy *TableStorageProxy) QueryEntity(tableName string, partitionKey string, rowKey string, selects string) {
	tableStorageProxy.executeCommonRequest("GET", tableName + "%28PartitionKey=%27" + partitionKey + "%27,RowKey=%27" + rowKey + "%27%29", "?$select="+selects, nil, false, true, false)
}

func (tableStorageProxy *TableStorageProxy) QueryEntities(tableName string, selects string, filter string, top string) {
	tableStorageProxy.executeCommonRequest("GET", tableName, "?$filter="+filter + "&$select=" + selects+"&$top="+top, nil, false, true, false)
}

func (tableStorageProxy *TableStorageProxy) DeleteEntity(tableName string, partitionKey string, rowKey string) {
	tableStorageProxy.executeEntityRequest("DELETE",tableName, partitionKey, rowKey, nil, true)
}

func (tableStorageProxy *TableStorageProxy) UpdateEntity(tableName string, partitionKey string, rowKey string, json []byte) {
	tableStorageProxy.executeEntityRequest("PUT",tableName, partitionKey, rowKey, json, true)
}

func (tableStorageProxy *TableStorageProxy) MergeEntity(tableName string, partitionKey string, rowKey string, json []byte) {
	tableStorageProxy.executeEntityRequest("MERGE",tableName, partitionKey, rowKey, json, true)
}

func (tableStorageProxy *TableStorageProxy) InsertOrMergeEntity(tableName string, partitionKey string, rowKey string, json []byte) {
	tableStorageProxy.executeEntityRequest("MERGE",tableName, partitionKey, rowKey, json, false)
}

func (tableStorageProxy *TableStorageProxy) InsertOrReplaceEntity(tableName string, partitionKey string, rowKey string, json []byte) {
	tableStorageProxy.executeEntityRequest("PUT",tableName, partitionKey, rowKey, json, false)
}

func (tableStorageProxy *TableStorageProxy) DeleteTable(tableName string) {
	tableStorageProxy.executeCommonRequest("DELETE", tableName, "", nil, false, false, true)
}

type CreateTableArgs struct {
	TableName string
}

func (tableStorageProxy *TableStorageProxy) CreateTable(tableName string) {
	json, _ := json.Marshal(CreateTableArgs{TableName: tableName})
	tableStorageProxy.executeCommonRequest("POST", "Tables", "", json, false, false, false)
}

func (tableStorageProxy *TableStorageProxy) InsertEntity(tableName string, json []byte) {
	tableStorageProxy.executeCommonRequest("POST", tableName, "", json, false, false, false)
}

func (tableStorageProxy *TableStorageProxy)  executeEntityRequest(httpVerb string, tableName string, partitionKey string, rowKey string, json []byte, useIfMatch bool) {
	tableStorageProxy.executeCommonRequest(httpVerb, tableName + "%28PartitionKey=%27" + partitionKey + "%27,RowKey=%27" + rowKey + "%27%29", "", json, useIfMatch, false, false)
}

func (tableStorageProxy *TableStorageProxy)  executeCommonRequest(httpVerb string, target string, query string, json []byte, useIfMatch bool, useAccept bool, useContentTypeXML bool) {
	xmsdate, Authentication := tableStorageProxy.calculateDateAndAuthentication(target)

	client := &http.Client{}
	request, _ := http.NewRequest(httpVerb, tableStorageProxy.baseUrl+target+query, bytes.NewBuffer(json))

	if json != nil {
		request.Header.Set("Content-Type", "application/json")
		request.Header.Set("Content-Length", string(len(json)))
	}

	if useContentTypeXML {
		request.Header.Set("Content-Type", "application/atom+xml")
	}

	if useIfMatch {
		request.Header.Set("If-Match", "*")
	}

	if useAccept {
		request.Header.Set("Accept", "application/json;odata=nometadata")
	}

	request.Header.Set("x-ms-date", xmsdate)
	request.Header.Set("x-ms-version", "2013-08-15")
	request.Header.Set("Authorization", Authentication)

	requestDump, _ := httputil.DumpRequest(request, true)

	fmt.Printf("Request: %s\n", requestDump)

	response, _ := client.Do(request)

	responseDump, _ := httputil.DumpResponse(response, true)
	fmt.Printf("Response: %s\n", responseDump)
}

func (tableStorageProxy *TableStorageProxy) calculateDateAndAuthentication(target string) (string, string) {
	xmsdate := strings.Replace(time.Now().UTC().Add(-time.Minute).Format(time.RFC1123), "UTC", "GMT", -1)
	SignatureString := xmsdate + "\n/" + tableStorageProxy.goHaveStorage.GetAccount() + "/" + target
	Authentication := "SharedKeyLite " + tableStorageProxy.goHaveStorage.GetAccount() + ":" + computeHmac256(SignatureString, tableStorageProxy.goHaveStorage.GetKey())
	return xmsdate, Authentication
}

func computeHmac256(message string, key []byte) string {
	h := hmac.New(sha256.New, key)
	h.Write([]byte(message))
	return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package cloudsigma

import (
	"github.com/juju/errors"
	"github.com/juju/loggo"

	"github.com/juju/juju/cloudconfig/instancecfg"
	"github.com/juju/juju/cloudconfig/providerinit"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/environs/imagemetadata"
	"github.com/juju/juju/environs/simplestreams"
	"github.com/juju/juju/instance"
	"github.com/juju/juju/network"
	"github.com/juju/juju/tools"
)

//
// Imlementation of InstanceBroker: methods for starting and stopping instances.
//

var findInstanceImage = func(env *environ, ic *imagemetadata.ImageConstraint) (*imagemetadata.ImageMetadata, error) {

	sources, err := environs.ImageMetadataSources(env)
	if err != nil {
		return nil, err
	}

	matchingImages, _, err := imagemetadata.Fetch(sources, ic, false)
	if err != nil {
		return nil, err
	}
	if len(matchingImages) == 0 {
		return nil, errors.New("no matching image meta data")
	}

	return matchingImages[0], nil
}

// MaintainInstance is specified in the InstanceBroker interface.
func (*environ) MaintainInstance(args environs.StartInstanceParams) error {
	return nil
}

// StartInstance asks for a new instance to be created, associated with
// the provided config in machineConfig. The given config describes the juju
// state for the new instance to connect to. The config MachineNonce, which must be
// unique within an environment, is used by juju to protect against the
// consequences of multiple instances being started with the same machine id.
func (env *environ) StartInstance(args environs.StartInstanceParams) (*environs.StartInstanceResult, error) {
	logger.Infof("sigmaEnviron.StartInstance...")

	if args.InstanceConfig == nil {
		return nil, errors.New("instance configuration is nil")
	}

	if args.InstanceConfig.HasNetworks() {
		return nil, errors.New("starting instances with networks is not supported yet")
	}

	if len(args.Tools) == 0 {
		return nil, errors.New("tools not found")
	}

	region, _ := env.Region()
	img, err := findInstanceImage(env, imagemetadata.NewImageConstraint(simplestreams.LookupParams{
		CloudSpec: region,
		Series:    args.Tools.AllSeries(),
		Arches:    args.Tools.Arches(),
		Stream:    env.Config().ImageStream(),
	}))
	if err != nil {
		return nil, err
	}

	tools, err := args.Tools.Match(tools.Filter{Arch: img.Arch})
	if err != nil {
		return nil, errors.Errorf("chosen architecture %v not present in %v", img.Arch, args.Tools.Arches())
	}

	args.InstanceConfig.Tools = tools[0]
	if err := instancecfg.FinishInstanceConfig(args.InstanceConfig, env.Config()); err != nil {
		return nil, err
	}
	userData, err := providerinit.ComposeUserData(args.InstanceConfig, nil)
	if err != nil {
		return nil, errors.Annotate(err, "cannot make user data")
	}

	logger.Debugf("cloudsigma user data; %d bytes", len(userData))

	client := env.client
	server, rootdrive, arch, err := client.newInstance(args, img, userData)
	if err != nil {
		return nil, errors.Errorf("failed start instance: %v", err)
	}

	inst := &sigmaInstance{server: server}

	// prepare hardware characteristics
	hwch, err := inst.hardware(arch, rootdrive.Size())
	if err != nil {
		return nil, err
	}

	logger.Debugf("hardware: %v", hwch)
	return &environs.StartInstanceResult{
		Instance: inst,
		Hardware: hwch,
	}, nil
}

// AllInstances returns all instances currently known to the broker.
func (env *environ) AllInstances() ([]instance.Instance, error) {
	// Please note that this must *not* return instances that have not been
	// allocated as part of this environment -- if it does, juju will see they
	// are not tracked in state, assume they're stale/rogue, and shut them down.

	logger.Tracef("environ.AllInstances...")

	servers, err := env.client.instances()
	if err != nil {
		logger.Tracef("environ.AllInstances failed: %v", err)
		return nil, err
	}

	instances := make([]instance.Instance, 0, len(servers))
	for _, server := range servers {
		instance := sigmaInstance{server: server}
		instances = append(instances, instance)
	}

	if logger.LogLevel() <= loggo.TRACE {
		logger.Tracef("All instances, len = %d:", len(instances))
		for _, instance := range instances {
			logger.Tracef("... id: %q, status: %q", instance.Id(), instance.Status())
		}
	}

	return instances, nil
}

// Instances returns a slice of instances corresponding to the
// given instance ids.  If no instances were found, but there
// was no other error, it will return ErrNoInstances.  If
// some but not all the instances were found, the returned slice
// will have some nil slots, and an ErrPartialInstances error
// will be returned.
func (env *environ) Instances(ids []instance.Id) ([]instance.Instance, error) {
	logger.Tracef("environ.Instances %#v", ids)
	// Please note that this must *not* return instances that have not been
	// allocated as part of this environment -- if it does, juju will see they
	// are not tracked in state, assume they're stale/rogue, and shut them down.
	// This advice applies even if an instance id passed in corresponds to a
	// real instance that's not part of the environment -- the Environ should
	// treat that no differently to a request for one that does not exist.

	m, err := env.client.instanceMap()
	if err != nil {
		logger.Warningf("environ.Instances failed: %v", err)
		return nil, err
	}

	var found int
	r := make([]instance.Instance, len(ids))
	for i, id := range ids {
		if s, ok := m[string(id)]; ok {
			r[i] = sigmaInstance{server: s}
			found++
		}
	}

	if found == 0 {
		err = environs.ErrNoInstances
	} else if found != len(ids) {
		err = environs.ErrPartialInstances
	}

	return r, err
}

// StopInstances shuts down the given instances.
func (env *environ) StopInstances(instances ...instance.Id) error {
	logger.Debugf("stop instances %+v", instances)

	var err error

	for _, instance := range instances {
		if e := env.client.stopInstance(instance); e != nil {
			err = e
		}
	}

	return err
}

// AllocateAddress requests a new address to be allocated for the
// given instance on the given network.
func (env *environ) AllocateAddress(instID instance.Id, netID network.Id, addr network.Address) error {
	return errors.NotSupportedf("AllocateAddress")
}
func (env *environ) ReleaseAddress(instId instance.Id, netId network.Id, addr network.Address) error {
	return errors.NotSupportedf("ReleaseAddress")
}
func (env *environ) Subnets(inst instance.Id) ([]network.SubnetInfo, error) {
	return nil, errors.NotSupportedf("Subnets")
}

// ListNetworks returns basic information about all networks known
// by the provider for the environment. They may be unknown to juju
// yet (i.e. when called initially or when a new network was created).
func (env *environ) ListNetworks() ([]network.SubnetInfo, error) {
	return nil, errors.NotImplementedf("ListNetworks")
}
<commit_msg>Update cloudsigma provider AllocateAddress<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package cloudsigma

import (
	"github.com/juju/errors"
	"github.com/juju/loggo"

	"github.com/juju/juju/cloudconfig/instancecfg"
	"github.com/juju/juju/cloudconfig/providerinit"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/environs/imagemetadata"
	"github.com/juju/juju/environs/simplestreams"
	"github.com/juju/juju/instance"
	"github.com/juju/juju/network"
	"github.com/juju/juju/tools"
)

//
// Imlementation of InstanceBroker: methods for starting and stopping instances.
//

var findInstanceImage = func(env *environ, ic *imagemetadata.ImageConstraint) (*imagemetadata.ImageMetadata, error) {

	sources, err := environs.ImageMetadataSources(env)
	if err != nil {
		return nil, err
	}

	matchingImages, _, err := imagemetadata.Fetch(sources, ic, false)
	if err != nil {
		return nil, err
	}
	if len(matchingImages) == 0 {
		return nil, errors.New("no matching image meta data")
	}

	return matchingImages[0], nil
}

// MaintainInstance is specified in the InstanceBroker interface.
func (*environ) MaintainInstance(args environs.StartInstanceParams) error {
	return nil
}

// StartInstance asks for a new instance to be created, associated with
// the provided config in machineConfig. The given config describes the juju
// state for the new instance to connect to. The config MachineNonce, which must be
// unique within an environment, is used by juju to protect against the
// consequences of multiple instances being started with the same machine id.
func (env *environ) StartInstance(args environs.StartInstanceParams) (*environs.StartInstanceResult, error) {
	logger.Infof("sigmaEnviron.StartInstance...")

	if args.InstanceConfig == nil {
		return nil, errors.New("instance configuration is nil")
	}

	if args.InstanceConfig.HasNetworks() {
		return nil, errors.New("starting instances with networks is not supported yet")
	}

	if len(args.Tools) == 0 {
		return nil, errors.New("tools not found")
	}

	region, _ := env.Region()
	img, err := findInstanceImage(env, imagemetadata.NewImageConstraint(simplestreams.LookupParams{
		CloudSpec: region,
		Series:    args.Tools.AllSeries(),
		Arches:    args.Tools.Arches(),
		Stream:    env.Config().ImageStream(),
	}))
	if err != nil {
		return nil, err
	}

	tools, err := args.Tools.Match(tools.Filter{Arch: img.Arch})
	if err != nil {
		return nil, errors.Errorf("chosen architecture %v not present in %v", img.Arch, args.Tools.Arches())
	}

	args.InstanceConfig.Tools = tools[0]
	if err := instancecfg.FinishInstanceConfig(args.InstanceConfig, env.Config()); err != nil {
		return nil, err
	}
	userData, err := providerinit.ComposeUserData(args.InstanceConfig, nil)
	if err != nil {
		return nil, errors.Annotate(err, "cannot make user data")
	}

	logger.Debugf("cloudsigma user data; %d bytes", len(userData))

	client := env.client
	server, rootdrive, arch, err := client.newInstance(args, img, userData)
	if err != nil {
		return nil, errors.Errorf("failed start instance: %v", err)
	}

	inst := &sigmaInstance{server: server}

	// prepare hardware characteristics
	hwch, err := inst.hardware(arch, rootdrive.Size())
	if err != nil {
		return nil, err
	}

	logger.Debugf("hardware: %v", hwch)
	return &environs.StartInstanceResult{
		Instance: inst,
		Hardware: hwch,
	}, nil
}

// AllInstances returns all instances currently known to the broker.
func (env *environ) AllInstances() ([]instance.Instance, error) {
	// Please note that this must *not* return instances that have not been
	// allocated as part of this environment -- if it does, juju will see they
	// are not tracked in state, assume they're stale/rogue, and shut them down.

	logger.Tracef("environ.AllInstances...")

	servers, err := env.client.instances()
	if err != nil {
		logger.Tracef("environ.AllInstances failed: %v", err)
		return nil, err
	}

	instances := make([]instance.Instance, 0, len(servers))
	for _, server := range servers {
		instance := sigmaInstance{server: server}
		instances = append(instances, instance)
	}

	if logger.LogLevel() <= loggo.TRACE {
		logger.Tracef("All instances, len = %d:", len(instances))
		for _, instance := range instances {
			logger.Tracef("... id: %q, status: %q", instance.Id(), instance.Status())
		}
	}

	return instances, nil
}

// Instances returns a slice of instances corresponding to the
// given instance ids.  If no instances were found, but there
// was no other error, it will return ErrNoInstances.  If
// some but not all the instances were found, the returned slice
// will have some nil slots, and an ErrPartialInstances error
// will be returned.
func (env *environ) Instances(ids []instance.Id) ([]instance.Instance, error) {
	logger.Tracef("environ.Instances %#v", ids)
	// Please note that this must *not* return instances that have not been
	// allocated as part of this environment -- if it does, juju will see they
	// are not tracked in state, assume they're stale/rogue, and shut them down.
	// This advice applies even if an instance id passed in corresponds to a
	// real instance that's not part of the environment -- the Environ should
	// treat that no differently to a request for one that does not exist.

	m, err := env.client.instanceMap()
	if err != nil {
		logger.Warningf("environ.Instances failed: %v", err)
		return nil, err
	}

	var found int
	r := make([]instance.Instance, len(ids))
	for i, id := range ids {
		if s, ok := m[string(id)]; ok {
			r[i] = sigmaInstance{server: s}
			found++
		}
	}

	if found == 0 {
		err = environs.ErrNoInstances
	} else if found != len(ids) {
		err = environs.ErrPartialInstances
	}

	return r, err
}

// StopInstances shuts down the given instances.
func (env *environ) StopInstances(instances ...instance.Id) error {
	logger.Debugf("stop instances %+v", instances)

	var err error

	for _, instance := range instances {
		if e := env.client.stopInstance(instance); e != nil {
			err = e
		}
	}

	return err
}

// AllocateAddress requests a new address to be allocated for the
// given instance on the given network.
func (env *environ) AllocateAddress(instID instance.Id, netID network.Id, addr network.Address, macAddress string) error {
	return errors.NotSupportedf("AllocateAddress")
}
func (env *environ) ReleaseAddress(instId instance.Id, netId network.Id, addr network.Address) error {
	return errors.NotSupportedf("ReleaseAddress")
}
func (env *environ) Subnets(inst instance.Id) ([]network.SubnetInfo, error) {
	return nil, errors.NotSupportedf("Subnets")
}

// ListNetworks returns basic information about all networks known
// by the provider for the environment. They may be unknown to juju
// yet (i.e. when called initially or when a new network was created).
func (env *environ) ListNetworks() ([]network.SubnetInfo, error) {
	return nil, errors.NotImplementedf("ListNetworks")
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"html/template"
	"log"
	"os"
	"path"
	"sort"
	"time"

	"github.com/gorilla/feeds"
)

const (
	dateLayout        = `"2006-01-02"`
	displayDateLayout = "02 Jan 2006"
)

var tmpl = template.Must(template.ParseFiles("index.gohtml"))

type episode struct {
	Show  string `json:"show"`
	Title string `json:"title"`
	About string `json:"about"`
	URL   string `json:"url"`
	Date  date   `json:"date"`
}

func (e *episode) GetDisplayDate() string {
	return (time.Time)(e.Date).Format(displayDateLayout)
}

type date time.Time

func (d *date) UnmarshalJSON(data []byte) error {
	t, err := time.Parse(dateLayout, string(data))
	*d = date(t)
	return err
}

func (d date) Before(t date) bool {
	return (time.Time)(d).Before(time.Time(t))
}

type episodes []episode

func (e episodes) Len() int               { return len(e) }
func (e episodes) Less(i int, j int) bool { return e[i].Date.Before(e[j].Date) }
func (e episodes) Swap(i int, j int)      { e[i], e[j] = e[j], e[i] }

func main() {
	episodes := parseEpisodes()
	sort.Sort(sort.Reverse(episodes))
	createSite(episodes)
	createFeeds(episodes)
}

func parseEpisodes() episodes {
	jsonFile, err := os.Open("episodes.json")
	if err != nil {
		log.Fatal(err)
	}

	var episodes episodes
	if err := json.NewDecoder(jsonFile).Decode(&episodes); err != nil {
		log.Fatal(err)
	}

	return episodes
}

func createSite(episodes episodes) {
	indexFile := createFile("index.html")
	if err := tmpl.ExecuteTemplate(indexFile, "index", episodes); err != nil {
		log.Fatal(err)
	}
}

func createFile(fileName string) *os.File {
	file, err := os.Create(path.Join("static", fileName))
	if err != nil {
		log.Fatal(err)
	}
	return file
}

func createFeeds(episodes episodes) {
	rssFile := createFile("rss.xml")
	atomFile := createFile("atom.xml")

	feed := &feeds.Feed{
		Title:       "GopherPods",
		Link:        &feeds.Link{Href: "https://gopherpods.netlify.com"},
		Description: "GopherPods is a community-driven list of podcast episodes that cover the Go programming language and Go related projects.",
		Author:      &feeds.Author{Name: "John Reuterswärd", Email: "john.reutersward@gmail.com"},
	}

	for _, episode := range episodes {
		item := &feeds.Item{
			Title:       episode.Title,
			Description: episode.About,
			Link:        &feeds.Link{Href: episode.URL},
			Author:      &feeds.Author{Name: episode.Show},
			Created:     time.Time(episode.Date),
		}
		feed.Add(item)
	}

	feed.Created = feed.Items[0].Created

	if err := feed.WriteRss(rssFile); err != nil {
		log.Fatal(err)
	}

	if err := feed.WriteAtom(atomFile); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Add ability to get new episodes from a feed.<commit_after>package main

import (
	"encoding/json"
	"flag"
	"html/template"
	"log"
	"os"
	"sort"
	"time"

	"github.com/SlyMarbo/rss"
	"github.com/gorilla/feeds"
	"github.com/jaytaylor/html2text"
)

const (
	dateLayout        = `"2006-01-02"`
	displayDateLayout = "02 Jan 2006"
)

var tmpl = template.Must(template.ParseFiles("index.gohtml"))

type episode struct {
	Show  string `json:"show"`
	Title string `json:"title"`
	About string `json:"about"`
	URL   string `json:"url"`
	Date  date   `json:"date"`
}

func (e *episode) GetDisplayDate() string { return (time.Time)(e.Date).Format(displayDateLayout) }

type date time.Time

func (d date) Before(t date) bool            { return (time.Time)(d).Before(time.Time(t)) }
func (d date) Format() string                { return (time.Time)(d).Format(dateLayout) }
func (d *date) MarshalJSON() ([]byte, error) { return []byte(d.Format()), nil }
func (d *date) UnmarshalJSON(data []byte) error {
	t, err := time.Parse(dateLayout, string(data))
	*d = date(t)
	return err
}

type episodes []episode

func (e episodes) Len() int               { return len(e) }
func (e episodes) Less(i int, j int) bool { return e[i].Date.Before(e[j].Date) }
func (e episodes) Swap(i int, j int)      { e[i], e[j] = e[j], e[i] }

func main() {
	feedUrl := flag.String("feed", "", "Get new episodes from feed URL and add to episodes.json.")
	feedStartUrl := flag.String("start-url", "", "Start from episode with this url when using -feed.")
	flag.Parse()

	if *feedUrl == "" {
		episodes := parseEpisodes()
		sort.Sort(sort.Reverse(episodes))
		createSite(episodes)
		createFeeds(episodes)
	} else {
		newEpisodes := getEpisodesFromFeed(*feedUrl, *feedStartUrl)
		episodes := parseEpisodes()
		episodes = append(episodes, newEpisodes...)
		sort.Sort(episodes)
		writeFeed(episodes)
	}
}

func parseEpisodes() episodes {
	episodesFile, err := os.Open("episodes.json")
	if err != nil {
		log.Fatal(err)
	}
	defer episodesFile.Close()

	var episodes episodes
	if err := json.NewDecoder(episodesFile).Decode(&episodes); err != nil {
		log.Fatal(err)
	}

	return episodes
}

func createSite(episodes episodes) {
	indexFile := createFile("static/index.html")
	defer indexFile.Close()

	if err := tmpl.ExecuteTemplate(indexFile, "index", episodes); err != nil {
		log.Fatal(err)
	}
}

func createFeeds(episodes episodes) {
	rssFile := createFile("static/rss.xml")
	defer rssFile.Close()

	atomFile := createFile("static/atom.xml")
	defer atomFile.Close()

	feed := &feeds.Feed{
		Title:       "GopherPods",
		Link:        &feeds.Link{Href: "https://gopherpods.netlify.com"},
		Description: "GopherPods is a community-driven list of podcast episodes that cover the Go programming language and Go related projects.",
		Author:      &feeds.Author{Name: "John Reuterswärd", Email: "john.reutersward@gmail.com"},
	}

	for _, episode := range episodes {
		item := &feeds.Item{
			Title:       episode.Title,
			Description: episode.About,
			Link:        &feeds.Link{Href: episode.URL},
			Author:      &feeds.Author{Name: episode.Show},
			Created:     time.Time(episode.Date),
		}
		feed.Add(item)
	}

	feed.Created = feed.Items[0].Created

	if err := feed.WriteRss(rssFile); err != nil {
		log.Fatal(err)
	}

	if err := feed.WriteAtom(atomFile); err != nil {
		log.Fatal(err)
	}
}

func getEpisodesFromFeed(feedUrl string, startUrl string) episodes {
	feed, err := rss.Fetch(feedUrl)
	if err != nil {
		log.Fatal(err)
	}

	var newEpisodes episodes
	for i := range feed.Items {
		about, err := html2text.FromString(feed.Items[i].Summary, html2text.Options{OmitLinks: true})
		if err != nil {
			log.Fatal(err)
		}

		newEpisode := episode{
			Show:  feed.Title,
			Title: feed.Items[i].Title,
			About: about,
			URL:   feed.Items[i].Link,
			Date:  date(feed.Items[i].Date),
		}
		newEpisodes = append(newEpisodes, newEpisode)
	}

	var episodes episodes

	if startUrl != "" {
		sort.Sort(newEpisodes)
		var foundStart bool
		for i := range newEpisodes {
			if !foundStart {
				if startUrl == newEpisodes[i].URL {
					foundStart = true
				} else {
					continue
				}
			}
			episodes = append(episodes, newEpisodes[i])
		}
	} else {
		episodes = newEpisodes
	}

	return episodes
}

func writeFeed(episodes episodes) {
	episodesFile := createFile("episodes.json")
	defer episodesFile.Close()

	enc := json.NewEncoder(episodesFile)
	enc.SetIndent("", "	")
	if err := enc.Encode(episodes); err != nil {
		log.Fatal(err)
	}
}

func createFile(filePath string) *os.File {
	file, err := os.Create(filePath)
	if err != nil {
		log.Fatal(err)
	}
	return file
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/csv"
	"errors"
	"fmt"
	"reflect"
	"strconv"
	"strings"
)

func main() {
	in := `A,B,C,D
1,2,3,4
5,6,7,8`
	//textColumn := []string{"One", "Two", "Three"}
	//intColumn := []int{1, 2, 3}
	//c1 := Column{}
	//c1.fillColumn(textColumn)
	//c2 := Column{}
	//c2.fillColumn(intColumn)
	df := DataFrame{}
	//df := DataFrame{
	//columns:  []Column{c1, c2},
	//nCols:    2,
	//nRows:    3,
	//colnames: []string{"Text", "Ints"},
	//}
	//fmt.Println(df)
	df.readCsvFromStringToString(in)

	//type mystruct struct {
	//a int
	//b string
	//c int
	//d float64
	//}
	//type dataframe []mystruct

	//r := csv.NewReader(strings.NewReader(in))
	//records, err := r.ReadAll()
	//if err != nil {
	//panic(err.Error())
	//}
	//headers := records[0]
	//for _, v := range records[1:] {
	//mystr := make(map[string]interface{})
	//for k, m := range headers {
	//mystr[m] = v[k]
	//}
	//fmt.Println(mystr)
	//}
}

type DataFrame struct {
	columns  []Column
	nCols    int
	nRows    int
	colnames []string
	coltypes []string
}

//func (df *DataFrame) readCsvFromString(in string, out string) error {
//return nil
//}
func (df *DataFrame) readCsvFromStringTyped(in string, types []string) error {
	r := csv.NewReader(strings.NewReader(in))
	records, err := r.ReadAll()
	if err != nil {
		return err
	}

	// TODO: Check if empty records

	// Get DataFrame dimensions
	nRows := len(records) - 1
	if nRows == 0 {
		return errors.New("Empty dataframe")
	}
	nCols := len(records[0])

	// Generate a virtual df to store the temporary values
	newDf := DataFrame{
		columns:  []Column{},
		colnames: records[0],
		nRows:    nRows,
		nCols:    nCols,
	}

	for j := 0; j < nCols; j++ {
		col := []string{}
		for i := 1; i < nRows+1; i++ {
			// TODO: Parse the column elements with the appropriate type
			col = append(col, records[i][j])
		}
		column := Column{}
		column.fillColumn(col)
		newDf.columns = append(newDf.columns, column)
	}
	fmt.Println(newDf)

	//fmt.Println(nRows)
	//fmt.Println(nCols)
	//fmt.Println(records)
	return nil
}

func (df *DataFrame) readCsvFromStringToString(in string) error {
	r := csv.NewReader(strings.NewReader(in))
	records, err := r.ReadAll()
	if err != nil {
		return err
	}

	// TODO: Check if empty records

	// Get DataFrame dimensions
	nRows := len(records) - 1
	if nRows == 0 {
		return errors.New("Empty dataframe")
	}
	nCols := len(records[0])

	// Generate a virtual df to store the temporary values
	newDf := DataFrame{
		columns:  []Column{},
		colnames: records[0],
		nRows:    nRows,
		nCols:    nCols,
	}

	for j := 0; j < nCols; j++ {
		col := []string{}
		for i := 1; i < nRows+1; i++ {
			col = append(col, records[i][j])
		}
		column := Column{}
		column.fillColumn(col)
		newDf.columns = append(newDf.columns, column)
	}
	fmt.Println(newDf)

	//fmt.Println(nRows)
	//fmt.Println(nCols)
	//fmt.Println(records)
	return nil
}

func (df DataFrame) String() string {
	str := ""
	if len(df.colnames) != 0 {
		str += "\t"
		for _, v := range df.colnames {
			str += v
			str += "\t"
		}
		str += "\n"
		str += "\n"
	}
	for i := 0; i < df.nRows; i++ {
		str += strconv.Itoa(i+1) + ":\t"
		for j := 0; j < df.nCols; j++ {
			str += fmt.Sprint(df.columns[j].row[i])
			str += "\t"
		}
		str += "\n"
	}
	return str
}

type Column struct {
	row []interface{}
}

func (c *Column) fillColumn(values interface{}) {
	switch reflect.TypeOf(values).Kind() {
	case reflect.Slice:
		s := reflect.ValueOf(values)
		for i := 0; i < s.Len(); i++ {
			c.row = append(c.row, s.Index(i).Interface())
		}
	}
}
<commit_msg>Polishing the initial concept with @DataFrame.loadData now requires a [][]string object and always create strings columns. Column structure will now carry the column type<commit_after>package main

import (
	"encoding/csv"
	"errors"
	"fmt"
	"reflect"
	"strconv"
	"strings"
)

func main() {
	// Test 02
	//textColumn := []string{"One", "Two", "Three"}
	//intColumn := []int{1, 2, 3}
	//c1 := Column{}
	//c1.fillColumn(textColumn)
	//c2 := Column{}
	//c2.fillColumn(intColumn)
	//df := DataFrame{
	//columns:  []Column{c1, c2},
	//nCols:    2,
	//nRows:    3,
	//colnames: []string{"Text", "Ints"},
	//}
	//fmt.Println(df)

	// Test 01
	in := `A,B,C,D
1,2,3,4
5,6,7,8`
	df := DataFrame{}
	r := csv.NewReader(strings.NewReader(in))
	records, err := r.ReadAll()
	if err != nil {
		panic(err)
	}
	df.loadData(records)

	for _, v := range df.columns {
		fmt.Println(v)
	}
	fmt.Println(df)
}

// DataFrame Definition
// ====================
type DataFrame struct {
	columns  []Column
	colnames []string
	nCols    int
	nRows    int
}

// DataFrame Methods
// =================
func (df *DataFrame) loadData(records [][]string) error {
	// TODO: Check if empty records

	// Get DataFrame dimensions
	nRows := len(records) - 1
	if nRows == 0 {
		return errors.New("Empty dataframe")
	}
	nCols := len(records[0])

	// Generate a virtual df to store the temporary values
	newDf := DataFrame{
		columns:  []Column{},
		colnames: records[0],
		nRows:    nRows,
		nCols:    nCols,
	}

	for j := 0; j < nCols; j++ {
		col := []string{}
		for i := 1; i < nRows+1; i++ {
			col = append(col, records[i][j])
		}
		column := Column{}
		column.fillColumn(col)
		newDf.columns = append(newDf.columns, column)
	}
	*df = newDf
	return nil
}

func (df DataFrame) String() string {
	str := ""
	if len(df.colnames) != 0 {
		str += "\t"
		for _, v := range df.colnames {
			str += v
			str += "\t"
		}
		str += "\n"
		str += "\n"
	}
	for i := 0; i < df.nRows; i++ {
		str += strconv.Itoa(i+1) + ":\t"
		for j := 0; j < df.nCols; j++ {
			str += fmt.Sprint(df.columns[j].row[i])
			str += "\t"
		}
		str += "\n"
	}
	return str
}

// Column Definition
// =================
type Column struct {
	row     []interface{}
	colType string
}

// Column Methods
// ==============
func (c Column) String() string {
	return fmt.Sprint(c.row)
}

// TODO: Should this return an error?
func (c *Column) fillColumn(values interface{}) {
	switch reflect.TypeOf(values).Kind() {
	case reflect.Slice:
		s := reflect.ValueOf(values)
		for i := 0; i < s.Len(); i++ {
			c.row = append(c.row, s.Index(i).Interface())
			c.colType = fmt.Sprint(s.Index(i).Type())
		}
	}
}
<|endoftext|>


<commit_before>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package awsproxy

import (
	"context"
	"fmt"
	"net/http"
	"net/http/httptest"
	"os"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
	"go.opentelemetry.io/collector/component"
	"go.opentelemetry.io/collector/component/componenttest"
	"go.opentelemetry.io/collector/config"
	"go.opentelemetry.io/collector/config/confignet"
	"go.opentelemetry.io/collector/config/configtest"

	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/proxy"
	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/testutil"
)

func TestFactory_CreateDefaultConfig(t *testing.T) {
	cfg := createDefaultConfig()
	assert.Equal(t, &Config{
		ExtensionSettings: config.NewExtensionSettings(config.NewComponentID(typeStr)),
		ProxyConfig: proxy.Config{
			TCPAddr: confignet.TCPAddr{
				Endpoint: defaultEndpoint,
			},
		},
	}, cfg)

	assert.NoError(t, configtest.CheckConfigStruct(cfg))
}

func TestFactory_CreateExtension(t *testing.T) {
	backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		auth := r.Header.Get("Authorization")
		// Verify a signature was added, indicating the reverse proxy is doing its job.
		if !strings.HasPrefix(auth, "AWS4-HMAC-SHA256") {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintln(w, "No signature")
			return
		}
		w.Header().Set("Test", "Passed")
		fmt.Fprintln(w, "OK")
	}))
	defer backend.Close()

	cfg := createDefaultConfig().(*Config)
	address := testutil.GetAvailableLocalAddress(t)
	cfg.ProxyConfig.AWSEndpoint = backend.URL
	cfg.ProxyConfig.TCPAddr.Endpoint = address
	cfg.ProxyConfig.Region = "us-east-2"

	// Simplest way to get SDK to use fake credentials
	os.Setenv("AWS_ACCESS_KEY_ID", "fakeAccessKeyID")
	os.Setenv("AWS_SECRET_ACCESS_KEY", "fakeSecretAccessKey")

	ctx := context.Background()
	ext, err := createExtension(ctx, componenttest.NewNopExtensionCreateSettings(), cfg)
	assert.NoError(t, err)
	assert.NotNil(t, ext)

	mh := newAssertNoErrorHost(t)
	err = ext.Start(ctx, mh)
	assert.NoError(t, err)

	resp, err := http.Post(
		"http://"+address+"/GetSamplingRules",
		"application/json",
		strings.NewReader(`{"NextToken": null}`))

	assert.NoError(t, err)

	assert.Equal(t, http.StatusOK, resp.StatusCode)
	assert.Equal(t, "Passed", resp.Header.Get("Test"))

	err = ext.Shutdown(ctx)
	assert.NoError(t, err)
}

// assertNoErrorHost implements a component.Host that asserts that there were no errors.
type assertNoErrorHost struct {
	component.Host
	*testing.T
}

// newAssertNoErrorHost returns a new instance of assertNoErrorHost.
func newAssertNoErrorHost(t *testing.T) component.Host {
	return &assertNoErrorHost{
		Host: componenttest.NewNopHost(),
		T:    t,
	}
}

func (aneh *assertNoErrorHost) ReportFatalError(err error) {
	assert.NoError(aneh, err)
}
<commit_msg>Fix flaky awsproxy - TestFactory_CreateExtension Test (#6157)<commit_after>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package awsproxy

import (
	"context"
	"fmt"
	"net/http"
	"net/http/httptest"
	"os"
	"strings"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"go.opentelemetry.io/collector/component"
	"go.opentelemetry.io/collector/component/componenttest"
	"go.opentelemetry.io/collector/config"
	"go.opentelemetry.io/collector/config/confignet"
	"go.opentelemetry.io/collector/config/configtest"

	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/proxy"
	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/testutil"
)

func TestFactory_CreateDefaultConfig(t *testing.T) {
	cfg := createDefaultConfig()
	assert.Equal(t, &Config{
		ExtensionSettings: config.NewExtensionSettings(config.NewComponentID(typeStr)),
		ProxyConfig: proxy.Config{
			TCPAddr: confignet.TCPAddr{
				Endpoint: defaultEndpoint,
			},
		},
	}, cfg)

	assert.NoError(t, configtest.CheckConfigStruct(cfg))
}

func TestFactory_CreateExtension(t *testing.T) {
	backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		auth := r.Header.Get("Authorization")
		// Verify a signature was added, indicating the reverse proxy is doing its job.
		if !strings.HasPrefix(auth, "AWS4-HMAC-SHA256") {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintln(w, "No signature")
			return
		}
		w.Header().Set("Test", "Passed")
		fmt.Fprintln(w, "OK")
	}))
	defer backend.Close()

	cfg := createDefaultConfig().(*Config)
	address := testutil.GetAvailableLocalAddress(t)
	cfg.ProxyConfig.AWSEndpoint = backend.URL
	cfg.ProxyConfig.TCPAddr.Endpoint = address
	cfg.ProxyConfig.Region = "us-east-2"

	// Simplest way to get SDK to use fake credentials
	os.Setenv("AWS_ACCESS_KEY_ID", "fakeAccessKeyID")
	os.Setenv("AWS_SECRET_ACCESS_KEY", "fakeSecretAccessKey")

	ctx := context.Background()
	ext, err := createExtension(ctx, componenttest.NewNopExtensionCreateSettings(), cfg)
	assert.NoError(t, err)
	assert.NotNil(t, ext)

	mh := newAssertNoErrorHost(t)
	err = ext.Start(ctx, mh)
	assert.NoError(t, err)

	var resp *http.Response
	require.Eventually(t, func() bool {
		resp, err = http.Post(
			"http://"+address+"/GetSamplingRules",
			"application/json",
			strings.NewReader(`{"NextToken": null}`))
		return err == nil
	}, 3*time.Second, 10*time.Millisecond)

	assert.NoError(t, err)

	assert.Equal(t, http.StatusOK, resp.StatusCode)
	assert.Equal(t, "Passed", resp.Header.Get("Test"))

	err = ext.Shutdown(ctx)
	assert.NoError(t, err)
}

// assertNoErrorHost implements a component.Host that asserts that there were no errors.
type assertNoErrorHost struct {
	component.Host
	*testing.T
}

// newAssertNoErrorHost returns a new instance of assertNoErrorHost.
func newAssertNoErrorHost(t *testing.T) component.Host {
	return &assertNoErrorHost{
		Host: componenttest.NewNopHost(),
		T:    t,
	}
}

func (aneh *assertNoErrorHost) ReportFatalError(err error) {
	assert.NoError(aneh, err)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"regexp"
	"strings"
	"time"

	"github.com/rcrowley/go-metrics"
	"github.com/skia-dev/glog"
	"github.com/skia-dev/influxdb/client"
	"go.skia.org/infra/alertserver/go/alerting"
	"go.skia.org/infra/go/autoroll"
	"go.skia.org/infra/go/buildbot"
	"go.skia.org/infra/go/util"
)

/*
	This file contains goroutines which trigger more complex alerts than
	can be expressed using the rule format in alerts.cfg.
*/

const (
	ANDROID_DISCONNECT = `The Android device for %s appears to be disconnected.

Build: https://uberchromegw.corp.google.com/i/%s/builders/%s/builds/%d
Dashboard: https://status.skia.org/buildbots?botGrouping=buildslave&filterBy=buildslave&include=%%5E%s%%24&tab=builds
Host info: https://status.skia.org/hosts?filter=%s`
	AUTOROLL_ALERT_NAME = "AutoRoll Failed"
	BUILDSLAVE_OFFLINE  = `Buildslave %s is not connected to https://uberchromegw.corp.google.com/i/%s/buildslaves/%s

Dashboard: https://status.skia.org/buildbots?botGrouping=buildslave&filterBy=buildslave&include=%%5E%s%%24&tab=builds
Host info: https://status.skia.org/hosts?filter=%s`
	HUNG_BUILDSLAVE = `Possibly hung buildslave (%s)

A step has been running for over %s:
https://uberchromegw.corp.google.com/i/%s/builders/%s/builds/%d
Dashboard: https://status.skia.org/buildbots?botGrouping=buildslave&filterBy=buildslave&include=%%5E%s%%24&tab=builds
Host info: https://status.skia.org/hosts?filter=%s`
	UPDATE_SCRIPTS = `update_scripts failed on %s

Build: https://uberchromegw.corp.google.com/i/%s/builders/%s/builds/%d
Dashboard: https://status.skia.org/buildbots?botGrouping=builder&filterBy=builder&include=%%5E%s%%24&tab=builds
Host info: https://status.skia.org/hosts?filter=%s`
)

type BuildSlice []*buildbot.Build

func (s BuildSlice) Len() int {
	return len(s)
}

func (s BuildSlice) Less(i, j int) bool {
	return s[i].Finished < s[j].Finished
}

func (s BuildSlice) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func StartAlertRoutines(am *alerting.AlertManager, tickInterval time.Duration, c *client.Client) {
	emailAction, err := alerting.ParseAction("Email(infra-alerts@skia.org)")
	if err != nil {
		glog.Fatal(err)
	}
	actions := []alerting.Action{emailAction}

	// Disconnected buildslaves.
	go func() {
		seriesTmpl := "buildbot.buildslaves.%s.connected"
		re := regexp.MustCompile("[^A-Za-z0-9]+")
		for _ = range time.Tick(tickInterval) {
			glog.Info("Loading buildslave data.")
			slaves, err := buildbot.GetBuildSlaves()
			if err != nil {
				glog.Error(err)
				continue
			}
			for masterName, m := range slaves {
				for _, s := range m {
					v := int64(0)
					if s.Connected {
						v = int64(1)
					}
					metric := fmt.Sprintf(seriesTmpl, re.ReplaceAllString(s.Name, "_"))
					metrics.GetOrRegisterGauge(metric, metrics.DefaultRegistry).Update(v)
					if !s.Connected {
						// This buildslave is offline. Figure out which one it is.
						if err := am.AddAlert(&alerting.Alert{
							Name:        fmt.Sprintf("Buildslave %s offline", s.Name),
							Category:    alerting.INFRA_ALERT,
							Message:     fmt.Sprintf(BUILDSLAVE_OFFLINE, s.Name, masterName, s.Name, s.Name, s.Name),
							Nag:         int64(time.Hour),
							AutoDismiss: int64(2 * tickInterval),
							Actions:     actions,
						}); err != nil {
							glog.Error(err)
						}
					}
				}
			}
		}
	}()

	// AutoRoll failure.
	go func() {
		lastSearch := time.Now()
		for now := range time.Tick(time.Minute) {
			glog.Infof("Searching for DEPS rolls.")
			results, err := autoroll.GetRecentRolls(lastSearch)
			if err != nil {
				glog.Errorf("Failed to search for DEPS rolls: %v", err)
				continue
			}
			lastSearch = now
			activeAlert := am.ActiveAlert(AUTOROLL_ALERT_NAME)
			for _, issue := range results {
				if issue.Closed {
					if issue.Committed {
						if activeAlert != 0 {
							msg := fmt.Sprintf("Subsequent roll succeeded: %s/%d", autoroll.RIETVELD_URL, issue.Issue)
							if err := am.Dismiss(activeAlert, alerting.USER_ALERTSERVER, msg); err != nil {
								glog.Error(err)
							}
						}
					} else {
						if err := am.AddAlert(&alerting.Alert{
							Name:    AUTOROLL_ALERT_NAME,
							Message: fmt.Sprintf("DEPS roll failed: %s/%d", autoroll.RIETVELD_URL, issue.Issue),
							Nag:     int64(3 * time.Hour),
							Actions: actions,
						}); err != nil {
							glog.Error(err)
						}
					}
				}
			}
		}
	}()

	// Android device disconnects, hung buildslaves.
	go func() {
		// These builders are frequently slow. Ignore them when looking for hung buildslaves.
		hungSlavesIgnore := []string{
			"Housekeeper-Nightly-RecreateSKPs_Canary",
			"Housekeeper-Weekly-RecreateSKPs",
			"Linux Builder",
			"Mac Builder",
			"Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Valgrind",
			"Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind",
			"Win Builder",
		}
		hangTimePeriod := 2 * time.Hour
		for _ = range time.Tick(tickInterval) {
			glog.Infof("Searching for hung buildslaves and disconnected Android devices.")
			builds, err := buildbot.GetUnfinishedBuilds()
			if err != nil {
				glog.Error(err)
				continue
			}
			for _, b := range builds {
				// Disconnected Android device?
				disconnectedAndroid := false
				if strings.Contains(b.Builder, "Android") && !strings.Contains(b.Builder, "Build") {
					for _, s := range b.Steps {
						if strings.Contains(s.Name, "wait for device") {
							// If "wait for device" has been running for 10 minutes, the device is probably offline.
							if s.Finished == 0 && time.Since(time.Unix(int64(s.Started), 0)) > 10*time.Minute {
								if err := am.AddAlert(&alerting.Alert{
									Name:     fmt.Sprintf("Android device disconnected (%s)", b.BuildSlave),
									Category: alerting.INFRA_ALERT,
									Message:  fmt.Sprintf(ANDROID_DISCONNECT, b.BuildSlave, b.Master, b.Builder, b.Number, b.BuildSlave, b.BuildSlave),
									Nag:      int64(3 * time.Hour),
									Actions:  actions,
								}); err != nil {
									glog.Error(err)
								}
								disconnectedAndroid = true
							}
						}
					}
				}
				if !disconnectedAndroid && !util.ContainsAny(b.Builder, hungSlavesIgnore) {
					// Hung buildslave?
					for _, s := range b.Steps {
						if s.Name == "steps" {
							continue
						}
						// If the step has been running for over an hour, it's probably hung.
						if s.Finished == 0 && time.Since(time.Unix(int64(s.Started), 0)) > hangTimePeriod {
							if err := am.AddAlert(&alerting.Alert{
								Name:        fmt.Sprintf("Possibly hung buildslave (%s)", b.BuildSlave),
								Category:    alerting.INFRA_ALERT,
								Message:     fmt.Sprintf(HUNG_BUILDSLAVE, b.BuildSlave, hangTimePeriod.String(), b.Master, b.Builder, b.Number, b.BuildSlave, b.BuildSlave),
								Nag:         int64(time.Hour),
								Actions:     actions,
								AutoDismiss: int64(10 * tickInterval),
							}); err != nil {
								glog.Error(err)
							}
						}
					}
				}
			}
		}
	}()

	// Failed update_scripts.
	go func() {
		lastSearch := time.Now()
		for _ = range time.Tick(tickInterval) {
			glog.Infof("Searching for builds which failed update_scripts.")
			currentSearch := time.Now()
			builds, err := buildbot.GetBuildsFromDateRange(lastSearch, currentSearch)
			lastSearch = currentSearch
			if err != nil {
				glog.Error(err)
				continue
			}
			for _, b := range builds {
				for _, s := range b.Steps {
					if s.Name == "update_scripts" {
						if s.Results != 0 {
							if err := am.AddAlert(&alerting.Alert{
								Name:     "update_scripts failed",
								Category: alerting.INFRA_ALERT,
								Message:  fmt.Sprintf(UPDATE_SCRIPTS, b.Builder, b.Master, b.Builder, b.Number, b.Builder, b.BuildSlave),
								Actions:  actions,
							}); err != nil {
								glog.Error(err)
							}
						}
						break
					}
				}
			}
		}
	}()
}
<commit_msg>Exclude canary buildslaves from offline alerts<commit_after>package main

import (
	"fmt"
	"regexp"
	"strings"
	"time"

	"github.com/rcrowley/go-metrics"
	"github.com/skia-dev/glog"
	"github.com/skia-dev/influxdb/client"
	"go.skia.org/infra/alertserver/go/alerting"
	"go.skia.org/infra/go/autoroll"
	"go.skia.org/infra/go/buildbot"
	"go.skia.org/infra/go/util"
)

/*
	This file contains goroutines which trigger more complex alerts than
	can be expressed using the rule format in alerts.cfg.
*/

const (
	ANDROID_DISCONNECT = `The Android device for %s appears to be disconnected.

Build: https://uberchromegw.corp.google.com/i/%s/builders/%s/builds/%d
Dashboard: https://status.skia.org/buildbots?botGrouping=buildslave&filterBy=buildslave&include=%%5E%s%%24&tab=builds
Host info: https://status.skia.org/hosts?filter=%s`
	AUTOROLL_ALERT_NAME = "AutoRoll Failed"
	BUILDSLAVE_OFFLINE  = `Buildslave %s is not connected to https://uberchromegw.corp.google.com/i/%s/buildslaves/%s

Dashboard: https://status.skia.org/buildbots?botGrouping=buildslave&filterBy=buildslave&include=%%5E%s%%24&tab=builds
Host info: https://status.skia.org/hosts?filter=%s`
	HUNG_BUILDSLAVE = `Possibly hung buildslave (%s)

A step has been running for over %s:
https://uberchromegw.corp.google.com/i/%s/builders/%s/builds/%d
Dashboard: https://status.skia.org/buildbots?botGrouping=buildslave&filterBy=buildslave&include=%%5E%s%%24&tab=builds
Host info: https://status.skia.org/hosts?filter=%s`
	UPDATE_SCRIPTS = `update_scripts failed on %s

Build: https://uberchromegw.corp.google.com/i/%s/builders/%s/builds/%d
Dashboard: https://status.skia.org/buildbots?botGrouping=builder&filterBy=builder&include=%%5E%s%%24&tab=builds
Host info: https://status.skia.org/hosts?filter=%s`
)

var BUILDSLAVE_OFFLINE_BLACKLIST = []string{
	"build3-a3",
	"build4-a3",
	"vm255-m3",
}

type BuildSlice []*buildbot.Build

func (s BuildSlice) Len() int {
	return len(s)
}

func (s BuildSlice) Less(i, j int) bool {
	return s[i].Finished < s[j].Finished
}

func (s BuildSlice) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func StartAlertRoutines(am *alerting.AlertManager, tickInterval time.Duration, c *client.Client) {
	emailAction, err := alerting.ParseAction("Email(infra-alerts@skia.org)")
	if err != nil {
		glog.Fatal(err)
	}
	actions := []alerting.Action{emailAction}

	// Disconnected buildslaves.
	go func() {
		seriesTmpl := "buildbot.buildslaves.%s.connected"
		re := regexp.MustCompile("[^A-Za-z0-9]+")
		for _ = range time.Tick(tickInterval) {
			glog.Info("Loading buildslave data.")
			slaves, err := buildbot.GetBuildSlaves()
			if err != nil {
				glog.Error(err)
				continue
			}
			for masterName, m := range slaves {
				for _, s := range m {
					if util.In(s.Name, BUILDSLAVE_OFFLINE_BLACKLIST) {
						continue
					}
					v := int64(0)
					if s.Connected {
						v = int64(1)
					}
					metric := fmt.Sprintf(seriesTmpl, re.ReplaceAllString(s.Name, "_"))
					metrics.GetOrRegisterGauge(metric, metrics.DefaultRegistry).Update(v)
					if !s.Connected {
						// This buildslave is offline. Figure out which one it is.
						if err := am.AddAlert(&alerting.Alert{
							Name:        fmt.Sprintf("Buildslave %s offline", s.Name),
							Category:    alerting.INFRA_ALERT,
							Message:     fmt.Sprintf(BUILDSLAVE_OFFLINE, s.Name, masterName, s.Name, s.Name, s.Name),
							Nag:         int64(time.Hour),
							AutoDismiss: int64(2 * tickInterval),
							Actions:     actions,
						}); err != nil {
							glog.Error(err)
						}
					}
				}
			}
		}
	}()

	// AutoRoll failure.
	go func() {
		lastSearch := time.Now()
		for now := range time.Tick(time.Minute) {
			glog.Infof("Searching for DEPS rolls.")
			results, err := autoroll.GetRecentRolls(lastSearch)
			if err != nil {
				glog.Errorf("Failed to search for DEPS rolls: %v", err)
				continue
			}
			lastSearch = now
			activeAlert := am.ActiveAlert(AUTOROLL_ALERT_NAME)
			for _, issue := range results {
				if issue.Closed {
					if issue.Committed {
						if activeAlert != 0 {
							msg := fmt.Sprintf("Subsequent roll succeeded: %s/%d", autoroll.RIETVELD_URL, issue.Issue)
							if err := am.Dismiss(activeAlert, alerting.USER_ALERTSERVER, msg); err != nil {
								glog.Error(err)
							}
						}
					} else {
						if err := am.AddAlert(&alerting.Alert{
							Name:    AUTOROLL_ALERT_NAME,
							Message: fmt.Sprintf("DEPS roll failed: %s/%d", autoroll.RIETVELD_URL, issue.Issue),
							Nag:     int64(3 * time.Hour),
							Actions: actions,
						}); err != nil {
							glog.Error(err)
						}
					}
				}
			}
		}
	}()

	// Android device disconnects, hung buildslaves.
	go func() {
		// These builders are frequently slow. Ignore them when looking for hung buildslaves.
		hungSlavesIgnore := []string{
			"Housekeeper-Nightly-RecreateSKPs_Canary",
			"Housekeeper-Weekly-RecreateSKPs",
			"Linux Builder",
			"Mac Builder",
			"Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Valgrind",
			"Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind",
			"Win Builder",
		}
		hangTimePeriod := 2 * time.Hour
		for _ = range time.Tick(tickInterval) {
			glog.Infof("Searching for hung buildslaves and disconnected Android devices.")
			builds, err := buildbot.GetUnfinishedBuilds()
			if err != nil {
				glog.Error(err)
				continue
			}
			for _, b := range builds {
				// Disconnected Android device?
				disconnectedAndroid := false
				if strings.Contains(b.Builder, "Android") && !strings.Contains(b.Builder, "Build") {
					for _, s := range b.Steps {
						if strings.Contains(s.Name, "wait for device") {
							// If "wait for device" has been running for 10 minutes, the device is probably offline.
							if s.Finished == 0 && time.Since(time.Unix(int64(s.Started), 0)) > 10*time.Minute {
								if err := am.AddAlert(&alerting.Alert{
									Name:     fmt.Sprintf("Android device disconnected (%s)", b.BuildSlave),
									Category: alerting.INFRA_ALERT,
									Message:  fmt.Sprintf(ANDROID_DISCONNECT, b.BuildSlave, b.Master, b.Builder, b.Number, b.BuildSlave, b.BuildSlave),
									Nag:      int64(3 * time.Hour),
									Actions:  actions,
								}); err != nil {
									glog.Error(err)
								}
								disconnectedAndroid = true
							}
						}
					}
				}
				if !disconnectedAndroid && !util.ContainsAny(b.Builder, hungSlavesIgnore) {
					// Hung buildslave?
					for _, s := range b.Steps {
						if s.Name == "steps" {
							continue
						}
						// If the step has been running for over an hour, it's probably hung.
						if s.Finished == 0 && time.Since(time.Unix(int64(s.Started), 0)) > hangTimePeriod {
							if err := am.AddAlert(&alerting.Alert{
								Name:        fmt.Sprintf("Possibly hung buildslave (%s)", b.BuildSlave),
								Category:    alerting.INFRA_ALERT,
								Message:     fmt.Sprintf(HUNG_BUILDSLAVE, b.BuildSlave, hangTimePeriod.String(), b.Master, b.Builder, b.Number, b.BuildSlave, b.BuildSlave),
								Nag:         int64(time.Hour),
								Actions:     actions,
								AutoDismiss: int64(10 * tickInterval),
							}); err != nil {
								glog.Error(err)
							}
						}
					}
				}
			}
		}
	}()

	// Failed update_scripts.
	go func() {
		lastSearch := time.Now()
		for _ = range time.Tick(tickInterval) {
			glog.Infof("Searching for builds which failed update_scripts.")
			currentSearch := time.Now()
			builds, err := buildbot.GetBuildsFromDateRange(lastSearch, currentSearch)
			lastSearch = currentSearch
			if err != nil {
				glog.Error(err)
				continue
			}
			for _, b := range builds {
				for _, s := range b.Steps {
					if s.Name == "update_scripts" {
						if s.Results != 0 {
							if err := am.AddAlert(&alerting.Alert{
								Name:     "update_scripts failed",
								Category: alerting.INFRA_ALERT,
								Message:  fmt.Sprintf(UPDATE_SCRIPTS, b.Builder, b.Master, b.Builder, b.Number, b.Builder, b.BuildSlave),
								Actions:  actions,
							}); err != nil {
								glog.Error(err)
							}
						}
						break
					}
				}
			}
		}
	}()
}
<|endoftext|>


<commit_before>package groupmebot

import (
	"bufio"
	"encoding/csv"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"strings"
)

type GroupMeBot struct {
	ID      string `json:"bot_id"`
	GroupID string `json:"group_id"`
	Host    string `json:"host"`
	Port    string `json:"port"`
	LogFile string `json:logfile`
	Server  string
	Hooks   map[string]func(InboundMessage) string
	ID               string `json:"bot_id"`
	GroupID          string `json:"group_id"`
	Host             string `json:"host"`
	Port             string `json:"port"`
	LogFile          string `json:"logfile"`
	LogMethod        string `json:"logmethod"`
	Server           string
	TrackBotMessages bool `json:"trackbotmessags"`
	TrackBotMessages bool `json:"trackbotmessages"`
	Hooks            map[string]func(InboundMessage) string
}

type InboundMessage struct {
	Avatar_url  string `json:"avatar_url"`
	Id          string `json:"id"`
	Name        string `json:"name"`
	Sender_id   string `json:"sender_id"`
	Sender_type string `json:"sender_type"`
	System      bool   `json:"system"`
	Text        string `json:"text"`
	User_id     string `json:"user_id"`
}

type OutboundMessage struct {
	ID   string `json:"bot_id"`
	Text string `json:"text"`
}

/// NewBotFromJson (json cfg file name)
/// This reads a json file containing the keys
/// See the example bot_cfg.json
/// Returns err from ioutil if file can not be read
func NewBotFromJson(filename string) (*GroupMeBot, error) {
	file, err := ioutil.ReadFile(filename)

	var bot GroupMeBot
	if err != nil {
		log.Fatal("Error reading bot configuration json file")
		return nil, err
	}

	// Parse out information from file
	json.Unmarshal(file, &bot)

	bot.Server = bot.Host + ":" + bot.Port
	log.Printf("Creating bot at %s\nLogging at %s\n", bot.Server, bot.LogFile)
	bot.Hooks = make(map[string]func(InboundMessage) string)

	return &bot, err
}

func (b *GroupMeBot) SendMessage(outMessage string) (*http.Response, error) {
	msg := OutboundMessage{b.ID, outMessage}
	payload, err := json.Marshal(msg)
	if err != nil {
		return nil, err
	}

	j_payload := string(payload)
	return http.Post("https://api.groupme.com/v3/bots/post", "application/json", strings.NewReader(j_payload))
}

func (b *GroupMeBot) AddHook(trigger string, response func(InboundMessage) string) {
	b.Hooks[trigger] = response
}

func (b *GroupMeBot) HandleMessage(msg InboundMessage) {
	resp := ""
	for trig, hook := range b.Hooks {
		matched, err := regexp.MatchString(trig, msg.Text)

		if matched {
			resp = hook(msg)
		} else if err != nil {
			log.Fatal("Error matching:", err)
		}

	}
	if len(resp) > 0 {
		log.Printf("Sending message: %v\n", resp)
		_, err := b.SendMessage(resp)
		if err != nil {
			log.Fatal("Error when sending.", err)
		}
	}

}

func (b *GroupMeBot) LogMessage(msg InboundMessage) {
	id := fmt.Sprintf("%s", msg.Sender_id)
	txt := fmt.Sprintf("%s", msg.Text)
	name := fmt.Sprintf("%s", msg.Name)
	values := []string{id, txt, name}

	log.Printf("%s: %s [Type: %s]\n", msg.Name, msg.Text, msg.Sender_type)

	f, err := os.OpenFile(b.LogFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	if err != nil {
		log.Fatal("Couldn't open file to log messages")
	}

	defer f.Close()
	fwriter := bufio.NewWriter(f)
	csvWriter := csv.NewWriter(fwriter)

	csvWriter.Write(values)
	csvWriter.Flush()
	fwriter.Flush()
}

/*
 This is legitimate black magic, this is pretty cool, not usually able to do
 things like this in other languages. This is a function that takes
 a list of trigger functions and returns a function that can handle the Server
 Requests
*/
func (b *GroupMeBot) Handler() http.HandlerFunc {
	// Request Handler function

	return func(w http.ResponseWriter, req *http.Request) {
		if req.Method == "POST" {
			//log.Println("Bot recieving and handling message.")
			defer req.Body.Close()
			var msg InboundMessage
			err := json.NewDecoder(req.Body).Decode(&msg)
			if err != nil {
				log.Println("Couldn't parse the request body")
				msg.Sender_type = "bot"
			}
			if msg.Sender_type != "bot" || b.TrackBotMessages {
				b.LogMessage(msg)
				// Find hook by running through hooklist
				b.HandleMessage(msg)
			}
		} else {
			//log.Println("Bot not responding to unknown message")
			//io.WriteString(w, "GOTEM")
		}
	}
}
<commit_msg>Removed duplicate fields<commit_after>package groupmebot

import (
	"bufio"
	"encoding/csv"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"strings"
)

type GroupMeBot struct {
	ID               string `json:"bot_id"`
	GroupID          string `json:"group_id"`
	Host             string `json:"host"`
	Port             string `json:"port"`
	LogFile          string `json:"logfile"`
	LogMethod        string `json:"logmethod"`
	Server           string
	TrackBotMessages bool `json:"trackbotmessages"`
	Hooks            map[string]func(InboundMessage) string
}

type InboundMessage struct {
	Id           string                   `json:"id"`
	Avatar_url   string                   `json:"avatar_url"`
	Name         string                   `json:"name"`
	Sender_id    string                   `json:"sender_id"`
	Sender_type  string                   `json:"sender_type"`
	System       bool                     `json:"system"`
	Text         string                   `json:"text"`
	Source_guid  string                   `json:"source_guid"`
	Created_at   int                      `json:"created_at"`
	User_id      string                   `json:"user_id"`
	Group_id     string                   `json:"group_id"`
	Favorited_by []string                 `json:"favorited_by"`
	Attachments  []map[string]interface{} `json:"attachments"`
}

type OutboundMessage struct {
	ID   string `json:"bot_id"`
	Text string `json:"text"`
}

/// NewBotFromJson (json cfg file name)
/// This reads a json file containing the keys
/// See the example bot_cfg.json
/// Returns err from ioutil if file can not be read
func NewBotFromJson(filename string) (*GroupMeBot, error) {
	file, err := ioutil.ReadFile(filename)

	var bot GroupMeBot
	if err != nil {
		log.Fatal("Error reading bot configuration json file")
		return nil, err
	}

	// Parse out information from file
	json.Unmarshal(file, &bot)

	bot.Server = bot.Host + ":" + bot.Port
	log.Printf("Creating bot at %s\nLogging at %s\n", bot.Server, bot.LogFile)
	bot.Hooks = make(map[string]func(InboundMessage) string)

	return &bot, err
}

func (b *GroupMeBot) SendMessage(outMessage string) (*http.Response, error) {
	msg := OutboundMessage{b.ID, outMessage}
	payload, err := json.Marshal(msg)
	if err != nil {
		return nil, err
	}

	j_payload := string(payload)
	return http.Post("https://api.groupme.com/v3/bots/post", "application/json", strings.NewReader(j_payload))
}

func (b *GroupMeBot) AddHook(trigger string, response func(InboundMessage) string) {
	b.Hooks[trigger] = response
}

func (b *GroupMeBot) HandleMessage(msg InboundMessage) {
	resp := ""
	for trig, hook := range b.Hooks {
		matched, err := regexp.MatchString(trig, msg.Text)

		if matched {
			resp = hook(msg)
		} else if err != nil {
			log.Fatal("Error matching:", err)
		}

	}
	if len(resp) > 0 {
		log.Printf("Sending message: %v\n", resp)
		_, err := b.SendMessage(resp)
		if err != nil {
			log.Fatal("Error when sending.", err)
		}
	}

}

func (b *GroupMeBot) LogMessage(msg InboundMessage) {
	id := fmt.Sprintf("%s", msg.Sender_id)
	txt := fmt.Sprintf("%s", msg.Text)
	name := fmt.Sprintf("%s", msg.Name)
	values := []string{id, txt, name}

	log.Printf("%s: %s [Type: %s]\n", msg.Name, msg.Text, msg.Sender_type)

	f, err := os.OpenFile(b.LogFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	if err != nil {
		log.Fatal("Couldn't open file to log messages")
	}

	defer f.Close()
	fwriter := bufio.NewWriter(f)
	csvWriter := csv.NewWriter(fwriter)

	csvWriter.Write(values)
	csvWriter.Flush()
	fwriter.Flush()
}

/*
 This is legitimate black magic, this is pretty cool, not usually able to do
 things like this in other languages. This is a function that takes
 a list of trigger functions and returns a function that can handle the Server
 Requests
*/
func (b *GroupMeBot) Handler() http.HandlerFunc {
	// Request Handler function

	return func(w http.ResponseWriter, req *http.Request) {
		if req.Method == "POST" {
			//log.Println("Bot recieving and handling message.")
			defer req.Body.Close()
			var msg InboundMessage
			err := json.NewDecoder(req.Body).Decode(&msg)
			if err != nil {
				log.Println("Couldn't parse the request body")
				msg.Sender_type = "bot"
			}
			if msg.Sender_type != "bot" || b.TrackBotMessages {
				b.LogMessage(msg)
				// Find hook by running through hooklist
				b.HandleMessage(msg)
			}
		} else {
			//log.Println("Bot not responding to unknown message")
			//io.WriteString(w, "GOTEM")
		}
	}
}
<|endoftext|>