<commit_before>/*
    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

// Package redis_queue provides a simple queue abstraction that is backed by Redis.
package redis_queue

import redis "github.com/garyburd/redigo/redis"

// Connect opens a connection to a Redis server and returns the connection.
// The connection should be closed by invoking Disconnect(conn),
// likely with defer.
func Connect(address string) (redis.Conn, error) {
	return redis.Dial("tcp", address)
}

// Close the Redis connection
func Disconnect(conn redis.Conn) {
	conn.Close()
}

// Push will perform a right-push onto a Redis list/queue with the supplied
// key and value.  An error will be returned if the operation failed.
func Push(conn redis.Conn, key string, value string) (error) {
	err := conn.Send("RPUSH", key, value)
	if err == nil {
		return conn.Flush()
	} else {
		return err
	}
}

// Pop will perform a blocking left-pop from a Redis list/queue with the supplied
// key.  An error will be returned if the operation failed.
func Pop(conn redis.Conn, key string, timeout int) (string, error) {
	rep, err := redis.Strings(conn.Do("BLPOP", key, timeout))
	if err == nil {
		return rep[1], nil
	} else {
		return "", err
	}
}

// Length will return the number of items in the specified list/queue
func Length(conn redis.Conn, key string) (int, error) {
	rep, err := redis.Int(conn.Do("LLEN", key))
	if err == nil {
		return rep, nil
	} else {
		return 0, err
	}
}
<commit_msg>Updated license in the Go source<commit_after>// Copyright 2014 Brighcove Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package redis_queue provides a simple queue abstraction that is backed by Redis.
package redis_queue

import redis "github.com/garyburd/redigo/redis"

// Connect opens a connection to a Redis server and returns the connection.
// The connection should be closed by invoking Disconnect(conn),
// likely with defer.
func Connect(address string) (redis.Conn, error) {
	return redis.Dial("tcp", address)
}

// Close the Redis connection
func Disconnect(conn redis.Conn) {
	conn.Close()
}

// Push will perform a right-push onto a Redis list/queue with the supplied
// key and value.  An error will be returned if the operation failed.
func Push(conn redis.Conn, key string, value string) (error) {
	err := conn.Send("RPUSH", key, value)
	if err == nil {
		return conn.Flush()
	} else {
		return err
	}
}

// Pop will perform a blocking left-pop from a Redis list/queue with the supplied
// key.  An error will be returned if the operation failed.
func Pop(conn redis.Conn, key string, timeout int) (string, error) {
	rep, err := redis.Strings(conn.Do("BLPOP", key, timeout))
	if err == nil {
		return rep[1], nil
	} else {
		return "", err
	}
}

// Length will return the number of items in the specified list/queue
func Length(conn redis.Conn, key string) (int, error) {
	rep, err := redis.Int(conn.Do("LLEN", key))
	if err == nil {
		return rep, nil
	} else {
		return 0, err
	}
}
<|endoftext|>


<commit_before>package cast

import (
	"testing"
	"sync"
	"log"
)

const msg int = 17

func TestCast(t *testing.T) {
	wgBefore := sync.WaitGroup{}
	wgAfter := sync.WaitGroup{}
	c := New()
	defer c.Close()
	for i := 1; i < 6; i++ {
		wgBefore.Add(1)
		go client(c, t, &wgBefore, &wgAfter)
	}
	// Give some time for goroutines to settle
	wgBefore.Wait()
	c.Send(msg)
	wgAfter.Wait()
}

func client(c *caster, t *testing.T, wg1, wg2 *sync.WaitGroup) {
	ch := make(chan interface{})
	c.Join(ch)
	wg1.Done()
	defer c.Leave(ch)
	defer wg2.Done()
	r := <-ch
	log.Printf("Got message %v", r)
	if r == msg {
		return
	} else {
		t.Errorf("Got wrong broadcast message")
	}

}
<commit_msg>Fix potential infinite wait<commit_after>package cast

import (
	"testing"
	"sync"
)

const msg int = 17

func TestCast(t *testing.T) {
	wgBefore := sync.WaitGroup{}
	wgAfter := sync.WaitGroup{}
	c := New()
	defer c.Close()
	for i := 1; i < 6; i++ {
		wgBefore.Add(1)
		go client(c, t, &wgBefore, &wgAfter)
	}
	// Give some time for goroutines to settle
	wgBefore.Wait()
	c.Send(msg)
	wgAfter.Wait()
}

func client(c *caster, t *testing.T, wg1, wg2 *sync.WaitGroup) {
	ch := make(chan interface{})
	c.Join(ch)
	wg1.Done()
	defer c.Leave(ch)
	defer wg2.Done()
	select {
	case r := <-ch:
		if r == msg {
			return
		} else {
			t.Errorf("Got wrong broadcast message")
		}
	}
	t.Error("Did not get anything :(")
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"net"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/alienth/go-fastly"
)

type IPList struct {
	name string
	IPs  []*net.IP
	Nets []*ipNet

	Limit           bool
	Requests        int64
	ListFile        string
	DimensionType   DimensionType `toml:"Dimension"`
	DimensionShared bool          `toml:"MeasureRateByDimension"`
	DimensionValues []string

	Time          duration
	Expire        duration
	LimitDuration duration

	sharedBuckets *sharedBucketMap
}

type IPLists map[string]*IPList

type sharedBucketMap struct {
	sync.RWMutex
	m map[Dimension]*rateBucket
}

func (l *IPList) init(name string) {
	l.name = name
	if l.ListFile != "" {
		l.readListFile()
	}
	l.sharedBuckets = &sharedBucketMap{m: make(map[Dimension]*rateBucket)}
}

func (l *IPList) contains(checkIP *net.IP) (bool, int) {
	if l == nil {
		return false, 0
	}
	for _, ip := range l.IPs {
		if ip.Equal(*checkIP) {
			return true, 32
		}
	}
	for _, net := range l.Nets {
		if net.Contains(*checkIP) {
			size, _ := net.IPNet.Mask.Size()
			return true, size
		}
	}

	return false, 0
}

// readListFile reads a ListFile and parses the content
// into the IPLists' IPs and Nets fields.
func (l *IPList) readListFile() error {
	f, err := os.Open(l.ListFile)
	if err != nil {
		return err
	}

	scanner := bufio.NewScanner(f)

	for scanner.Scan() {
		line := scanner.Text()
		if strings.Index(strings.TrimSpace(line), "#") == 0 {
			continue
		}
		if strings.TrimSpace(line) == "" {
			continue
		}
		if strings.Contains(line, "/") {
			_, parsedNet, err := net.ParseCIDR(line)
			if err != nil {
				return fmt.Errorf("Unable to parse CIDR.\nLine:\n%s\nError:\n%s\n", line, err)
			}
			if parsedNet != nil {
				var n ipNet
				n.IP = parsedNet.IP
				n.Mask = parsedNet.Mask
				l.Nets = append(l.Nets, &n)
			}
		} else {
			ip := net.ParseIP(line)
			if ip != nil {
				l.IPs = append(l.IPs, &ip)
			} else {
				return fmt.Errorf("Unable to parse IP address in list: %s\n", line)
			}
		}
	}

	return nil
}

func (lists IPLists) getRate(ip *net.IP) *ipRate {
	var ipr ipRate
	var ipList *IPList
	var maskSize int
	// Iterate through all of the lists and find the list with the most
	// specific match.
	for _, l := range lists {
		found, size := l.contains(ip)
		if found {
			if size > maskSize {
				maskSize = size
				ipList = l
			}
		}
	}
	if ipList == nil {
		ipList = lists["_default_"]
	}

	ipr.buckets = make(map[Dimension]*rateBucket)
	ipr.Expire = time.Now().Add(ipList.Expire.Duration)
	ipr.ip = ip
	ipr.shouldLimit = ipList.Limit
	ipr.list = ipList
	return &ipr
}

// getDimension takes in a service and a logEntry and spits out the Dimensions
// we want to bucket by for this IPList.
func (l *IPList) getDimension(log *logEntry, service *fastly.Service) *Dimension {
	switch l.DimensionType {
	case DimensionBackend:
		return &log.backend
	case DimensionFrontend:
		return &log.frontend
	case DimensionHost:
		return &log.host
	case DimensionService:
		return &Dimension{Type: l.DimensionType, Value: service.Name}
	}
	return &Dimension{}
}
<commit_msg>Actually utilize the useragent dimension.<commit_after>package main

import (
	"bufio"
	"fmt"
	"net"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/alienth/go-fastly"
)

type IPList struct {
	name string
	IPs  []*net.IP
	Nets []*ipNet

	Limit           bool
	Requests        int64
	ListFile        string
	DimensionType   DimensionType `toml:"Dimension"`
	DimensionShared bool          `toml:"MeasureRateByDimension"`
	DimensionValues []string

	Time          duration
	Expire        duration
	LimitDuration duration

	sharedBuckets *sharedBucketMap
}

type IPLists map[string]*IPList

type sharedBucketMap struct {
	sync.RWMutex
	m map[Dimension]*rateBucket
}

func (l *IPList) init(name string) {
	l.name = name
	if l.ListFile != "" {
		l.readListFile()
	}
	l.sharedBuckets = &sharedBucketMap{m: make(map[Dimension]*rateBucket)}
}

func (l *IPList) contains(checkIP *net.IP) (bool, int) {
	if l == nil {
		return false, 0
	}
	for _, ip := range l.IPs {
		if ip.Equal(*checkIP) {
			return true, 32
		}
	}
	for _, net := range l.Nets {
		if net.Contains(*checkIP) {
			size, _ := net.IPNet.Mask.Size()
			return true, size
		}
	}

	return false, 0
}

// readListFile reads a ListFile and parses the content
// into the IPLists' IPs and Nets fields.
func (l *IPList) readListFile() error {
	f, err := os.Open(l.ListFile)
	if err != nil {
		return err
	}

	scanner := bufio.NewScanner(f)

	for scanner.Scan() {
		line := scanner.Text()
		if strings.Index(strings.TrimSpace(line), "#") == 0 {
			continue
		}
		if strings.TrimSpace(line) == "" {
			continue
		}
		if strings.Contains(line, "/") {
			_, parsedNet, err := net.ParseCIDR(line)
			if err != nil {
				return fmt.Errorf("Unable to parse CIDR.\nLine:\n%s\nError:\n%s\n", line, err)
			}
			if parsedNet != nil {
				var n ipNet
				n.IP = parsedNet.IP
				n.Mask = parsedNet.Mask
				l.Nets = append(l.Nets, &n)
			}
		} else {
			ip := net.ParseIP(line)
			if ip != nil {
				l.IPs = append(l.IPs, &ip)
			} else {
				return fmt.Errorf("Unable to parse IP address in list: %s\n", line)
			}
		}
	}

	return nil
}

func (lists IPLists) getRate(ip *net.IP) *ipRate {
	var ipr ipRate
	var ipList *IPList
	var maskSize int
	// Iterate through all of the lists and find the list with the most
	// specific match.
	for _, l := range lists {
		found, size := l.contains(ip)
		if found {
			if size > maskSize {
				maskSize = size
				ipList = l
			}
		}
	}
	if ipList == nil {
		ipList = lists["_default_"]
	}

	ipr.buckets = make(map[Dimension]*rateBucket)
	ipr.Expire = time.Now().Add(ipList.Expire.Duration)
	ipr.ip = ip
	ipr.shouldLimit = ipList.Limit
	ipr.list = ipList
	return &ipr
}

// getDimension takes in a service and a logEntry and spits out the Dimensions
// we want to bucket by for this IPList.
func (l *IPList) getDimension(log *logEntry, service *fastly.Service) *Dimension {
	switch l.DimensionType {
	case DimensionBackend:
		return &log.backend
	case DimensionFrontend:
		return &log.frontend
	case DimensionHost:
		return &log.host
	case DimensionUseragent:
		return &log.useragent
	case DimensionService:
		return &Dimension{Type: l.DimensionType, Value: service.Name}
	}
	return &Dimension{}
}
<|endoftext|>


<commit_before>package plugins

type Error int

func (e Error) Error() string {
	if s, ok := errors[e]; ok {
		return s
	}

	if e < 100 {
		return "Unknown transmission error"
	} else if e < 200 {
		return "Unknown server error"
	} else if e < 300 {
		return "Unknown client error"
	} else {
		return "Unknown error"
	}
}

var errors = map[Error]string{
	Success:           "No error. Everything is fine.",
	NoSupportedFormat: "No supported formats in common",
	Unblocking:        "This is a unblocking recieve",
	NotImplemented:    "Operation is not implemented",
	NotDirectory:      "Path supplied is not a directory",
}

//Transmission errors
const (
	Success Error = iota
	NoSupportedFormat
)

//Server errors
const (
	Unblocking Error = 100 + iota
	NotImplemented
	NotDirectory
)
<commit_msg>Added DuplicatePlugin error to signal plugins that thay are duplicated.<commit_after>package plugins

type Error int

func (e Error) Error() string {
	if s, ok := errors[e]; ok {
		return s
	}

	if e < 100 {
		return "Unknown transmission error"
	} else if e < 200 {
		return "Unknown server error"
	} else if e < 300 {
		return "Unknown client error"
	} else {
		return "Unknown error"
	}
}

var errors = map[Error]string{
	Success:           "No error. Everything is fine.",
	NoSupportedFormat: "No supported formats in common",
	Unblocking:        "This is a unblocking recieve",
	NotImplemented:    "Operation is not implemented",
	NotDirectory:      "Path supplied is not a directory",
	DuplicatePlugin:   "Plugin is a duplicate. Already handled.",
}

//Transmission errors
const (
	Success Error = iota
	NoSupportedFormat
)

//Server errors
const (
	Unblocking Error = 100 + iota
	NotImplemented
	NotDirectory
	DuplicatePlugin
)
<|endoftext|>


<commit_before>package jobber

import (
	"fmt"
	stdlog "log"
	"os"
)

const logPrefixStr = "[jobber] "

type (
	Job interface {
		RespondTo() chan error
		Process() error
	}
	Logger interface {
		Print(v ...interface{})
		Printf(format string, v ...interface{})
		Println(v ...interface{})
	}
	logWrap struct{}
)

var (
	logPrefixSlice = []interface{}{logPrefixStr}
	log            *logWrap
	logger         Logger = stdlog.New(os.Stderr, "", stdlog.LstdFlags)
	debug          bool
)

func SetLogger(l Logger) {
	logger = l
}

func Debug() {
	debug = true
}

func DisableDebug() {
	debug = false
}

func (l *logWrap) Printf(format string, v ...interface{}) {
	logger.Printf(fmt.Sprintf("%s%s", logPrefixStr, format), v...)
}
func (l *logWrap) Print(v ...interface{}) {
	logger.Print(append(logPrefixSlice, v...)...)
}
func (l *logWrap) Println(v ...interface{}) {
	logger.Println(append(logPrefixSlice, v...)...)
}
<commit_msg>making job response chan write only as far as we're concerned<commit_after>package jobber

import (
	"fmt"
	stdlog "log"
	"os"
)

const logPrefixStr = "[jobber] "

type (
	Job interface {
		RespondTo() chan<- error
		Process() error
	}
	Logger interface {
		Print(v ...interface{})
		Printf(format string, v ...interface{})
		Println(v ...interface{})
	}
	logWrap struct{}
)

var (
	logPrefixSlice = []interface{}{logPrefixStr}
	log            *logWrap
	logger         Logger = stdlog.New(os.Stderr, "", stdlog.LstdFlags)
	debug          bool
)

func SetLogger(l Logger) {
	logger = l
}

func Debug() {
	debug = true
}

func DisableDebug() {
	debug = false
}

func (l *logWrap) Printf(format string, v ...interface{}) {
	logger.Printf(fmt.Sprintf("%s%s", logPrefixStr, format), v...)
}
func (l *logWrap) Print(v ...interface{}) {
	logger.Print(append(logPrefixSlice, v...)...)
}
func (l *logWrap) Println(v ...interface{}) {
	logger.Println(append(logPrefixSlice, v...)...)
}
<|endoftext|>


<commit_before>// Copyright 2009 Thomas Jager <mail@jager.no> All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Driver for ADAM-4000 series I/O Modules from Advantech

package main

import (
	"bufio"
	"net"
        "fmt"
)

func main() {
	conn, err := net.Dial("tcp", "192.168.0.60:5301")
	if err != nil {
		fmt.Printf("%s", err)
		return
	}
	adam := NewADAM4000(0, bufio.NewReader(conn), bufio.NewWriter(conn))
	err = adam.GetConfig()
	if err != nil {
		fmt.Printf("%s", err)
	}
	_, err = adam.GetVersion()
	if err != nil {
		fmt.Printf("%s", err)
	}
	_, err = adam.GetName()
	if err != nil {
		fmt.Printf("%s", err)
	}
	err = adam.SetChannelRange(4, cKTc)
	if err != nil {
		fmt.Printf("%s", err)
	}
	rangec, err := adam.GetChannelRange(4)
	fmt.Printf("%s\n", rangec)
	if err != nil {
		fmt.Printf("%s", err)
	}
	_, err = adam.GetAllValue()
	if err != nil {
		fmt.Printf("%s", err)
	}
	val, err := adam.GetChannelValue(3)
	if err != nil {
		fmt.Printf("%s", err)
	}
	fmt.Printf("%X: %s, %s, %s, %s %v %f\n", adam.Address, adam.BaudRate, adam.InputRange, adam.Version, adam.Name, adam.Value, val)
	conn.Close()
}
<commit_msg>Extended testing<commit_after>// Copyright 2009 Thomas Jager <mail@jager.no> All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Driver for ADAM-4000 series I/O Modules from Advantech

package main

import (
	"bufio"
	"fmt"
	"net"
)

func main() {
	conn, err := net.Dial("tcp", "192.168.0.60:5301")
	if err != nil {
		fmt.Printf("%s", err)
		return
	}
	adam := NewADAM4000(0, bufio.NewReader(conn), bufio.NewWriter(conn))
	err = adam.GetConfig()
	if err != nil {
		fmt.Printf("%s", err)
	}
	_, err = adam.GetVersion()
	if err != nil {
		fmt.Printf("%s", err)
	}
	_, err = adam.GetName()
	if err != nil {
		fmt.Printf("%s", err)
	}
	err = adam.SetChannelRange(4, cKTc)
	if err != nil {
		fmt.Printf("%s", err)
	}
	rangec, err := adam.GetChannelRange(4)
	fmt.Printf("%s\n", rangec)
	if err != nil {
		fmt.Printf("%s", err)
	}
	_, err = adam.GetAllValue()
	if err != nil {
		fmt.Printf("%s", err)
	}
	val, err := adam.GetChannelValue(3)
	if err != nil {
		fmt.Printf("%s", err)
	}
	fmt.Printf("%s\n", adam)
	fmt.Printf("%X: %s, %s, %s, %s %v %f\n", adam.Address, adam.BaudRate, adam.InputRange, adam.Version, adam.Name, adam.Value, val)
	adam.Address = 7
	adam.DataFormat = EngineeringUnits
	adam.SetConfig()
	conn.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/danryan/hal"
)

var eventHandler = hal.Hear(`.events`, func(res *hal.Response) error {
	events, err := getCalendarEvents()
	if err != nil {
		return res.Send(err.Error())
	}

	return res.Send(events)
})

var (
	baseURL     = "https://www.googleapis.com/calendar/v3/calendars"
	accessToken AccessToken
)

type AccessToken struct {
	Token string
	Valid time.Time
}

func (a *AccessToken) getExpireTime(o int) {
	n := time.Now()
	d := time.Duration(o) * time.Second

	a.Valid = n.Add(d)
}

func (a *AccessToken) expiredToken() bool {
	n := time.Now()
	if a.Valid.After(n) {
		return true
	}

	return false
}

type Event struct {
	// Kind is the type of collection ("calendar#events")
	Kind string `json:"kind"`
	// Etag is the etag of the collection
	Etag string `json:"etag"`
	// Summary is the name of the calendar
	Summary string `json:"summary"`
	// Description is the description of the calendar
	Description string `json:"descritpion"`
	// Updated is the last modified time for the calendar
	Updated string `json:"updated"`
	// TimeZone is the timezone for the calendar
	TimeZone string `json:"timeZone"`
	// AccessRole is the role of the current user.
	// possible values include: "none", "freeBuzyReader", "reader", "writer", "owner"
	AccessRole string `json:"accessRole"`
	// NextPageToken is used to access the next page of this result. Omitted if no
	// further results are available, in which case nextSyncToken is provided.
	NextPageToken string `json:"nextPageToken,omitempty"`
	// Items is the list of events on the calendar.
	Items []EventItem `json:"items"`
	// NextSyncToken is used at a later point in time to retrieve only the entries
	// that have changed since this result was returned. Omitted if further results
	// are available, in which case nextPageToken is provided.
	NextSyncToken string `json:"nextSyncToken"`
}

type EventItem struct {
	// Kind is the type of collection ("calendar#event")
	Kind string `json:"kind"`
	// Etag string `json:"etag"`
	Etag string `json:"etag"`
	// ID is the ID of the event
	ID string `json:"id"`
	// Status is the status of the event
	Status string `json:"status"`
	// HTMLLink is the link to the event on the calendar
	HTMLLink string `json:"htmlLink"`
	// Created is the datetime stamp that the event was created
	Created string `json:"created"`
	// Updated is the datetame stamp that the event was last modified
	Updated string `json:"updated"`
	// Summary is the name of the event
	Summary string `json:"summary"`
	// Description is the body of the event
	Description string `json:"description"`
	// Location is the address for the event
	Location string `json:"location"`
	// Creator is the user that created the event
	Creator EventCreator `json:"creator"`
	// Organizer is the organizer information
	organizer EventOrganizer `json:"organizer"`
	// Start is the start date and time
	Start EventDateTime `json:"start"`
	// End is the end date and time
	End EventDateTime `json:"end"`
	// ICalUID is the iCal ID for the event
	ICalUID string `json:"uCalUID"`
	// Sequence denotes if this is a repeating event
	Sequence int `json:"sequence"`
	// Reminders are the event reminders
	Reminders EventReminder `json:"reminders"`
}

type EventCreator struct {
	// Email address for the event creator
	Email string `json:"email"`
	// DisplayName is the real name of the creator
	DisplayName string `json:"displayName"`
}

type EventOrganizer struct {
	// Email address for the event organizer
	Email string `json:"email"`
	// DisplayName is the name of the event organizer
	DisplayName string `json:"displayName"`
	// Self denotes if the organizer is the current user
	Self bool `json:self"`
}

type EventDateTime struct {
	// DateTime is the start/end datetime in  RFC 3339 format
	DateTime string `json:"dateTime"`
	// TimeZone is the Timezone for the event
	TimeZone string `json:"timeZone,omitempty"`
}

type EventReminder struct {
	// UseDefault denotes if Google Calendar default reminders are used
	UseDefault bool `json:"useDefault"`
}

func getCalendarEvents() (string, error) {
	var err error

	if accessToken.Token == "" ||
		accessToken.expiredToken() {
		accessToken, err = getOauth2Token()
		if err != nil {
			return "", err
		}
	}

	URL := fmt.Sprintf("%s/4qc3thgj9ocunpfist563utr6g@group.calendar.google.com/events?access_token=%s&singleEvents=true&orderBy=startTime&timeMin=%s&maxResults=7",
		baseURL, url.QueryEscape(accessToken.Token),
		url.QueryEscape(time.Now().Format("2006-01-02T15:04:05Z")))
	resp, err := http.Get(URL)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	var Events Event

	err = json.Unmarshal(body, &Events)
	if err != nil {
		return "", err
	}

	eventList := getEventList(Events)

	return eventList, nil
}

func getOauth2Token() (AccessToken, error) {
	type Responce struct {
		Token   string `json:"access_token"`
		Expires int    `json:"expires_in"`
	}

	clientID := os.Getenv("CHADEV_ID")
	clientSecret := os.Getenv("CHADEV_SECRET")
	refreshToken := os.Getenv("CHADEV_TOKEN")

	var r Responce

	body := fmt.Sprintf("client_id=%s&client_secret=%s&refresh_token=%s&grant_type=refresh_token",
		clientID, clientSecret, refreshToken)
	b := strings.NewReader(body)

	resp, err := http.Post("https://accounts.google.com/o/oauth2/token", "application/x-www-form-urlencoded", b)
	if err != nil {
		return accessToken, err
	}
	defer resp.Body.Close()

	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return accessToken, err
	}

	err = json.Unmarshal(respBody, &r)
	if err != nil {
		return accessToken, err
	}

	accessToken.Token = r.Token
	accessToken.getExpireTime(r.Expires)

	return accessToken, nil
}

func getEventList(events Event) string {
	var output string

	output = "Next 7 events: "
	for key, event := range events.Items {
		if key == 0 {
			output += event.Summary
		} else {
			output += ", " + event.Summary
		}
		output += " (" + formatDate(event.Start.DateTime) + ")"
	}

	return output
}

func formatDate(s string) string {
	date, err := time.Parse(time.RFC3339, s)
	if err != nil {
		return ""
	}

	return date.Format("01/02")
}
<commit_msg>Linted events.go<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/danryan/hal"
)

var eventHandler = hal.Hear(`.events`, func(res *hal.Response) error {
	events, err := getCalendarEvents()
	if err != nil {
		return res.Send(err.Error())
	}

	return res.Send(events)
})

var (
	baseURL     = "https://www.googleapis.com/calendar/v3/calendars"
	accessToken AccessToken
)

// AccessToken contains the current oauth2 token and its expiry time.
type AccessToken struct {
	Token string
	Valid time.Time
}

func (a *AccessToken) getExpireTime(o int) {
	n := time.Now()
	d := time.Duration(o) * time.Second

	a.Valid = n.Add(d)
}

func (a *AccessToken) expiredToken() bool {
	n := time.Now()
	if a.Valid.After(n) {
		return true
	}

	return false
}

// Event is the top-level of the JSON return from the calendar API.
type Event struct {
	// Kind is the type of collection ("calendar#events")
	Kind string `json:"kind"`
	// Etag is the etag of the collection
	Etag string `json:"etag"`
	// Summary is the name of the calendar
	Summary string `json:"summary"`
	// Description is the description of the calendar
	Description string `json:"descritpion"`
	// Updated is the last modified time for the calendar
	Updated string `json:"updated"`
	// TimeZone is the timezone for the calendar
	TimeZone string `json:"timeZone"`
	// AccessRole is the role of the current user.
	// possible values include: "none", "freeBuzyReader", "reader", "writer", "owner"
	AccessRole string `json:"accessRole"`
	// NextPageToken is used to access the next page of this result. Omitted if no
	// further results are available, in which case nextSyncToken is provided.
	NextPageToken string `json:"nextPageToken,omitempty"`
	// Items is the list of events on the calendar.
	Items []EventItem `json:"items"`
	// NextSyncToken is used at a later point in time to retrieve only the entries
	// that have changed since this result was returned. Omitted if further results
	// are available, in which case nextPageToken is provided.
	NextSyncToken string `json:"nextSyncToken"`
}

// EventItem contains items with-in the "items" list of the JSON.
type EventItem struct {
	// Kind is the type of collection ("calendar#event")
	Kind string `json:"kind"`
	// Etag string `json:"etag"`
	Etag string `json:"etag"`
	// ID is the ID of the event
	ID string `json:"id"`
	// Status is the status of the event
	Status string `json:"status"`
	// HTMLLink is the link to the event on the calendar
	HTMLLink string `json:"htmlLink"`
	// Created is the datetime stamp that the event was created
	Created string `json:"created"`
	// Updated is the datetame stamp that the event was last modified
	Updated string `json:"updated"`
	// Summary is the name of the event
	Summary string `json:"summary"`
	// Description is the body of the event
	Description string `json:"description"`
	// Location is the address for the event
	Location string `json:"location"`
	// Creator is the user that created the event
	Creator EventCreator `json:"creator"`
	// Organizer is the organizer information
	organizer EventOrganizer `json:"organizer"`
	// Start is the start date and time
	Start EventDateTime `json:"start"`
	// End is the end date and time
	End EventDateTime `json:"end"`
	// ICalUID is the iCal ID for the event
	ICalUID string `json:"uCalUID"`
	// Sequence denotes if this is a repeating event
	Sequence int `json:"sequence"`
	// Reminders are the event reminders
	Reminders EventReminder `json:"reminders"`
}

// EventCreator contains the fields for the creator object.
type EventCreator struct {
	// Email address for the event creator
	Email string `json:"email"`
	// DisplayName is the real name of the creator
	DisplayName string `json:"displayName"`
}

// EventOrganizer contains the fields for the organizer object.
type EventOrganizer struct {
	// Email address for the event organizer
	Email string `json:"email"`
	// DisplayName is the name of the event organizer
	DisplayName string `json:"displayName"`
	// Self denotes if the organizer is the current user
	Self bool `json:self"`
}

// EventDateTime contains the fields for the "start" and "end" objects.
type EventDateTime struct {
	// DateTime is the start/end datetime in  RFC 3339 format
	DateTime string `json:"dateTime"`
	// TimeZone is the Timezone for the event
	TimeZone string `json:"timeZone,omitempty"`
}

//EventReminder contains the fields for the reminders object.
type EventReminder struct {
	// UseDefault denotes if Google Calendar default reminders are used
	UseDefault bool `json:"useDefault"`
}

func getCalendarEvents() (string, error) {
	var err error

	if accessToken.Token == "" ||
		accessToken.expiredToken() {
		accessToken, err = getOauth2Token()
		if err != nil {
			return "", err
		}
	}

	URL := fmt.Sprintf("%s/4qc3thgj9ocunpfist563utr6g@group.calendar.google.com/events?access_token=%s&singleEvents=true&orderBy=startTime&timeMin=%s&maxResults=7",
		baseURL, url.QueryEscape(accessToken.Token),
		url.QueryEscape(time.Now().Format("2006-01-02T15:04:05Z")))
	resp, err := http.Get(URL)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	var Events Event

	err = json.Unmarshal(body, &Events)
	if err != nil {
		return "", err
	}

	eventList := getEventList(Events)

	return eventList, nil
}

func getOauth2Token() (AccessToken, error) {
	type Responce struct {
		Token   string `json:"access_token"`
		Expires int    `json:"expires_in"`
	}

	clientID := os.Getenv("CHADEV_ID")
	clientSecret := os.Getenv("CHADEV_SECRET")
	refreshToken := os.Getenv("CHADEV_TOKEN")

	var r Responce

	body := fmt.Sprintf("client_id=%s&client_secret=%s&refresh_token=%s&grant_type=refresh_token",
		clientID, clientSecret, refreshToken)
	b := strings.NewReader(body)

	resp, err := http.Post("https://accounts.google.com/o/oauth2/token", "application/x-www-form-urlencoded", b)
	if err != nil {
		return accessToken, err
	}
	defer resp.Body.Close()

	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return accessToken, err
	}

	err = json.Unmarshal(respBody, &r)
	if err != nil {
		return accessToken, err
	}

	accessToken.Token = r.Token
	accessToken.getExpireTime(r.Expires)

	return accessToken, nil
}

func getEventList(events Event) string {
	var output string

	output = "Next 7 events: "
	for key, event := range events.Items {
		if key == 0 {
			output += event.Summary
		} else {
			output += ", " + event.Summary
		}
		output += " (" + formatDate(event.Start.DateTime) + ")"
	}

	return output
}

func formatDate(s string) string {
	date, err := time.Parse(time.RFC3339, s)
	if err != nil {
		return ""
	}

	return date.Format("01/02")
}
<|endoftext|>


<commit_before>// Package heroku is a client interface to the Heroku API.
package heroku

import (
	"bytes"
	"code.google.com/p/go-uuid/uuid"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/http/httputil"
	"os"
	"runtime"
	"strings"
)

const (
	Version          = "0.1"
	DefaultUserAgent = "heroku-go " + Version + " " + runtime.GOOS + " " + runtime.GOARCH
)

// A Client is a Heroku API client. Its zero value is a usable client that uses
// default settings for the Heroku API. The Client has an internal HTTP client
// (HTTP) which defaults to http.DefaultClient.
//
// As with all http.Clients, this Client's Transport has internal state (cached
// HTTP connections), so Clients should be reused instead of created as needed.
// Clients are safe for use by multiple goroutines.
type Client struct {
	// HTTP is the Client's internal http.Client, handling HTTP requests to the
	// Heroku API.
	HTTP *http.Client

	// The URL of the Heroku API to communicate with. Defaults to
	// "https://api.heroku.com".
	URL string

	// Username is the HTTP basic auth username for API calls made by this Client.
	Username string

	// Password is the HTTP basic auth password for API calls made by this Client.
	Password string

	// UserAgent to be provided in API requests. Set to DefaultUserAgent if not
	// specified.
	UserAgent string
}

func (c *Client) Get(v interface{}, path string) error {
	return c.APIReq(v, "GET", path, nil)
}

func (c *Client) Patch(v interface{}, path string, body interface{}) error {
	return c.APIReq(v, "PATCH", path, body)
}

func (c *Client) Post(v interface{}, path string, body interface{}) error {
	return c.APIReq(v, "POST", path, body)
}

func (c *Client) Put(v interface{}, path string, body interface{}) error {
	return c.APIReq(v, "PUT", path, body)
}

func (c *Client) Delete(path string) error {
	return c.APIReq(nil, "DELETE", path, nil)
}

// Generates an HTTP request for the Heroku API, but does not
// perform the request. The request's Accept header field will be
// set to:
//
//   Accept: application/vnd.heroku+json; version=3
//
// The Request-Id header will be set to a random UUID. The User-Agent header
// will be set to the Client's UserAgent, or DefaultUserAgent if UserAgent is
// not set.
//
// The type of body determines how to encode the request:
//
//   nil         no body
//   io.Reader   body is sent verbatim
//   else        body is encoded as application/json
func (c *Client) NewRequest(method, path string, body interface{}) (*http.Request, error) {
	var ctype string
	var rbody io.Reader

	switch t := body.(type) {
	case nil:
	case string:
		rbody = bytes.NewBufferString(t)
	case io.Reader:
		rbody = t
	default:
		j, err := json.Marshal(body)
		if err != nil {
			log.Fatal(err)
		}
		rbody = bytes.NewReader(j)
		ctype = "application/json"
	}
	apiURL := strings.TrimRight(c.URL, "/")
	if apiURL == "" {
		apiURL = "https://api.heroku.com"
	}
	req, err := http.NewRequest(method, apiURL+path, rbody)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Accept", "application/vnd.heroku+json; version=3")
	req.Header.Set("Request-Id", uuid.New())
	useragent := c.UserAgent
	if useragent == "" {
		useragent = DefaultUserAgent
	}
	req.Header.Set("User-Agent", useragent)
	if ctype != "" {
		req.Header.Set("Content-Type", ctype)
	}
	req.SetBasicAuth(c.Username, c.Password)
	for _, h := range strings.Split(os.Getenv("HKHEADER"), "\n") {
		if i := strings.Index(h, ":"); i >= 0 {
			req.Header.Set(
				strings.TrimSpace(h[:i]),
				strings.TrimSpace(h[i+1:]),
			)
		}
	}
	return req, nil
}

// Sends a Heroku API request and decodes the response into v. As
// described in NewRequest(), the type of body determines how to
// encode the request body. As described in DoReq(), the type of
// v determines how to handle the response body.
func (c *Client) APIReq(v interface{}, meth, path string, body interface{}) error {
	req, err := c.NewRequest(meth, path, body)
	if err != nil {
		return err
	}
	return c.DoReq(req, v)
}

// Submits an HTTP request, checks its response, and deserializes
// the response into v. The type of v determines how to handle
// the response body:
//
//   nil        body is discarded
//   io.Writer  body is copied directly into v
//   else       body is decoded into v as json
//
func (c *Client) DoReq(req *http.Request, v interface{}) error {
	debug := os.Getenv("HKDEBUG") != ""
	if debug {
		dump, err := httputil.DumpRequestOut(req, true)
		if err != nil {
			log.Println(err)
		} else {
			os.Stderr.Write(dump)
			os.Stderr.Write([]byte{'\n', '\n'})
		}
	}

	httpClient := c.HTTP
	if httpClient == nil {
		httpClient = http.DefaultClient
	}

	res, err := httpClient.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if debug {
		dump, err := httputil.DumpResponse(res, true)
		if err != nil {
			log.Println(err)
		} else {
			os.Stderr.Write(dump)
			os.Stderr.Write([]byte{'\n'})
		}
	}
	if err = checkResp(res); err != nil {
		return err
	}
	switch t := v.(type) {
	case nil:
	case io.Writer:
		_, err = io.Copy(t, res.Body)
	default:
		err = json.NewDecoder(res.Body).Decode(v)
	}
	return err
}

func checkResp(res *http.Response) error {
	if res.StatusCode == 401 {
		return errors.New("Unauthorized")
	}
	if res.StatusCode == 403 {
		return errors.New("Unauthorized")
	}
	if res.StatusCode/100 != 2 { // 200, 201, 202, etc
		return errors.New("Unexpected error: " + res.Status)
	}
	if msg := res.Header.Get("X-Heroku-Warning"); msg != "" {
		fmt.Fprintln(os.Stderr, strings.TrimSpace(msg))
	}
	return nil
}

type ListRange struct {
	Field      string
	Max        int
	Descending bool
	FirstId    string
	LastId     string
}

func (r *ListRange) SetHeader(req *http.Request) {
	var hdrval string
	if r.Field != "" {
		hdrval += r.Field + " "
	}
	hdrval += r.FirstId + ".." + r.LastId
	if r.Max != 0 {
		hdrval += fmt.Sprintf("; max=%d", r.Max)
		if r.Descending {
			hdrval += ", "
		}
	}

	if r.Descending {
		hdrval += ", order=desc"
	}

	req.Header.Set("Range", hdrval)
	return
}
<commit_msg>r -> lr<commit_after>// Package heroku is a client interface to the Heroku API.
package heroku

import (
	"bytes"
	"code.google.com/p/go-uuid/uuid"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/http/httputil"
	"os"
	"runtime"
	"strings"
)

const (
	Version          = "0.1"
	DefaultUserAgent = "heroku-go " + Version + " " + runtime.GOOS + " " + runtime.GOARCH
)

// A Client is a Heroku API client. Its zero value is a usable client that uses
// default settings for the Heroku API. The Client has an internal HTTP client
// (HTTP) which defaults to http.DefaultClient.
//
// As with all http.Clients, this Client's Transport has internal state (cached
// HTTP connections), so Clients should be reused instead of created as needed.
// Clients are safe for use by multiple goroutines.
type Client struct {
	// HTTP is the Client's internal http.Client, handling HTTP requests to the
	// Heroku API.
	HTTP *http.Client

	// The URL of the Heroku API to communicate with. Defaults to
	// "https://api.heroku.com".
	URL string

	// Username is the HTTP basic auth username for API calls made by this Client.
	Username string

	// Password is the HTTP basic auth password for API calls made by this Client.
	Password string

	// UserAgent to be provided in API requests. Set to DefaultUserAgent if not
	// specified.
	UserAgent string
}

func (c *Client) Get(v interface{}, path string) error {
	return c.APIReq(v, "GET", path, nil)
}

func (c *Client) Patch(v interface{}, path string, body interface{}) error {
	return c.APIReq(v, "PATCH", path, body)
}

func (c *Client) Post(v interface{}, path string, body interface{}) error {
	return c.APIReq(v, "POST", path, body)
}

func (c *Client) Put(v interface{}, path string, body interface{}) error {
	return c.APIReq(v, "PUT", path, body)
}

func (c *Client) Delete(path string) error {
	return c.APIReq(nil, "DELETE", path, nil)
}

// Generates an HTTP request for the Heroku API, but does not
// perform the request. The request's Accept header field will be
// set to:
//
//   Accept: application/vnd.heroku+json; version=3
//
// The Request-Id header will be set to a random UUID. The User-Agent header
// will be set to the Client's UserAgent, or DefaultUserAgent if UserAgent is
// not set.
//
// The type of body determines how to encode the request:
//
//   nil         no body
//   io.Reader   body is sent verbatim
//   else        body is encoded as application/json
func (c *Client) NewRequest(method, path string, body interface{}) (*http.Request, error) {
	var ctype string
	var rbody io.Reader

	switch t := body.(type) {
	case nil:
	case string:
		rbody = bytes.NewBufferString(t)
	case io.Reader:
		rbody = t
	default:
		j, err := json.Marshal(body)
		if err != nil {
			log.Fatal(err)
		}
		rbody = bytes.NewReader(j)
		ctype = "application/json"
	}
	apiURL := strings.TrimRight(c.URL, "/")
	if apiURL == "" {
		apiURL = "https://api.heroku.com"
	}
	req, err := http.NewRequest(method, apiURL+path, rbody)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Accept", "application/vnd.heroku+json; version=3")
	req.Header.Set("Request-Id", uuid.New())
	useragent := c.UserAgent
	if useragent == "" {
		useragent = DefaultUserAgent
	}
	req.Header.Set("User-Agent", useragent)
	if ctype != "" {
		req.Header.Set("Content-Type", ctype)
	}
	req.SetBasicAuth(c.Username, c.Password)
	for _, h := range strings.Split(os.Getenv("HKHEADER"), "\n") {
		if i := strings.Index(h, ":"); i >= 0 {
			req.Header.Set(
				strings.TrimSpace(h[:i]),
				strings.TrimSpace(h[i+1:]),
			)
		}
	}
	return req, nil
}

// Sends a Heroku API request and decodes the response into v. As
// described in NewRequest(), the type of body determines how to
// encode the request body. As described in DoReq(), the type of
// v determines how to handle the response body.
func (c *Client) APIReq(v interface{}, meth, path string, body interface{}) error {
	req, err := c.NewRequest(meth, path, body)
	if err != nil {
		return err
	}
	return c.DoReq(req, v)
}

// Submits an HTTP request, checks its response, and deserializes
// the response into v. The type of v determines how to handle
// the response body:
//
//   nil        body is discarded
//   io.Writer  body is copied directly into v
//   else       body is decoded into v as json
//
func (c *Client) DoReq(req *http.Request, v interface{}) error {
	debug := os.Getenv("HKDEBUG") != ""
	if debug {
		dump, err := httputil.DumpRequestOut(req, true)
		if err != nil {
			log.Println(err)
		} else {
			os.Stderr.Write(dump)
			os.Stderr.Write([]byte{'\n', '\n'})
		}
	}

	httpClient := c.HTTP
	if httpClient == nil {
		httpClient = http.DefaultClient
	}

	res, err := httpClient.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if debug {
		dump, err := httputil.DumpResponse(res, true)
		if err != nil {
			log.Println(err)
		} else {
			os.Stderr.Write(dump)
			os.Stderr.Write([]byte{'\n'})
		}
	}
	if err = checkResp(res); err != nil {
		return err
	}
	switch t := v.(type) {
	case nil:
	case io.Writer:
		_, err = io.Copy(t, res.Body)
	default:
		err = json.NewDecoder(res.Body).Decode(v)
	}
	return err
}

func checkResp(res *http.Response) error {
	if res.StatusCode == 401 {
		return errors.New("Unauthorized")
	}
	if res.StatusCode == 403 {
		return errors.New("Unauthorized")
	}
	if res.StatusCode/100 != 2 { // 200, 201, 202, etc
		return errors.New("Unexpected error: " + res.Status)
	}
	if msg := res.Header.Get("X-Heroku-Warning"); msg != "" {
		fmt.Fprintln(os.Stderr, strings.TrimSpace(msg))
	}
	return nil
}

type ListRange struct {
	Field      string
	Max        int
	Descending bool
	FirstId    string
	LastId     string
}

func (lr *ListRange) SetHeader(req *http.Request) {
	var hdrval string
	if lr.Field != "" {
		hdrval += lr.Field + " "
	}
	hdrval += lr.FirstId + ".." + lr.LastId
	if lr.Max != 0 {
		hdrval += fmt.Sprintf("; max=%d", lr.Max)
		if lr.Descending {
			hdrval += ", "
		}
	}

	if lr.Descending {
		hdrval += ", order=desc"
	}

	req.Header.Set("Range", hdrval)
	return
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/resourcegroups"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/helper/validation"
)

func resourceAwsResourceGroupsGroup() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsResourceGroupsGroupCreate,
		Read:   resourceAwsResourceGroupsGroupRead,
		Update: resourceAwsResourceGroupsGroupUpdate,
		Delete: resourceAwsResourceGroupsGroupDelete,

		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"description": {
				Type:     schema.TypeString,
				Optional: true,
			},

			"resource_query": {
				Type:     schema.TypeList,
				Required: true,
				MinItems: 1,
				MaxItems: 1,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"query": {
							Type:     schema.TypeString,
							Required: true,
						},

						"type": {
							Type:     schema.TypeString,
							Optional: true,
							Default:  resourcegroups.QueryTypeTagFilters10,
							ValidateFunc: validation.StringInSlice([]string{
								resourcegroups.QueryTypeTagFilters10,
							}, false),
						},
					},
				},
			},

			"arn": {
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func extractResourceGroupResourceQuery(resourceQueryList []interface{}) *resourcegroups.ResourceQuery {
	resourceQuery := resourceQueryList[0].(map[string]interface{})

	return &resourcegroups.ResourceQuery{
		Query: aws.String(resourceQuery["query"].(string)),
		Type:  aws.String(resourceQuery["type"].(string)),
	}
}

func extractResourceGroupTags(m map[string]interface{}) map[string]*string {
	result := make(map[string]*string)
	for k, v := range m {
		result[k] = aws.String(v.(string))
	}

	return result
}

func resourceAwsResourceGroupsGroupCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).resourcegroupsconn

	input := resourcegroups.CreateGroupInput{
		Description:   aws.String(d.Get("description").(string)),
		Name:          aws.String(d.Get("name").(string)),
		ResourceQuery: extractResourceGroupResourceQuery(d.Get("resource_query").([]interface{})),
	}

	res, err := conn.CreateGroup(&input)
	if err != nil {
		return fmt.Errorf("error creating resource group: %s", err)
	}

	d.SetId(aws.StringValue(res.Group.Name))

	return resourceAwsResourceGroupsGroupRead(d, meta)
}

func resourceAwsResourceGroupsGroupRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).resourcegroupsconn

	g, err := conn.GetGroup(&resourcegroups.GetGroupInput{
		GroupName: aws.String(d.Id()),
	})

	if err != nil {
		if isAWSErr(err, resourcegroups.ErrCodeNotFoundException, "") {
			log.Printf("[WARN] Resource Groups Group (%s) not found, removing from state", d.Id())
			d.SetId("")
			return nil
		}

		return fmt.Errorf("error reading resource group (%s): %s", d.Id(), err)
	}

	d.Set("name", aws.StringValue(g.Group.Name))
	d.Set("description", aws.StringValue(g.Group.Description))
	d.Set("arn", aws.StringValue(g.Group.GroupArn))

	q, err := conn.GetGroupQuery(&resourcegroups.GetGroupQueryInput{
		GroupName: aws.String(d.Id()),
	})

	if err != nil {
		return fmt.Errorf("error reading resource query for resource group (%s): %s", d.Id(), err)
	}

	resultQuery := map[string]interface{}{}
	resultQuery["query"] = aws.StringValue(q.GroupQuery.ResourceQuery.Query)
	resultQuery["type"] = aws.StringValue(q.GroupQuery.ResourceQuery.Type)
	if err := d.Set("resource_query", []map[string]interface{}{resultQuery}); err != nil {
		return fmt.Errorf("error setting resource_query: %s", err)
	}

	return nil
}

func resourceAwsResourceGroupsGroupUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).resourcegroupsconn

	if d.HasChange("description") {
		input := resourcegroups.UpdateGroupInput{
			GroupName:   aws.String(d.Id()),
			Description: aws.String(d.Get("description").(string)),
		}

		_, err := conn.UpdateGroup(&input)
		if err != nil {
			return fmt.Errorf("error updating resource group (%s): %s", d.Id(), err)
		}
	}

	if d.HasChange("resource_query") {
		input := resourcegroups.UpdateGroupQueryInput{
			GroupName:     aws.String(d.Id()),
			ResourceQuery: extractResourceGroupResourceQuery(d.Get("resource_query").([]interface{})),
		}

		_, err := conn.UpdateGroupQuery(&input)
		if err != nil {
			return fmt.Errorf("error updating resource query for resource group (%s): %s", d.Id(), err)
		}
	}

	return resourceAwsResourceGroupsGroupRead(d, meta)
}

func resourceAwsResourceGroupsGroupDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).resourcegroupsconn

	input := resourcegroups.DeleteGroupInput{
		GroupName: aws.String(d.Id()),
	}

	_, err := conn.DeleteGroup(&input)
	if err != nil {
		return fmt.Errorf("error deleting resource group (%s): %s", d.Id(), err)
	}

	return nil
}
<commit_msg>resource/aws_resourcegroups_group: Remove extraneous extractResourceGroupTags() function for now<commit_after>package aws

import (
	"fmt"
	"log"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/resourcegroups"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/helper/validation"
)

func resourceAwsResourceGroupsGroup() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsResourceGroupsGroupCreate,
		Read:   resourceAwsResourceGroupsGroupRead,
		Update: resourceAwsResourceGroupsGroupUpdate,
		Delete: resourceAwsResourceGroupsGroupDelete,

		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"description": {
				Type:     schema.TypeString,
				Optional: true,
			},

			"resource_query": {
				Type:     schema.TypeList,
				Required: true,
				MinItems: 1,
				MaxItems: 1,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"query": {
							Type:     schema.TypeString,
							Required: true,
						},

						"type": {
							Type:     schema.TypeString,
							Optional: true,
							Default:  resourcegroups.QueryTypeTagFilters10,
							ValidateFunc: validation.StringInSlice([]string{
								resourcegroups.QueryTypeTagFilters10,
							}, false),
						},
					},
				},
			},

			"arn": {
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func extractResourceGroupResourceQuery(resourceQueryList []interface{}) *resourcegroups.ResourceQuery {
	resourceQuery := resourceQueryList[0].(map[string]interface{})

	return &resourcegroups.ResourceQuery{
		Query: aws.String(resourceQuery["query"].(string)),
		Type:  aws.String(resourceQuery["type"].(string)),
	}
}

func resourceAwsResourceGroupsGroupCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).resourcegroupsconn

	input := resourcegroups.CreateGroupInput{
		Description:   aws.String(d.Get("description").(string)),
		Name:          aws.String(d.Get("name").(string)),
		ResourceQuery: extractResourceGroupResourceQuery(d.Get("resource_query").([]interface{})),
	}

	res, err := conn.CreateGroup(&input)
	if err != nil {
		return fmt.Errorf("error creating resource group: %s", err)
	}

	d.SetId(aws.StringValue(res.Group.Name))

	return resourceAwsResourceGroupsGroupRead(d, meta)
}

func resourceAwsResourceGroupsGroupRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).resourcegroupsconn

	g, err := conn.GetGroup(&resourcegroups.GetGroupInput{
		GroupName: aws.String(d.Id()),
	})

	if err != nil {
		if isAWSErr(err, resourcegroups.ErrCodeNotFoundException, "") {
			log.Printf("[WARN] Resource Groups Group (%s) not found, removing from state", d.Id())
			d.SetId("")
			return nil
		}

		return fmt.Errorf("error reading resource group (%s): %s", d.Id(), err)
	}

	d.Set("name", aws.StringValue(g.Group.Name))
	d.Set("description", aws.StringValue(g.Group.Description))
	d.Set("arn", aws.StringValue(g.Group.GroupArn))

	q, err := conn.GetGroupQuery(&resourcegroups.GetGroupQueryInput{
		GroupName: aws.String(d.Id()),
	})

	if err != nil {
		return fmt.Errorf("error reading resource query for resource group (%s): %s", d.Id(), err)
	}

	resultQuery := map[string]interface{}{}
	resultQuery["query"] = aws.StringValue(q.GroupQuery.ResourceQuery.Query)
	resultQuery["type"] = aws.StringValue(q.GroupQuery.ResourceQuery.Type)
	if err := d.Set("resource_query", []map[string]interface{}{resultQuery}); err != nil {
		return fmt.Errorf("error setting resource_query: %s", err)
	}

	return nil
}

func resourceAwsResourceGroupsGroupUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).resourcegroupsconn

	if d.HasChange("description") {
		input := resourcegroups.UpdateGroupInput{
			GroupName:   aws.String(d.Id()),
			Description: aws.String(d.Get("description").(string)),
		}

		_, err := conn.UpdateGroup(&input)
		if err != nil {
			return fmt.Errorf("error updating resource group (%s): %s", d.Id(), err)
		}
	}

	if d.HasChange("resource_query") {
		input := resourcegroups.UpdateGroupQueryInput{
			GroupName:     aws.String(d.Id()),
			ResourceQuery: extractResourceGroupResourceQuery(d.Get("resource_query").([]interface{})),
		}

		_, err := conn.UpdateGroupQuery(&input)
		if err != nil {
			return fmt.Errorf("error updating resource query for resource group (%s): %s", d.Id(), err)
		}
	}

	return resourceAwsResourceGroupsGroupRead(d, meta)
}

func resourceAwsResourceGroupsGroupDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).resourcegroupsconn

	input := resourcegroups.DeleteGroupInput{
		GroupName: aws.String(d.Id()),
	}

	_, err := conn.DeleteGroup(&input)
	if err != nil {
		return fmt.Errorf("error deleting resource group (%s): %s", d.Id(), err)
	}

	return nil
}
<|endoftext|>


<commit_before>//   Copyright 2019 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package route53

import (
	"context"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/route53"
	"github.com/trackit/jsonlog"

	"github.com/trackit/trackit/aws/usageReports"
)

// getRoute53Tags formats []*HostedZone.Tag to map[string]string
func getRoute53Tags(ctx context.Context, hostedZone *route53.HostedZone, svc *route53.Route53) []utils.Tag {
	logger := jsonlog.LoggerFromContextOrDefault(ctx)
	res := make([]utils.Tag, 0)
	var tagResourceType = route53.TagResourceTypeHostedzone
	tags, err := svc.ListTagsForResource(&route53.ListTagsForResourceInput{
		ResourceId: hostedZone.Id,
		ResourceType: &tagResourceType,
	})

	if err != nil {
		logger.Error("Failed to get Route53 tags", err.Error())
		return res
	}

	for _, tag := range tags.ResourceTagSet.Tags {
		res = append(res, utils.Tag{
			Key:   aws.StringValue(tag.Key),
			Value: aws.StringValue(tag.Value),
		})
	}
	return res
}
<commit_msg>Noted potential optimization of route53 tag gathering<commit_after>//   Copyright 2019 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package route53

import (
	"context"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/route53"
	"github.com/trackit/jsonlog"

	"github.com/trackit/trackit/aws/usageReports"
)

// getRoute53Tags formats []*HostedZone.Tag to map[string]string
func getRoute53Tags(ctx context.Context, hostedZone *route53.HostedZone, svc *route53.Route53) []utils.Tag {
	logger := jsonlog.LoggerFromContextOrDefault(ctx)
	res := make([]utils.Tag, 0)
	var tagResourceType = route53.TagResourceTypeHostedzone

	// It could be possible to optimise this to divide the amount of API calls by 10, by using ListTagsForResources which can list tags for up to 10 health checks or hosted zones at once
	tags, err := svc.ListTagsForResource(&route53.ListTagsForResourceInput{
		ResourceId: hostedZone.Id,
		ResourceType: &tagResourceType,
	})

	if err != nil {
		logger.Error("Failed to get Route53 tags", err.Error())
		return res
	}

	for _, tag := range tags.ResourceTagSet.Tags {
		res = append(res, utils.Tag{
			Key:   aws.StringValue(tag.Key),
			Value: aws.StringValue(tag.Value),
		})
	}
	return res
}
<|endoftext|>


<commit_before>package datadog

import (
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"sync/atomic"
	"testing"
	"time"

	"github.com/atlassian/gostatsd/types"

	"github.com/stretchr/testify/assert"
	"golang.org/x/net/context"
)

func TestRetries(t *testing.T) {
	var requestNum uint32
	mux := http.NewServeMux()
	mux.HandleFunc("/api/v1/series", func(w http.ResponseWriter, r *http.Request) {
		defer r.Body.Close()
		atomic.AddUint32(&requestNum, 1)
		data, err := ioutil.ReadAll(r.Body)
		if err != nil {
			t.Error(err)
			return
		}
		if len(data) == 0 {
			t.Errorf("empty body")
		}
		if r.ContentLength != int64(len(data)) {
			t.Errorf("unexpected body length: %d. Content-Length is %d", len(data), r.ContentLength)
		}
		if requestNum == 1 {
			// Return error on first request to trigger a retry
			w.WriteHeader(http.StatusBadRequest)
		}
	})
	ts := httptest.NewServer(mux)
	defer ts.Close()

	client, err := NewClient(ts.URL, "apiKey123", defaultMetricsPerBatch, 1*time.Second, 2*time.Second)
	if err != nil {
		t.Fatal(err)
	}
	res := make(chan []error, 1)
	client.SendMetricsAsync(context.Background(), twoCounters(), func(errs []error) {
		res <- errs
	})
	errs := <-res
	for _, err := range errs {
		if err != nil {
			t.Error(err)
		}
	}
	if requestNum != 2 {
		t.Errorf("unexpected number of requests: %d", requestNum)
	}
}

func TestSendMetricsInMultipleBatches(t *testing.T) {
	var requestNum uint32
	mux := http.NewServeMux()
	mux.HandleFunc("/api/v1/series", func(w http.ResponseWriter, r *http.Request) {
		defer r.Body.Close()
		atomic.AddUint32(&requestNum, 1)
		data, err := ioutil.ReadAll(r.Body)
		if err != nil {
			t.Error(err)
			return
		}
		if len(data) == 0 {
			t.Errorf("empty body")
			w.WriteHeader(http.StatusBadRequest)
		}
		if r.ContentLength != int64(len(data)) {
			t.Errorf("unexpected body length: %d. Content-Length is %d", len(data), r.ContentLength)
			w.WriteHeader(http.StatusBadRequest)
		}
	})
	ts := httptest.NewServer(mux)
	defer ts.Close()

	client, err := NewClient(ts.URL, "apiKey123", 1, 1*time.Second, 2*time.Second)
	if err != nil {
		t.Fatal(err)
	}
	res := make(chan []error, 1)
	client.SendMetricsAsync(context.Background(), twoCounters(), func(errs []error) {
		res <- errs
	})
	errs := <-res
	for _, err := range errs {
		if err != nil {
			t.Error(err)
		}
	}
	if requestNum != 2 {
		t.Errorf("unexpected number of requests: %d", requestNum)
	}
}

func TestSendMetrics(t *testing.T) {
	assert := assert.New(t)
	mux := http.NewServeMux()
	mux.HandleFunc("/api/v1/series", func(w http.ResponseWriter, r *http.Request) {
		defer r.Body.Close()
		data, err := ioutil.ReadAll(r.Body)
		if !assert.Nil(err) {
			return
		}
		expected := `{"series":[` +
			`{"host":"h1","interval":1.1,"metric":"c1","points":[[100,1.1]],"tags":["tag1"],"type":"rate"},` +
			`{"host":"h1","interval":1.1,"metric":"c1.count","points":[[100,5]],"tags":["tag1"],"type":"gauge"},` +
			`{"host":"h2","interval":1.1,"metric":"t1.lower","points":[[100,0]],"tags":["tag2"],"type":"gauge"},` +
			`{"host":"h2","interval":1.1,"metric":"t1.upper","points":[[100,1]],"tags":["tag2"],"type":"gauge"},` +
			`{"host":"h2","interval":1.1,"metric":"t1.count","points":[[100,1]],"tags":["tag2"],"type":"gauge"},` +
			`{"host":"h2","interval":1.1,"metric":"t1.count_ps","points":[[100,1.1]],"tags":["tag2"],"type":"rate"},` +
			`{"host":"h2","interval":1.1,"metric":"t1.mean","points":[[100,0.5]],"tags":["tag2"],"type":"gauge"},` +
			`{"host":"h2","interval":1.1,"metric":"t1.median","points":[[100,0.5]],"tags":["tag2"],"type":"gauge"},` +
			`{"host":"h2","interval":1.1,"metric":"t1.std","points":[[100,0.1]],"tags":["tag2"],"type":"gauge"},` +
			`{"host":"h2","interval":1.1,"metric":"t1.sum","points":[[100,1]],"tags":["tag2"],"type":"gauge"},` +
			`{"host":"h2","interval":1.1,"metric":"t1.sum_squares","points":[[100,1]],"tags":["tag2"],"type":"gauge"},` +
			`{"host":"h2","interval":1.1,"metric":"t1.count_90","points":[[100,0.1]],"tags":["tag2"],"type":"gauge"},` +
			`{"host":"h3","interval":1.1,"metric":"g1","points":[[100,3]],"tags":["tag3"],"type":"gauge"},` +
			`{"host":"h4","interval":1.1,"metric":"users","points":[[100,3]],"tags":["tag4"],"type":"gauge"}]}`
		assert.Equal([]byte(expected), data)
	})
	ts := httptest.NewServer(mux)
	defer ts.Close()

	cli, err := NewClient(ts.URL, "apiKey123", 1000, 1*time.Second, 2*time.Second)
	if err != nil {
		t.Fatal(err)
	}
	cli.(*client).now = func() time.Time {
		return time.Unix(100, 0)
	}
	res := make(chan []error, 1)
	cli.SendMetricsAsync(context.Background(), metricsOneOfEach(), func(errs []error) {
		res <- errs
	})
	errs := <-res
	for _, err := range errs {
		assert.Nil(err)
	}
}

// twoCounters returns two counters.
func twoCounters() *types.MetricMap {
	return &types.MetricMap{
		NumStats: 2,
		Counters: types.Counters{
			"stat1": map[string]types.Counter{
				"tag1": types.NewCounter(types.Nanotime(time.Now().UnixNano()), 5, "", nil),
			},
			"stat2": map[string]types.Counter{
				"tag2": types.NewCounter(types.Nanotime(time.Now().UnixNano()), 50, "", nil),
			},
		},
	}
}

func metricsOneOfEach() *types.MetricMap {
	return &types.MetricMap{
		NumStats:       4,
		ProcessingTime: 10 * time.Millisecond,
		FlushInterval:  1100 * time.Millisecond,
		Counters: types.Counters{
			"c1": map[string]types.Counter{
				"tag1": {PerSecond: 1.1, Value: 5, Timestamp: types.Nanotime(100), Hostname: "h1", Tags: types.Tags{"tag1"}},
			},
		},
		Timers: types.Timers{
			"t1": map[string]types.Timer{
				"tag2": {
					Count:      1,
					PerSecond:  1.1,
					Mean:       0.5,
					Median:     0.5,
					Min:        0,
					Max:        1,
					StdDev:     0.1,
					Sum:        1,
					SumSquares: 1,
					Values:     []float64{0, 1},
					Percentiles: types.Percentiles{
						types.Percentile{Float: 0.1, Str: "count_90"},
					},
					Timestamp: types.Nanotime(200),
					Hostname:  "h2",
					Tags:      types.Tags{"tag2"},
				},
			},
		},
		Gauges: types.Gauges{
			"g1": map[string]types.Gauge{
				"tag3": {Value: 3, Timestamp: types.Nanotime(300), Hostname: "h3", Tags: types.Tags{"tag3"}},
			},
		},
		Sets: types.Sets{
			"users": map[string]types.Set{
				"tag4": {
					Values: map[string]struct{}{
						"joe":  {},
						"bob":  {},
						"john": {},
					},
					Timestamp: types.Nanotime(400),
					Hostname:  "h4",
					Tags:      types.Tags{"tag4"},
				},
			},
		},
	}
}
<commit_msg>Cleanup tests in Datadog backend<commit_after>package datadog

import (
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"sync/atomic"
	"testing"
	"time"

	"github.com/atlassian/gostatsd/types"

	"github.com/stretchr/testify/assert"
	"golang.org/x/net/context"
)

func TestRetries(t *testing.T) {
	assert := assert.New(t)
	var requestNum uint32
	mux := http.NewServeMux()
	mux.HandleFunc("/api/v1/series", func(w http.ResponseWriter, r *http.Request) {
		defer r.Body.Close()
		n := atomic.AddUint32(&requestNum, 1)
		data, err := ioutil.ReadAll(r.Body)
		if !assert.Nil(err) {
			return
		}
		assert.NotEmpty(data)
		if n == 1 {
			// Return error on first request to trigger a retry
			w.WriteHeader(http.StatusBadRequest)
		}
	})
	ts := httptest.NewServer(mux)
	defer ts.Close()

	client, err := NewClient(ts.URL, "apiKey123", defaultMetricsPerBatch, 1*time.Second, 2*time.Second)
	if !assert.Nil(err) {
		return
	}
	res := make(chan []error, 1)
	client.SendMetricsAsync(context.Background(), twoCounters(), func(errs []error) {
		res <- errs
	})
	errs := <-res
	for _, err := range errs {
		assert.Nil(err)
	}
	assert.EqualValues(2, requestNum)
}

func TestSendMetricsInMultipleBatches(t *testing.T) {
	assert := assert.New(t)
	var requestNum uint32
	mux := http.NewServeMux()
	mux.HandleFunc("/api/v1/series", func(w http.ResponseWriter, r *http.Request) {
		defer r.Body.Close()
		atomic.AddUint32(&requestNum, 1)
		data, err := ioutil.ReadAll(r.Body)
		if !assert.Nil(err) {
			return
		}
		assert.NotEmpty(data)
	})
	ts := httptest.NewServer(mux)
	defer ts.Close()

	client, err := NewClient(ts.URL, "apiKey123", 1, 1*time.Second, 2*time.Second)
	if !assert.Nil(err) {
		return
	}
	res := make(chan []error, 1)
	client.SendMetricsAsync(context.Background(), twoCounters(), func(errs []error) {
		res <- errs
	})
	errs := <-res
	for _, err := range errs {
		assert.Nil(err)
	}
	assert.EqualValues(2, requestNum)
}

func TestSendMetrics(t *testing.T) {
	assert := assert.New(t)
	mux := http.NewServeMux()
	mux.HandleFunc("/api/v1/series", func(w http.ResponseWriter, r *http.Request) {
		defer r.Body.Close()
		data, err := ioutil.ReadAll(r.Body)
		if !assert.Nil(err) {
			return
		}
		expected := `{"series":[` +
			`{"host":"h1","interval":1.1,"metric":"c1","points":[[100,1.1]],"tags":["tag1"],"type":"rate"},` +
			`{"host":"h1","interval":1.1,"metric":"c1.count","points":[[100,5]],"tags":["tag1"],"type":"gauge"},` +
			`{"host":"h2","interval":1.1,"metric":"t1.lower","points":[[100,0]],"tags":["tag2"],"type":"gauge"},` +
			`{"host":"h2","interval":1.1,"metric":"t1.upper","points":[[100,1]],"tags":["tag2"],"type":"gauge"},` +
			`{"host":"h2","interval":1.1,"metric":"t1.count","points":[[100,1]],"tags":["tag2"],"type":"gauge"},` +
			`{"host":"h2","interval":1.1,"metric":"t1.count_ps","points":[[100,1.1]],"tags":["tag2"],"type":"rate"},` +
			`{"host":"h2","interval":1.1,"metric":"t1.mean","points":[[100,0.5]],"tags":["tag2"],"type":"gauge"},` +
			`{"host":"h2","interval":1.1,"metric":"t1.median","points":[[100,0.5]],"tags":["tag2"],"type":"gauge"},` +
			`{"host":"h2","interval":1.1,"metric":"t1.std","points":[[100,0.1]],"tags":["tag2"],"type":"gauge"},` +
			`{"host":"h2","interval":1.1,"metric":"t1.sum","points":[[100,1]],"tags":["tag2"],"type":"gauge"},` +
			`{"host":"h2","interval":1.1,"metric":"t1.sum_squares","points":[[100,1]],"tags":["tag2"],"type":"gauge"},` +
			`{"host":"h2","interval":1.1,"metric":"t1.count_90","points":[[100,0.1]],"tags":["tag2"],"type":"gauge"},` +
			`{"host":"h3","interval":1.1,"metric":"g1","points":[[100,3]],"tags":["tag3"],"type":"gauge"},` +
			`{"host":"h4","interval":1.1,"metric":"users","points":[[100,3]],"tags":["tag4"],"type":"gauge"}]}`
		assert.Equal([]byte(expected), data)
	})
	ts := httptest.NewServer(mux)
	defer ts.Close()

	cli, err := NewClient(ts.URL, "apiKey123", 1000, 1*time.Second, 2*time.Second)
	if !assert.Nil(err) {
		return
	}
	cli.(*client).now = func() time.Time {
		return time.Unix(100, 0)
	}
	res := make(chan []error, 1)
	cli.SendMetricsAsync(context.Background(), metricsOneOfEach(), func(errs []error) {
		res <- errs
	})
	errs := <-res
	for _, err := range errs {
		assert.Nil(err)
	}
}

// twoCounters returns two counters.
func twoCounters() *types.MetricMap {
	return &types.MetricMap{
		NumStats: 2,
		Counters: types.Counters{
			"stat1": map[string]types.Counter{
				"tag1": types.NewCounter(types.Nanotime(time.Now().UnixNano()), 5, "", nil),
			},
			"stat2": map[string]types.Counter{
				"tag2": types.NewCounter(types.Nanotime(time.Now().UnixNano()), 50, "", nil),
			},
		},
	}
}

func metricsOneOfEach() *types.MetricMap {
	return &types.MetricMap{
		NumStats:       4,
		ProcessingTime: 10 * time.Millisecond,
		FlushInterval:  1100 * time.Millisecond,
		Counters: types.Counters{
			"c1": map[string]types.Counter{
				"tag1": {PerSecond: 1.1, Value: 5, Timestamp: types.Nanotime(100), Hostname: "h1", Tags: types.Tags{"tag1"}},
			},
		},
		Timers: types.Timers{
			"t1": map[string]types.Timer{
				"tag2": {
					Count:      1,
					PerSecond:  1.1,
					Mean:       0.5,
					Median:     0.5,
					Min:        0,
					Max:        1,
					StdDev:     0.1,
					Sum:        1,
					SumSquares: 1,
					Values:     []float64{0, 1},
					Percentiles: types.Percentiles{
						types.Percentile{Float: 0.1, Str: "count_90"},
					},
					Timestamp: types.Nanotime(200),
					Hostname:  "h2",
					Tags:      types.Tags{"tag2"},
				},
			},
		},
		Gauges: types.Gauges{
			"g1": map[string]types.Gauge{
				"tag3": {Value: 3, Timestamp: types.Nanotime(300), Hostname: "h3", Tags: types.Tags{"tag3"}},
			},
		},
		Sets: types.Sets{
			"users": map[string]types.Set{
				"tag4": {
					Values: map[string]struct{}{
						"joe":  {},
						"bob":  {},
						"john": {},
					},
					Timestamp: types.Nanotime(400),
					Hostname:  "h4",
					Tags:      types.Tags{"tag4"},
				},
			},
		},
	}
}
<|endoftext|>


<commit_before>package remote

import (
	"context"
	"fmt"
	"reflect"
	"strconv"
	"time"

	"cloud.google.com/go/storage"
	"github.com/gruntwork-io/terragrunt/errors"
	"github.com/gruntwork-io/terragrunt/options"
	"github.com/gruntwork-io/terragrunt/shell"
	"github.com/gruntwork-io/terragrunt/util"
	"github.com/mitchellh/mapstructure"
)

/*
 * We use this construct to separate the config key 'gcs_bucket_labels' from the others, as they
 * are specific to the gcs backend, but only used by terragrunt to tag the gcs bucket in case it
 * has to create them.
 */
type ExtendedRemoteStateConfigGCS struct {
	remoteStateConfigGCS RemoteStateConfigGCS

	Project              string            `mapstructure:"project"`
	Location             string            `mapstructure:"location"`
	GCSBucketLabels      map[string]string `mapstructure:"gcs_bucket_labels"`
	SkipBucketVersioning bool              `mapstructure:"skip_bucket_versioning"`
}

// These are settings that can appear in the remote_state config that are ONLY used by Terragrunt and NOT forwarded
// to the underlying Terraform backend configuration.
var terragruntGCSOnlyConfigs = []string{
	"project",
	"location",
	"gcs_bucket_labels",
	"skip_bucket_versioning",
}

// A representation of the configuration options available for GCS remote state
type RemoteStateConfigGCS struct {
	Bucket        string `mapstructure:"bucket"`
	Credentials   string `mapstructure:"credentials"`
	Prefix        string `mapstructure:"prefix"`
	Path          string `mapstructure:"path"`
	EncryptionKey string `mapstructure:"encryption_key"`
}

const MAX_RETRIES_WAITING_FOR_GCS_BUCKET = 12
const SLEEP_BETWEEN_RETRIES_WAITING_FOR_GCS_BUCKET = 5 * time.Second

type GCSInitializer struct{}

// Returns true if:
//
// 1. Any of the existing backend settings are different than the current config
// 2. The configured GCS bucket does not exist
func (gcsInitializer GCSInitializer) NeedsInitialization(config map[string]interface{}, existingBackend *TerraformBackend, terragruntOptions *options.TerragruntOptions) (bool, error) {
	if !gcsConfigValuesEqual(config, existingBackend, terragruntOptions) {
		return true, nil
	}

	gcsConfig, err := parseGCSConfig(config)
	if err != nil {
		return false, err
	}

	gcsClient, err := CreateGCSClient()
	if err != nil {
		return false, err
	}

	if !DoesGCSBucketExist(gcsClient, gcsConfig) {
		return true, nil
	}

	return false, nil
}

// Return true if the given config is in any way different than what is configured for the backend
func gcsConfigValuesEqual(config map[string]interface{}, existingBackend *TerraformBackend, terragruntOptions *options.TerragruntOptions) bool {
	if existingBackend == nil {
		return len(config) == 0
	}

	if existingBackend.Type != "gcs" {
		terragruntOptions.Logger.Printf("Backend type has changed from gcs to %s", existingBackend.Type)
		return false
	}

	if len(config) == 0 && len(existingBackend.Config) == 0 {
		return true
	}

	// If other keys in config are bools, DeepEqual also will consider the maps to be different.
	for key, value := range existingBackend.Config {
		if util.KindOf(existingBackend.Config[key]) == reflect.String && util.KindOf(config[key]) == reflect.Bool {
			if convertedValue, err := strconv.ParseBool(value.(string)); err == nil {
				existingBackend.Config[key] = convertedValue
			}
		}
	}

	// Delete custom GCS labels that are only used in Terragrunt config and not in Terraform's backend
	for _, key := range terragruntGCSOnlyConfigs {
		delete(config, key)
	}

	if !terraformStateConfigEqual(existingBackend.Config, config) {
		terragruntOptions.Logger.Printf("Backend config has changed from %s to %s", existingBackend.Config, config)
		return false
	}

	return true
}

// Initialize the remote state GCS bucket specified in the given config. This function will validate the config
// parameters, create the GCS bucket if it doesn't already exist, and check that versioning is enabled.
func (gcsInitializer GCSInitializer) Initialize(config map[string]interface{}, terragruntOptions *options.TerragruntOptions) error {
	gcsConfigExtended, err := parseExtendedGCSConfig(config)
	if err != nil {
		return err
	}

	if err := validateGCSConfig(gcsConfigExtended, terragruntOptions); err != nil {
		return err
	}

	var gcsConfig = gcsConfigExtended.remoteStateConfigGCS

	gcsClient, err := CreateGCSClient()
	if err != nil {
		return err
	}

	if err := createGCSBucketIfNecessary(gcsClient, gcsConfigExtended, terragruntOptions); err != nil {
		return err
	}

	if err := checkIfGCSVersioningEnabled(gcsClient, &gcsConfig, terragruntOptions); err != nil {
		return err
	}

	return nil
}

func (gcsInitializer GCSInitializer) GetTerraformInitArgs(config map[string]interface{}) map[string]interface{} {
	var filteredConfig = make(map[string]interface{})

	for key, val := range config {
		if util.ListContainsElement(terragruntGCSOnlyConfigs, key) {
			continue
		}

		filteredConfig[key] = val
	}

	return filteredConfig
}

// Parse the given map into a GCS config
func parseGCSConfig(config map[string]interface{}) (*RemoteStateConfigGCS, error) {
	var gcsConfig RemoteStateConfigGCS
	if err := mapstructure.Decode(config, &gcsConfig); err != nil {
		return nil, errors.WithStackTrace(err)
	}

	return &gcsConfig, nil
}

// Parse the given map into a GCS config
func parseExtendedGCSConfig(config map[string]interface{}) (*ExtendedRemoteStateConfigGCS, error) {
	var gcsConfig RemoteStateConfigGCS
	var extendedConfig ExtendedRemoteStateConfigGCS

	if err := mapstructure.Decode(config, &gcsConfig); err != nil {
		return nil, errors.WithStackTrace(err)
	}

	if err := mapstructure.Decode(config, &extendedConfig); err != nil {
		return nil, errors.WithStackTrace(err)
	}

	extendedConfig.remoteStateConfigGCS = gcsConfig

	return &extendedConfig, nil
}

// Validate all the parameters of the given GCS remote state configuration
func validateGCSConfig(extendedConfig *ExtendedRemoteStateConfigGCS, terragruntOptions *options.TerragruntOptions) error {
	// A project must be specified in order for terragrunt to automatically create a storage bucket.
	if extendedConfig.Project == "" {
		return errors.WithStackTrace(MissingRequiredGCSRemoteStateConfig("project"))
	}

	var config = extendedConfig.remoteStateConfigGCS

	if config.Bucket == "" {
		return errors.WithStackTrace(MissingRequiredGCSRemoteStateConfig("bucket"))
	}

	if config.Prefix == "" {
		return errors.WithStackTrace(MissingRequiredGCSRemoteStateConfig("prefix"))
	}

	return nil
}

// If the bucket specified in the given config doesn't already exist, prompt the user to create it, and if the user
// confirms, create the bucket and enable versioning for it.
func createGCSBucketIfNecessary(gcsClient *storage.Client, config *ExtendedRemoteStateConfigGCS, terragruntOptions *options.TerragruntOptions) error {
	if !DoesGCSBucketExist(gcsClient, &config.remoteStateConfigGCS) {
		prompt := fmt.Sprintf("Remote state GCS bucket %s does not exist or you don't have permissions to access it. Would you like Terragrunt to create it?", config.remoteStateConfigGCS.Bucket)
		shouldCreateBucket, err := shell.PromptUserForYesNo(prompt, terragruntOptions)
		if err != nil {
			return err
		}

		if shouldCreateBucket {
			// To avoid any eventual consistency issues with creating a GCS bucket we use a retry loop.
			description := fmt.Sprintf("Create GCS bucket %s", config.remoteStateConfigGCS.Bucket)
			maxRetries := 3
			sleepBetweenRetries := 10 * time.Second

			return util.DoWithRetry(description, maxRetries, sleepBetweenRetries, terragruntOptions.Logger, func() error {
				return CreateGCSBucketWithVersioning(gcsClient, config, terragruntOptions)
			})
		}
	}

	return nil
}

// Check if versioning is enabled for the GCS bucket specified in the given config and warn the user if it is not
func checkIfGCSVersioningEnabled(gcsClient *storage.Client, config *RemoteStateConfigGCS, terragruntOptions *options.TerragruntOptions) error {
	ctx := context.Background()
	bucket := gcsClient.Bucket(config.Bucket)

	attrs, err := bucket.Attrs(ctx)
	if err != nil {
		// ErrBucketNotExist
		return errors.WithStackTrace(err)
	}

	if attrs.VersioningEnabled == false {
		terragruntOptions.Logger.Printf("WARNING: Versioning is not enabled for the remote state GCS bucket %s. We recommend enabling versioning so that you can roll back to previous versions of your Terraform state in case of error.", config.Bucket)
	}

	return nil
}

// CreateGCSBucketWithVersioning creates the given GCS bucket and enables versioning for it.
func CreateGCSBucketWithVersioning(gcsClient *storage.Client, config *ExtendedRemoteStateConfigGCS, terragruntOptions *options.TerragruntOptions) error {
	err := CreateGCSBucket(gcsClient, config, terragruntOptions)

	if err != nil {
		return err
	}

	if err := WaitUntilGCSBucketExists(gcsClient, &config.remoteStateConfigGCS, terragruntOptions); err != nil {
		return err
	}

	if err := AddLabelsToGCSBucket(gcsClient, config, terragruntOptions); err != nil {
		return err
	}

	return nil
}

func AddLabelsToGCSBucket(gcsClient *storage.Client, config *ExtendedRemoteStateConfigGCS, terragruntOptions *options.TerragruntOptions) error {
	if config.GCSBucketLabels == nil || len(config.GCSBucketLabels) == 0 {
		terragruntOptions.Logger.Printf("No labels specified for bucket %s.", config.remoteStateConfigGCS.Bucket)
		return nil
	}

	terragruntOptions.Logger.Printf("Adding labels to GCS bucket with %s", config.GCSBucketLabels)

	ctx := context.Background()
	bucket := gcsClient.Bucket(config.remoteStateConfigGCS.Bucket)

	bucketAttrs := *&storage.BucketAttrsToUpdate{}

	for key, value := range config.GCSBucketLabels {
		bucketAttrs.SetLabel(key, value)
	}

	_, err := bucket.Update(ctx, bucketAttrs)

	if err != nil {
		return errors.WithStackTrace(err)
	}

	return nil

}

// Create the GCS bucket specified in the given config
func CreateGCSBucket(gcsClient *storage.Client, config *ExtendedRemoteStateConfigGCS, terragruntOptions *options.TerragruntOptions) error {
	terragruntOptions.Logger.Printf("Creating GCS bucket %s in project %s", config.remoteStateConfigGCS.Bucket, config.Project)

	// The project ID to which the bucket belongs. This is only used when creating a new bucket during initialization.
	// Since buckets have globally unique names, the project ID is not required to access the bucket during normal
	// operation.
	projectID := config.Project

	ctx := context.Background()
	bucket := gcsClient.Bucket(config.remoteStateConfigGCS.Bucket)

	bucketAttrs := &storage.BucketAttrs{}

	if config.Location != "" {
		terragruntOptions.Logger.Printf("Creating GCS bucket in location %s.", config.Location)
	}

	if config.SkipBucketVersioning {
		terragruntOptions.Logger.Printf("Versioning is disabled for the remote state GCS bucket %s using 'skip_bucket_versioning' config.", config.remoteStateConfigGCS.Bucket)
	} else {
		terragruntOptions.Logger.Printf("Enabling versioning on GCS bucket %s", config.remoteStateConfigGCS.Bucket)
		bucketAttrs.VersioningEnabled = true
	}

	err := bucket.Create(ctx, projectID, bucketAttrs)
	return errors.WithStackTrace(err)
}

// GCP is eventually consistent, so after creating a GCS bucket, this method can be used to wait until the information
// about that GCS bucket has propagated everywhere.
func WaitUntilGCSBucketExists(gcsClient *storage.Client, config *RemoteStateConfigGCS, terragruntOptions *options.TerragruntOptions) error {
	terragruntOptions.Logger.Printf("Waiting for bucket %s to be created", config.Bucket)
	for retries := 0; retries < MAX_RETRIES_WAITING_FOR_GCS_BUCKET; retries++ {
		if DoesGCSBucketExist(gcsClient, config) {
			terragruntOptions.Logger.Printf("GCS bucket %s created.", config.Bucket)
			return nil
		} else if retries < MAX_RETRIES_WAITING_FOR_GCS_BUCKET-1 {
			terragruntOptions.Logger.Printf("GCS bucket %s has not been created yet. Sleeping for %s and will check again.", config.Bucket, SLEEP_BETWEEN_RETRIES_WAITING_FOR_GCS_BUCKET)
			time.Sleep(SLEEP_BETWEEN_RETRIES_WAITING_FOR_GCS_BUCKET)
		}
	}

	return errors.WithStackTrace(MaxRetriesWaitingForS3BucketExceeded(config.Bucket))
}

// DoesGCSBucketExist returns true if the GCS bucket specified in the given config exists and the current user has the
// ability to access it.
func DoesGCSBucketExist(gcsClient *storage.Client, config *RemoteStateConfigGCS) bool {
	ctx := context.Background()

	// Creates a Bucket instance.
	bucket := gcsClient.Bucket(config.Bucket)

	// TODO - the code below attempts to determine whether the storage bucket exists by making a making a number of API
	// calls, then attemping to list the contents of the bucket. It was adapted from Google's own integration tests and
	// should be improved once the appropriate API call is added. For more info see:
	// https://github.com/GoogleCloudPlatform/google-cloud-go/blob/de879f7be552d57556875b8aaa383bce9396cc8c/storage/integration_test.go#L1231
	if _, err := bucket.Attrs(ctx); err != nil {
		// ErrBucketNotExist
		return false
	}

	it := bucket.Objects(ctx, nil)
	if _, err := it.Next(); err == storage.ErrBucketNotExist {
		return false
	}

	return true
}

// CreateGCSClient creates an authenticated client for GCS
func CreateGCSClient() (*storage.Client, error) {
	ctx := context.Background()

	client, err := storage.NewClient(ctx)
	if err != nil {
		return nil, err
	}

	return client, nil
}

// Custom error types

type MissingRequiredGCSRemoteStateConfig string

func (configName MissingRequiredGCSRemoteStateConfig) Error() string {
	return fmt.Sprintf("Missing required GCS remote state configuration %s", string(configName))
}

type MaxRetriesWaitingForGCSBucketExceeded string

func (err MaxRetriesWaitingForGCSBucketExceeded) Error() string {
	return fmt.Sprintf("Exceeded max retries (%d) waiting for GCS bucket %s", MAX_RETRIES_WAITING_FOR_GCS_BUCKET, string(err))
}

type MaxRetriesWaitingForGCSACLExceeded string

func (err MaxRetriesWaitingForGCSACLExceeded) Error() string {
	return fmt.Sprintf("Exceeded max retries waiting for GCS bucket %s to have proper ACL for access logging", string(err))
}
<commit_msg>Fix merge conflict from new GCS initializer and disable_init config<commit_after>package remote

import (
	"context"
	"fmt"
	"reflect"
	"strconv"
	"time"

	"cloud.google.com/go/storage"
	"github.com/gruntwork-io/terragrunt/errors"
	"github.com/gruntwork-io/terragrunt/options"
	"github.com/gruntwork-io/terragrunt/shell"
	"github.com/gruntwork-io/terragrunt/util"
	"github.com/mitchellh/mapstructure"
)

/*
 * We use this construct to separate the config key 'gcs_bucket_labels' from the others, as they
 * are specific to the gcs backend, but only used by terragrunt to tag the gcs bucket in case it
 * has to create them.
 */
type ExtendedRemoteStateConfigGCS struct {
	remoteStateConfigGCS RemoteStateConfigGCS

	Project              string            `mapstructure:"project"`
	Location             string            `mapstructure:"location"`
	GCSBucketLabels      map[string]string `mapstructure:"gcs_bucket_labels"`
	SkipBucketVersioning bool              `mapstructure:"skip_bucket_versioning"`
}

// These are settings that can appear in the remote_state config that are ONLY used by Terragrunt and NOT forwarded
// to the underlying Terraform backend configuration.
var terragruntGCSOnlyConfigs = []string{
	"project",
	"location",
	"gcs_bucket_labels",
	"skip_bucket_versioning",
}

// A representation of the configuration options available for GCS remote state
type RemoteStateConfigGCS struct {
	Bucket        string `mapstructure:"bucket"`
	Credentials   string `mapstructure:"credentials"`
	Prefix        string `mapstructure:"prefix"`
	Path          string `mapstructure:"path"`
	EncryptionKey string `mapstructure:"encryption_key"`
}

const MAX_RETRIES_WAITING_FOR_GCS_BUCKET = 12
const SLEEP_BETWEEN_RETRIES_WAITING_FOR_GCS_BUCKET = 5 * time.Second

type GCSInitializer struct{}

// Returns true if:
//
// 1. Any of the existing backend settings are different than the current config
// 2. The configured GCS bucket does not exist
func (gcsInitializer GCSInitializer) NeedsInitialization(remoteState *RemoteState, existingBackend *TerraformBackend, terragruntOptions *options.TerragruntOptions) (bool, error) {
	if remoteState.DisableInit {
		return false, nil
	}

	if !gcsConfigValuesEqual(remoteState.Config, existingBackend, terragruntOptions) {
		return true, nil
	}

	gcsConfig, err := parseGCSConfig(remoteState.Config)
	if err != nil {
		return false, err
	}

	gcsClient, err := CreateGCSClient()
	if err != nil {
		return false, err
	}

	if !DoesGCSBucketExist(gcsClient, gcsConfig) {
		return true, nil
	}

	return false, nil
}

// Return true if the given config is in any way different than what is configured for the backend
func gcsConfigValuesEqual(config map[string]interface{}, existingBackend *TerraformBackend, terragruntOptions *options.TerragruntOptions) bool {
	if existingBackend == nil {
		return len(config) == 0
	}

	if existingBackend.Type != "gcs" {
		terragruntOptions.Logger.Printf("Backend type has changed from gcs to %s", existingBackend.Type)
		return false
	}

	if len(config) == 0 && len(existingBackend.Config) == 0 {
		return true
	}

	// If other keys in config are bools, DeepEqual also will consider the maps to be different.
	for key, value := range existingBackend.Config {
		if util.KindOf(existingBackend.Config[key]) == reflect.String && util.KindOf(config[key]) == reflect.Bool {
			if convertedValue, err := strconv.ParseBool(value.(string)); err == nil {
				existingBackend.Config[key] = convertedValue
			}
		}
	}

	// Delete custom GCS labels that are only used in Terragrunt config and not in Terraform's backend
	for _, key := range terragruntGCSOnlyConfigs {
		delete(config, key)
	}

	if !terraformStateConfigEqual(existingBackend.Config, config) {
		terragruntOptions.Logger.Printf("Backend config has changed from %s to %s", existingBackend.Config, config)
		return false
	}

	return true
}

// Initialize the remote state GCS bucket specified in the given config. This function will validate the config
// parameters, create the GCS bucket if it doesn't already exist, and check that versioning is enabled.
func (gcsInitializer GCSInitializer) Initialize(remoteState *RemoteState, terragruntOptions *options.TerragruntOptions) error {
	gcsConfigExtended, err := parseExtendedGCSConfig(remoteState.Config)
	if err != nil {
		return err
	}

	if err := validateGCSConfig(gcsConfigExtended, terragruntOptions); err != nil {
		return err
	}

	var gcsConfig = gcsConfigExtended.remoteStateConfigGCS

	gcsClient, err := CreateGCSClient()
	if err != nil {
		return err
	}

	if err := createGCSBucketIfNecessary(gcsClient, gcsConfigExtended, terragruntOptions); err != nil {
		return err
	}

	if err := checkIfGCSVersioningEnabled(gcsClient, &gcsConfig, terragruntOptions); err != nil {
		return err
	}

	return nil
}

func (gcsInitializer GCSInitializer) GetTerraformInitArgs(config map[string]interface{}) map[string]interface{} {
	var filteredConfig = make(map[string]interface{})

	for key, val := range config {
		if util.ListContainsElement(terragruntGCSOnlyConfigs, key) {
			continue
		}

		filteredConfig[key] = val
	}

	return filteredConfig
}

// Parse the given map into a GCS config
func parseGCSConfig(config map[string]interface{}) (*RemoteStateConfigGCS, error) {
	var gcsConfig RemoteStateConfigGCS
	if err := mapstructure.Decode(config, &gcsConfig); err != nil {
		return nil, errors.WithStackTrace(err)
	}

	return &gcsConfig, nil
}

// Parse the given map into a GCS config
func parseExtendedGCSConfig(config map[string]interface{}) (*ExtendedRemoteStateConfigGCS, error) {
	var gcsConfig RemoteStateConfigGCS
	var extendedConfig ExtendedRemoteStateConfigGCS

	if err := mapstructure.Decode(config, &gcsConfig); err != nil {
		return nil, errors.WithStackTrace(err)
	}

	if err := mapstructure.Decode(config, &extendedConfig); err != nil {
		return nil, errors.WithStackTrace(err)
	}

	extendedConfig.remoteStateConfigGCS = gcsConfig

	return &extendedConfig, nil
}

// Validate all the parameters of the given GCS remote state configuration
func validateGCSConfig(extendedConfig *ExtendedRemoteStateConfigGCS, terragruntOptions *options.TerragruntOptions) error {
	// A project must be specified in order for terragrunt to automatically create a storage bucket.
	if extendedConfig.Project == "" {
		return errors.WithStackTrace(MissingRequiredGCSRemoteStateConfig("project"))
	}

	var config = extendedConfig.remoteStateConfigGCS

	if config.Bucket == "" {
		return errors.WithStackTrace(MissingRequiredGCSRemoteStateConfig("bucket"))
	}

	if config.Prefix == "" {
		return errors.WithStackTrace(MissingRequiredGCSRemoteStateConfig("prefix"))
	}

	return nil
}

// If the bucket specified in the given config doesn't already exist, prompt the user to create it, and if the user
// confirms, create the bucket and enable versioning for it.
func createGCSBucketIfNecessary(gcsClient *storage.Client, config *ExtendedRemoteStateConfigGCS, terragruntOptions *options.TerragruntOptions) error {
	if !DoesGCSBucketExist(gcsClient, &config.remoteStateConfigGCS) {
		prompt := fmt.Sprintf("Remote state GCS bucket %s does not exist or you don't have permissions to access it. Would you like Terragrunt to create it?", config.remoteStateConfigGCS.Bucket)
		shouldCreateBucket, err := shell.PromptUserForYesNo(prompt, terragruntOptions)
		if err != nil {
			return err
		}

		if shouldCreateBucket {
			// To avoid any eventual consistency issues with creating a GCS bucket we use a retry loop.
			description := fmt.Sprintf("Create GCS bucket %s", config.remoteStateConfigGCS.Bucket)
			maxRetries := 3
			sleepBetweenRetries := 10 * time.Second

			return util.DoWithRetry(description, maxRetries, sleepBetweenRetries, terragruntOptions.Logger, func() error {
				return CreateGCSBucketWithVersioning(gcsClient, config, terragruntOptions)
			})
		}
	}

	return nil
}

// Check if versioning is enabled for the GCS bucket specified in the given config and warn the user if it is not
func checkIfGCSVersioningEnabled(gcsClient *storage.Client, config *RemoteStateConfigGCS, terragruntOptions *options.TerragruntOptions) error {
	ctx := context.Background()
	bucket := gcsClient.Bucket(config.Bucket)

	attrs, err := bucket.Attrs(ctx)
	if err != nil {
		// ErrBucketNotExist
		return errors.WithStackTrace(err)
	}

	if attrs.VersioningEnabled == false {
		terragruntOptions.Logger.Printf("WARNING: Versioning is not enabled for the remote state GCS bucket %s. We recommend enabling versioning so that you can roll back to previous versions of your Terraform state in case of error.", config.Bucket)
	}

	return nil
}

// CreateGCSBucketWithVersioning creates the given GCS bucket and enables versioning for it.
func CreateGCSBucketWithVersioning(gcsClient *storage.Client, config *ExtendedRemoteStateConfigGCS, terragruntOptions *options.TerragruntOptions) error {
	err := CreateGCSBucket(gcsClient, config, terragruntOptions)

	if err != nil {
		return err
	}

	if err := WaitUntilGCSBucketExists(gcsClient, &config.remoteStateConfigGCS, terragruntOptions); err != nil {
		return err
	}

	if err := AddLabelsToGCSBucket(gcsClient, config, terragruntOptions); err != nil {
		return err
	}

	return nil
}

func AddLabelsToGCSBucket(gcsClient *storage.Client, config *ExtendedRemoteStateConfigGCS, terragruntOptions *options.TerragruntOptions) error {
	if config.GCSBucketLabels == nil || len(config.GCSBucketLabels) == 0 {
		terragruntOptions.Logger.Printf("No labels specified for bucket %s.", config.remoteStateConfigGCS.Bucket)
		return nil
	}

	terragruntOptions.Logger.Printf("Adding labels to GCS bucket with %s", config.GCSBucketLabels)

	ctx := context.Background()
	bucket := gcsClient.Bucket(config.remoteStateConfigGCS.Bucket)

	bucketAttrs := *&storage.BucketAttrsToUpdate{}

	for key, value := range config.GCSBucketLabels {
		bucketAttrs.SetLabel(key, value)
	}

	_, err := bucket.Update(ctx, bucketAttrs)

	if err != nil {
		return errors.WithStackTrace(err)
	}

	return nil

}

// Create the GCS bucket specified in the given config
func CreateGCSBucket(gcsClient *storage.Client, config *ExtendedRemoteStateConfigGCS, terragruntOptions *options.TerragruntOptions) error {
	terragruntOptions.Logger.Printf("Creating GCS bucket %s in project %s", config.remoteStateConfigGCS.Bucket, config.Project)

	// The project ID to which the bucket belongs. This is only used when creating a new bucket during initialization.
	// Since buckets have globally unique names, the project ID is not required to access the bucket during normal
	// operation.
	projectID := config.Project

	ctx := context.Background()
	bucket := gcsClient.Bucket(config.remoteStateConfigGCS.Bucket)

	bucketAttrs := &storage.BucketAttrs{}

	if config.Location != "" {
		terragruntOptions.Logger.Printf("Creating GCS bucket in location %s.", config.Location)
	}

	if config.SkipBucketVersioning {
		terragruntOptions.Logger.Printf("Versioning is disabled for the remote state GCS bucket %s using 'skip_bucket_versioning' config.", config.remoteStateConfigGCS.Bucket)
	} else {
		terragruntOptions.Logger.Printf("Enabling versioning on GCS bucket %s", config.remoteStateConfigGCS.Bucket)
		bucketAttrs.VersioningEnabled = true
	}

	err := bucket.Create(ctx, projectID, bucketAttrs)
	return errors.WithStackTrace(err)
}

// GCP is eventually consistent, so after creating a GCS bucket, this method can be used to wait until the information
// about that GCS bucket has propagated everywhere.
func WaitUntilGCSBucketExists(gcsClient *storage.Client, config *RemoteStateConfigGCS, terragruntOptions *options.TerragruntOptions) error {
	terragruntOptions.Logger.Printf("Waiting for bucket %s to be created", config.Bucket)
	for retries := 0; retries < MAX_RETRIES_WAITING_FOR_GCS_BUCKET; retries++ {
		if DoesGCSBucketExist(gcsClient, config) {
			terragruntOptions.Logger.Printf("GCS bucket %s created.", config.Bucket)
			return nil
		} else if retries < MAX_RETRIES_WAITING_FOR_GCS_BUCKET-1 {
			terragruntOptions.Logger.Printf("GCS bucket %s has not been created yet. Sleeping for %s and will check again.", config.Bucket, SLEEP_BETWEEN_RETRIES_WAITING_FOR_GCS_BUCKET)
			time.Sleep(SLEEP_BETWEEN_RETRIES_WAITING_FOR_GCS_BUCKET)
		}
	}

	return errors.WithStackTrace(MaxRetriesWaitingForS3BucketExceeded(config.Bucket))
}

// DoesGCSBucketExist returns true if the GCS bucket specified in the given config exists and the current user has the
// ability to access it.
func DoesGCSBucketExist(gcsClient *storage.Client, config *RemoteStateConfigGCS) bool {
	ctx := context.Background()

	// Creates a Bucket instance.
	bucket := gcsClient.Bucket(config.Bucket)

	// TODO - the code below attempts to determine whether the storage bucket exists by making a making a number of API
	// calls, then attemping to list the contents of the bucket. It was adapted from Google's own integration tests and
	// should be improved once the appropriate API call is added. For more info see:
	// https://github.com/GoogleCloudPlatform/google-cloud-go/blob/de879f7be552d57556875b8aaa383bce9396cc8c/storage/integration_test.go#L1231
	if _, err := bucket.Attrs(ctx); err != nil {
		// ErrBucketNotExist
		return false
	}

	it := bucket.Objects(ctx, nil)
	if _, err := it.Next(); err == storage.ErrBucketNotExist {
		return false
	}

	return true
}

// CreateGCSClient creates an authenticated client for GCS
func CreateGCSClient() (*storage.Client, error) {
	ctx := context.Background()

	client, err := storage.NewClient(ctx)
	if err != nil {
		return nil, err
	}

	return client, nil
}

// Custom error types

type MissingRequiredGCSRemoteStateConfig string

func (configName MissingRequiredGCSRemoteStateConfig) Error() string {
	return fmt.Sprintf("Missing required GCS remote state configuration %s", string(configName))
}

type MaxRetriesWaitingForGCSBucketExceeded string

func (err MaxRetriesWaitingForGCSBucketExceeded) Error() string {
	return fmt.Sprintf("Exceeded max retries (%d) waiting for GCS bucket %s", MAX_RETRIES_WAITING_FOR_GCS_BUCKET, string(err))
}

type MaxRetriesWaitingForGCSACLExceeded string

func (err MaxRetriesWaitingForGCSACLExceeded) Error() string {
	return fmt.Sprintf("Exceeded max retries waiting for GCS bucket %s to have proper ACL for access logging", string(err))
}
<|endoftext|>