<commit_before>package haaasd

import (
	"bytes"
	"errors"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"io/ioutil"
	"os"
	"os/exec"
	"time"
)

func NewHaproxy(properties *Config, application string, platform string, version string) *Haproxy {
	if version == "" {
		version = "1.4.22"
	}
	return &Haproxy{
		Application: application,
		Platform:    platform,
		properties:  properties,
		Version:     version,
	}
}

type Haproxy struct {
	Application string
	Platform    string
	Version     string
	properties  *Config
	State       int
}

const (
	SUCCESS int = iota
	UNCHANGED int = iota
	ERR_SYSLOG int = iota
	ERR_CONF int = iota
	ERR_RELOAD int = iota
)

// ApplyConfiguration write the new configuration and reload
// A rollback is called on failure
func (hap *Haproxy) ApplyConfiguration(data *EventMessage) (int, error) {
	hap.createSkeleton(data.Correlationid)

	newConf := data.Conf
	path := hap.confPath()

	// Check conf diff
	oldConf, err := ioutil.ReadFile(path)

	if err != nil {
		log.WithFields(log.Fields{
			"correlationId" : data.Correlationid,
			"path": path,
		}).Error("Cannot read old configuration")
		return ERR_CONF, err
	}
	if log.GetLevel() == log.DebugLevel {
		hap.dumpConfiguration(hap.NewDebugPath(), newConf, data)
	}
	if bytes.Equal(oldConf, newConf) {
		log.WithFields(log.Fields{
			"correlationId": data.Correlationid,
			"application": data.Application,
			"plateform":   data.Platform,
		}).Info("Unchanged configuration")
		return UNCHANGED, nil
	}

	// Archive previous configuration
	archivePath := hap.confArchivePath()
	os.Rename(path, archivePath)
	log.WithFields(
		log.Fields{
			"correlationId": data.Correlationid,
			"application": data.Application,
			"plateform":   data.Platform,
			"archivePath": archivePath,
		}).Info("Old configuration saved")
	err = ioutil.WriteFile(path, newConf, 0644)
	if err != nil {
		return ERR_CONF, err
	}
	log.WithField("path", path).Info("New configuration written")

	// Reload haproxy
	err = hap.reload(data.Correlationid)
	if err != nil {
		log.WithFields(log.Fields{
			"application": data.Application,
			"plateform":   data.Platform,
		}).WithError(err).Error("Reload failed")
		hap.dumpConfiguration(hap.NewErrorPath(), newConf, data)
		err = hap.rollback(data.Correlationid)
		return ERR_RELOAD, err
	}
	// Write syslog fragment
	fragmentPath := hap.syslogFragmentPath()
	err = ioutil.WriteFile(fragmentPath, data.SyslogFragment, 0644)
	if err != nil {
		log.WithFields(log.Fields{
			"application": data.Application,
			"plateform":   data.Platform,
		}).WithError(err).Error("Failed to write syslog fragment")
		// TODO Should we rollback on syslog error ?
		return ERR_SYSLOG, err
	}

	return SUCCESS, nil
}

// dumpConfiguration dumps the new configuration file with context for debugging purpose
func (hap *Haproxy) dumpConfiguration(filename string, newConf []byte, data *EventMessage) {

	f, err2 := os.Create(filename)
	defer f.Close()
	if err2 == nil {
		f.WriteString("================================================================\n")
		f.WriteString(fmt.Sprintf("application: %s\n", data.Application))
		f.WriteString(fmt.Sprintf("platform: %s\n", data.Platform))
		f.WriteString(fmt.Sprintf("correlationid: %s\n", data.Correlationid))
		f.WriteString("================================================================\n")
		f.Write(newConf)
		f.Sync()

		log.WithFields(log.Fields{
			"correlationId": data.Correlationid,
			"filename": filename,
			"application": data.Application,
			"platform": data.Platform,
		}).Info("Dump configuration")
	}
}

// confPath give the path of the configuration file given an application context
// It returns the absolute path to the file
func (hap *Haproxy) confPath() string {
	baseDir := hap.properties.HapHome + "/" + hap.Application + "/Config"
	os.MkdirAll(baseDir, 0755)
	return baseDir + "/hap" + hap.Application + hap.Platform + ".conf"
}

// confPath give the path of the archived configuration file given an application context
func (hap *Haproxy) confArchivePath() string {
	baseDir := hap.properties.HapHome + "/" + hap.Application + "/version-1"
	// It returns the absolute path to the file
	os.MkdirAll(baseDir, 0755)
	return baseDir + "/hap" + hap.Application + hap.Platform + ".conf"
}

// NewErrorPath gives a unique path the error file given the hap context
// It returns the full path to the file
func (hap *Haproxy) NewErrorPath() string {
	baseDir := hap.properties.HapHome + "/" + hap.Application + "/errors"
	os.MkdirAll(baseDir, 0755)
	prefix := time.Now().Format("20060102150405")
	return baseDir + "/" + prefix + "_" + hap.Application + hap.Platform + ".log"
}

func (hap *Haproxy) NewDebugPath() string {
	baseDir := hap.properties.HapHome + "/" + hap.Application + "/dump"
	os.MkdirAll(baseDir, 0755)
	prefix := time.Now().Format("20060102150405")
	return baseDir + "/" + prefix + "_" + hap.Application + hap.Platform + ".log"
}

// reload calls external shell script to reload haproxy
// It returns error if the reload fails
func (hap *Haproxy) reload(correlationId string) error {

	reloadScript := hap.getReloadScript()
	cmd, err := exec.Command("sh", reloadScript, "reload", "-y").Output()
	if err != nil {
		log.WithError(err).Error("Error reloading")
	}
	log.WithFields(log.Fields{
		"correlationId" : correlationId,
		"application": hap.Application,
		"platform": hap.Platform,
		"reloadScript": reloadScript,
	}).WithField("cmd", cmd).Debug("Reload succeeded")
	return err
}

// rollbac reverts configuration files and call for reload
func (hap *Haproxy) rollback(correlationId string) error {
	lastConf := hap.confArchivePath()
	if _, err := os.Stat(lastConf); os.IsNotExist(err) {
		return errors.New("No configuration file to rollback")
	}
	os.Rename(lastConf, hap.confPath())
	hap.reload(correlationId)
	return nil
}

// createSkeleton creates the directory tree for a new haproxy context
func (hap *Haproxy) createSkeleton(correlationId string) error {
	baseDir := hap.properties.HapHome + "/" + hap.Application

	createDirectory(correlationId, baseDir + "/Config")
	createDirectory(correlationId, baseDir + "/logs/" + hap.Application + hap.Platform)
	createDirectory(correlationId, baseDir + "/scripts")
	createDirectory(correlationId, baseDir + "/version-1")

	updateSymlink(correlationId, hap.getHapctlFilename(), hap.getReloadScript())
	updateSymlink(correlationId, hap.getHapBinary(), baseDir + "/Config/haproxy")

	return nil
}

// confPath give the path of the configuration file given an application context
// It returns the absolute path to the file
func (hap *Haproxy) syslogFragmentPath() string {
	baseDir := hap.properties.HapHome + "/SYSLOG/Config/syslog.conf.d"
	os.MkdirAll(baseDir, 0755)
	return baseDir + "/syslog" + hap.Application + hap.Platform + ".conf"
}

// updateSymlink create or update a symlink
func updateSymlink(correlationId, oldname string, newname string) {
	if _, err := os.Stat(newname); err == nil {
		os.Remove(newname)
	}
	err := os.Symlink(oldname, newname)
	if err != nil {
		log.WithError(err).WithFields(log.Fields{
			"correlationId" : correlationId,
			"path": newname,
		}).Error("Symlink failed")
	} else {
		log.WithFields(log.Fields{
			"correlationId" : correlationId,
			"path": newname,
		}).Info("Symlink created")
	}
}

// createDirectory recursively creates directory if it doesn't exists
func createDirectory(correlationId string, dir string) {
	if _, err := os.Stat(dir); os.IsNotExist(err) {
		err := os.MkdirAll(dir, 0755)
		if err != nil {
			log.WithError(err).WithFields(log.Fields{
				"correlationId" : correlationId,
				"dir": dir,
			}).Error("Failed to create")
		} else {
			log.WithFields(log.Fields{
				"correlationId" : correlationId,
				"dir": dir,
			}).Info("Directory created")
		}
	}
}

// getHapctlFilename return the path to the vsc hapctl shell script
// This script is provided
func (hap *Haproxy) getHapctlFilename() string {
	return "/HOME/uxwadm/scripts/hapctl_unif"
}

// getReloadScript calculates reload script path given the hap context
// It returns the full script path
func (hap *Haproxy) getReloadScript() string {
	return fmt.Sprintf("%s/%s/scripts/hapctl%s%s", hap.properties.HapHome, hap.Application, hap.Application, hap.Platform)
}

// getHapBinary calculates the haproxy binary to use given the expected version
// It returns the full path to the haproxy binary
func (hap *Haproxy) getHapBinary() string {
	return fmt.Sprintf("/export/product/haproxy/product/%s/bin/haproxy", hap.Version)
}
<commit_msg>Update logs<commit_after>package haaasd

import (
	"bytes"
	"errors"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"io/ioutil"
	"os"
	"os/exec"
	"time"
)

func NewHaproxy(properties *Config, application string, platform string, version string) *Haproxy {
	if version == "" {
		version = "1.4.22"
	}
	return &Haproxy{
		Application: application,
		Platform:    platform,
		properties:  properties,
		Version:     version,
	}
}

type Haproxy struct {
	Application string
	Platform    string
	Version     string
	properties  *Config
	State       int
}

const (
	SUCCESS int = iota
	UNCHANGED int = iota
	ERR_SYSLOG int = iota
	ERR_CONF int = iota
	ERR_RELOAD int = iota
)

// ApplyConfiguration write the new configuration and reload
// A rollback is called on failure
func (hap *Haproxy) ApplyConfiguration(data *EventMessage) (int, error) {
	hap.createSkeleton(data.Correlationid)

	newConf := data.Conf
	path := hap.confPath()

	// Check conf diff
	oldConf, err := ioutil.ReadFile(path)

	if err != nil {
		log.WithFields(log.Fields{
			"correlationId" : data.Correlationid,
			"path": path,
		}).Error("Cannot read old configuration")
		return ERR_CONF, err
	}
	if log.GetLevel() == log.DebugLevel {
		hap.dumpConfiguration(hap.NewDebugPath(), newConf, data)
	}
	if bytes.Equal(oldConf, newConf) {
		log.WithFields(log.Fields{
			"correlationId": data.Correlationid,
			"application": data.Application,
			"plateform":   data.Platform,
		}).Info("Unchanged configuration")
		return UNCHANGED, nil
	}

	// Archive previous configuration
	archivePath := hap.confArchivePath()
	os.Rename(path, archivePath)
	log.WithFields(
		log.Fields{
			"correlationId": data.Correlationid,
			"application": data.Application,
			"plateform":   data.Platform,
			"archivePath": archivePath,
		}).Info("Old configuration saved")
	err = ioutil.WriteFile(path, newConf, 0644)
	if err != nil {
		return ERR_CONF, err
	}
	log.WithField("path", path).Info("New configuration written")

	// Reload haproxy
	err = hap.reload(data.Correlationid)
	if err != nil {
		log.WithFields(log.Fields{
			"application": data.Application,
			"plateform":   data.Platform,
		}).WithError(err).Error("Reload failed")
		hap.dumpConfiguration(hap.NewErrorPath(), newConf, data)
		err = hap.rollback(data.Correlationid)
		return ERR_RELOAD, err
	}
	// Write syslog fragment
	fragmentPath := hap.syslogFragmentPath()
	err = ioutil.WriteFile(fragmentPath, data.SyslogFragment, 0644)
	if err != nil {
		log.WithFields(log.Fields{
			"application": data.Application,
			"plateform":   data.Platform,
		}).WithError(err).Error("Failed to write syslog fragment")
		// TODO Should we rollback on syslog error ?
		return ERR_SYSLOG, err
	}

	return SUCCESS, nil
}

// dumpConfiguration dumps the new configuration file with context for debugging purpose
func (hap *Haproxy) dumpConfiguration(filename string, newConf []byte, data *EventMessage) {

	f, err2 := os.Create(filename)
	defer f.Close()
	if err2 == nil {
		f.WriteString("================================================================\n")
		f.WriteString(fmt.Sprintf("application: %s\n", data.Application))
		f.WriteString(fmt.Sprintf("platform: %s\n", data.Platform))
		f.WriteString(fmt.Sprintf("correlationid: %s\n", data.Correlationid))
		f.WriteString("================================================================\n")
		f.Write(newConf)
		f.Sync()

		log.WithFields(log.Fields{
			"correlationId": data.Correlationid,
			"filename": filename,
			"application": data.Application,
			"platform": data.Platform,
		}).Info("Dump configuration")
	}
}

// confPath give the path of the configuration file given an application context
// It returns the absolute path to the file
func (hap *Haproxy) confPath() string {
	baseDir := hap.properties.HapHome + "/" + hap.Application + "/Config"
	os.MkdirAll(baseDir, 0755)
	return baseDir + "/hap" + hap.Application + hap.Platform + ".conf"
}

// confPath give the path of the archived configuration file given an application context
func (hap *Haproxy) confArchivePath() string {
	baseDir := hap.properties.HapHome + "/" + hap.Application + "/version-1"
	// It returns the absolute path to the file
	os.MkdirAll(baseDir, 0755)
	return baseDir + "/hap" + hap.Application + hap.Platform + ".conf"
}

// NewErrorPath gives a unique path the error file given the hap context
// It returns the full path to the file
func (hap *Haproxy) NewErrorPath() string {
	baseDir := hap.properties.HapHome + "/" + hap.Application + "/errors"
	os.MkdirAll(baseDir, 0755)
	prefix := time.Now().Format("20060102150405")
	return baseDir + "/" + prefix + "_" + hap.Application + hap.Platform + ".log"
}

func (hap *Haproxy) NewDebugPath() string {
	baseDir := hap.properties.HapHome + "/" + hap.Application + "/dump"
	os.MkdirAll(baseDir, 0755)
	prefix := time.Now().Format("20060102150405")
	return baseDir + "/" + prefix + "_" + hap.Application + hap.Platform + ".log"
}

// reload calls external shell script to reload haproxy
// It returns error if the reload fails
func (hap *Haproxy) reload(correlationId string) error {

	reloadScript := hap.getReloadScript()
	cmd, err := exec.Command("sh", reloadScript, "reload", "-y").Output()
	if err != nil {
		log.WithError(err).Error("Error reloading")
	}
	log.WithFields(log.Fields{
		"correlationId" : correlationId,
		"application": hap.Application,
		"platform": hap.Platform,
		"reloadScript": reloadScript,
	}).WithField("cmd", cmd).Debug("Reload succeeded")
	return err
}

// rollbac reverts configuration files and call for reload
func (hap *Haproxy) rollback(correlationId string) error {
	lastConf := hap.confArchivePath()
	if _, err := os.Stat(lastConf); os.IsNotExist(err) {
		return errors.New("No configuration file to rollback")
	}
	os.Rename(lastConf, hap.confPath())
	hap.reload(correlationId)
	return nil
}

// createSkeleton creates the directory tree for a new haproxy context
func (hap *Haproxy) createSkeleton(correlationId string) error {
	baseDir := hap.properties.HapHome + "/" + hap.Application

	createDirectory(correlationId, baseDir + "/Config")
	createDirectory(correlationId, baseDir + "/logs/" + hap.Application + hap.Platform)
	createDirectory(correlationId, baseDir + "/scripts")
	createDirectory(correlationId, baseDir + "/version-1")

	updateSymlink(correlationId, hap.getHapctlFilename(), hap.getReloadScript())
	updateSymlink(correlationId, hap.getHapBinary(), baseDir + "/Config/haproxy")

	return nil
}

// confPath give the path of the configuration file given an application context
// It returns the absolute path to the file
func (hap *Haproxy) syslogFragmentPath() string {
	baseDir := hap.properties.HapHome + "/SYSLOG/Config/syslog.conf.d"
	os.MkdirAll(baseDir, 0755)
	return baseDir + "/syslog" + hap.Application + hap.Platform + ".conf"
}

// updateSymlink create or update a symlink
func updateSymlink(correlationId, oldname string, newname string) {
	newLink := true
	if _, err := os.Stat(newname); err == nil {
		os.Remove(newname)
		newLink=false
	}
	err := os.Symlink(oldname, newname)
	if err != nil {
		log.WithError(err).WithFields(log.Fields{
			"correlationId" : correlationId,
			"path": newname,
		}).Error("Symlink failed")
	}

	if newLink{
		log.WithFields(log.Fields{
			"correlationId" : correlationId,
			"path": newname,
		}).Info("Symlink created")
	}
}

// createDirectory recursively creates directory if it doesn't exists
func createDirectory(correlationId string, dir string) {
	if _, err := os.Stat(dir); os.IsNotExist(err) {
		err := os.MkdirAll(dir, 0755)
		if err != nil {
			log.WithError(err).WithFields(log.Fields{
				"correlationId" : correlationId,
				"dir": dir,
			}).Error("Failed to create")
		} else {
			log.WithFields(log.Fields{
				"correlationId" : correlationId,
				"dir": dir,
			}).Info("Directory created")
		}
	}
}

// getHapctlFilename return the path to the vsc hapctl shell script
// This script is provided
func (hap *Haproxy) getHapctlFilename() string {
	return "/HOME/uxwadm/scripts/hapctl_unif"
}

// getReloadScript calculates reload script path given the hap context
// It returns the full script path
func (hap *Haproxy) getReloadScript() string {
	return fmt.Sprintf("%s/%s/scripts/hapctl%s%s", hap.properties.HapHome, hap.Application, hap.Application, hap.Platform)
}

// getHapBinary calculates the haproxy binary to use given the expected version
// It returns the full path to the haproxy binary
func (hap *Haproxy) getHapBinary() string {
	return fmt.Sprintf("/export/product/haproxy/product/%s/bin/haproxy", hap.Version)
}
<|endoftext|>


<commit_before>package helper

import (
	"compress/gzip"
	"crypto/tls"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"

	"github.com/dustin/go-humanize"
)

type FetchConfig struct {
	URL          string
	Checksum     string
	ChecksumType string
	DownloadPath string
}

func FetchFile(config FetchConfig) (string, error) {
	if config.URL == "" {
		panic("URL is required")
	}

	if config.Checksum == "" {
		panic("Checksum is required")
	}

	if config.ChecksumType == "" {
		panic("Checksum type is required")
	}

	if config.DownloadPath == "" {
		config.DownloadPath = os.TempDir()
	}

	u, err := url.Parse(config.URL)
	if err != nil {
		return "", err
	}

	_, filename := path.Split(u.Path)
	if filename == "" {
		filename = "unnamed"
	}

	os.MkdirAll(config.DownloadPath, 0740)

	filePath := filepath.Join(config.DownloadPath, filename)

	unpack := false
	vmPath := config.DownloadPath

	log.Printf("[DEBUG] Opening %s...", filePath)
	file, err := os.Open(filePath)
	if err != nil {
		log.Printf("[DEBUG] %s file does not exist. Downloading it...", filename)

		data, err := download(config.URL)
		if err != nil {
			return "", err
		}

		file, err = write(data, filePath)
		if err != nil {
			return "", err
		}
		data.Close()

		unpack = true
	}

	// We need to make sure the reader is pointing to the beginning of the file
	// so verifying integrity does not fail
	file.Seek(0, 0)

	if err = VerifyChecksum(file, config.ChecksumType, config.Checksum); err != nil {
		log.Printf("[DEBUG] File on disk does not match current checksum.\n Downloading file again...")

		data, err := download(config.URL)
		if err != nil {
			return "", err
		}

		file, err = write(data, filePath)
		if err != nil {
			return "", err
		}
		data.Close()

		file.Seek(0, 0)
		if err = VerifyChecksum(file, config.ChecksumType, config.Checksum); err != nil {
			return "", err
		}

		unpack = true
	}
	defer file.Close()

	// Only unpacks file if checksum changed or file does not exists
	if unpack {
		// TODO(c4milo): Make sure the file is a tgz file before attempting
		// to unpack it.
		_, err = UnpackFile(file, vmPath)
		if err != nil {
			return "", err
		}
	}

	return vmPath, nil
}

func write(reader io.Reader, filePath string) (*os.File, error) {
	log.Printf("[DEBUG] Downloading file data to %s", filePath)

	gzfile, err := os.Create(filePath)
	if err != nil {
		return nil, err
	}

	written, err := io.Copy(gzfile, reader)
	if err != nil {
		return nil, err
	}
	log.Printf("[DEBUG] %s written to %s", humanize.Bytes(uint64(written)), filePath)

	return gzfile, nil
}

func download(URL string) (io.ReadCloser, error) {
	client := &http.Client{
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: false,
			},
		},
	}

	resp, err := client.Get(URL)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("Unable to fetch data, server returned code %d", resp.StatusCode)
	}

	return resp.Body, nil
}

func UnpackFile(file *os.File, destPath string) (string, error) {
	os.MkdirAll(destPath, 0740)

	//unzip
	log.Printf("[DEBUG] Unzipping file stream ...")
	file.Seek(0, 0)

	unzippedFile, err := gzip.NewReader(file)
	if err != nil && err != io.EOF {
		return "", err
	}
	defer unzippedFile.Close()

	//untar
	return Untar(unzippedFile, destPath)
}
<commit_msg>Fixes issue when destroying and subsequently creating the resource again<commit_after>package helper

import (
	"compress/gzip"
	"crypto/tls"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"

	"github.com/dustin/go-humanize"
)

type FetchConfig struct {
	URL          string
	Checksum     string
	ChecksumType string
	DownloadPath string
}

func FetchFile(config FetchConfig) (string, error) {
	if config.URL == "" {
		panic("URL is required")
	}

	if config.Checksum == "" {
		panic("Checksum is required")
	}

	if config.ChecksumType == "" {
		panic("Checksum type is required")
	}

	if config.DownloadPath == "" {
		config.DownloadPath = os.TempDir()
	}

	u, err := url.Parse(config.URL)
	if err != nil {
		return "", err
	}

	_, filename := path.Split(u.Path)
	if filename == "" {
		filename = "unnamed"
	}

	os.MkdirAll(config.DownloadPath, 0740)

	filePath := filepath.Join(config.DownloadPath, filename)
	vmPath := filepath.Join(config.DownloadPath, config.Checksum)

	log.Printf("[DEBUG] Opening %s...", filePath)
	file, err := os.Open(filePath)
	if err != nil {
		log.Printf("[DEBUG] %s file does not exist. Downloading it...", filename)

		data, err := download(config.URL)
		if err != nil {
			return "", err
		}

		file, err = write(data, filePath)
		if err != nil {
			return "", err
		}
		data.Close()
	}
	defer file.Close()

	// We need to make sure the reader is pointing to the beginning of the file
	// so verifying integrity does not fail
	file.Seek(0, 0)

	if err = VerifyChecksum(file, config.ChecksumType, config.Checksum); err != nil {
		log.Printf("[DEBUG] File on disk does not match current checksum.\n Downloading file again...")

		data, err := download(config.URL)
		if err != nil {
			return "", err
		}

		file, err = write(data, filePath)
		if err != nil {
			return "", err
		}
		data.Close()

		file.Seek(0, 0)
		if err = VerifyChecksum(file, config.ChecksumType, config.Checksum); err != nil {
			return "", err
		}
	}

	// If an unpacked VM folder does not exist or is empty then unpack image.
	_, err = os.Stat(vmPath)
	vmPathExist := err != nil && os.IsNotExist(err)

	// There is no need to get the error as the slice will be empty anyways
	finfo, _ := ioutil.ReadDir(vmPath)
	vmPathEmpty := len(finfo) == 0

	if !vmPathExist || vmPathEmpty {
		// TODO(c4milo): Make sure the file is a tgz file before attempting
		// to unpack it.
		_, err = UnpackFile(file, vmPath)
		if err != nil {
			return "", err
		}
	}

	return vmPath, nil
}

func write(reader io.Reader, filePath string) (*os.File, error) {
	log.Printf("[DEBUG] Downloading file data to %s", filePath)

	gzfile, err := os.Create(filePath)
	if err != nil {
		return nil, err
	}

	written, err := io.Copy(gzfile, reader)
	if err != nil {
		return nil, err
	}
	log.Printf("[DEBUG] %s written to %s", humanize.Bytes(uint64(written)), filePath)

	return gzfile, nil
}

func download(URL string) (io.ReadCloser, error) {
	client := &http.Client{
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: false,
			},
		},
	}

	resp, err := client.Get(URL)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("Unable to fetch data, server returned code %d", resp.StatusCode)
	}

	return resp.Body, nil
}

func UnpackFile(file *os.File, destPath string) (string, error) {
	os.MkdirAll(destPath, 0740)

	//unzip
	log.Printf("[DEBUG] Unzipping file stream ...")
	file.Seek(0, 0)

	unzippedFile, err := gzip.NewReader(file)
	if err != nil && err != io.EOF {
		return "", err
	}
	defer unzippedFile.Close()

	//untar
	return Untar(unzippedFile, destPath)
}
<|endoftext|>


<commit_before>package configstack

import (
	"bytes"
	"fmt"
	"sort"
	"strings"

	"github.com/gruntwork-io/terragrunt/config"
	"github.com/gruntwork-io/terragrunt/errors"
	"github.com/gruntwork-io/terragrunt/options"
	"github.com/gruntwork-io/terragrunt/util"
)

// Represents a stack of Terraform modules (i.e. folders with Terraform templates) that you can "spin up" or
// "spin down" in a single command
type Stack struct {
	Path    string
	Modules []*TerraformModule
}

// Render this stack as a human-readable string
func (stack *Stack) String() string {
	modules := []string{}
	for _, module := range stack.Modules {
		modules = append(modules, fmt.Sprintf("  => %s", module.String()))
	}
	sort.Strings(modules)
	return fmt.Sprintf("Stack at %s:\n%s", stack.Path, strings.Join(modules, "\n"))
}

// Graph creates a graphviz representation of the modules
func (stack *Stack) Graph(terragruntOptions *options.TerragruntOptions) {
	WriteDot(terragruntOptions.Writer, terragruntOptions, stack.Modules)
}

func (stack *Stack) Run(terragruntOptions *options.TerragruntOptions) error {
	stackCmd := terragruntOptions.TerraformCommand

	// For any command that needs input, run in non-interactive mode to avoid cominglint stdin across multiple
	// concurrent runs.
	if util.ListContainsElement(config.TERRAFORM_COMMANDS_NEED_INPUT, stackCmd) {
		// to support potential positional args in the args list, we append the input=false arg after the first element,
		// which is the target command.
		terragruntOptions.TerraformCliArgs = util.StringListInsert(terragruntOptions.TerraformCliArgs, "-input=false", 1)
		stack.syncTerraformCliArgs(terragruntOptions)
	}

	// For apply and destroy, run with auto-approve due to the co-mingling of the prompts. This is not ideal, but until
	// we have a better way of handling interactivity with run-all, we take the evil of having a global prompt (managed
	// in cli/cli_app.go) be the gate keeper.
	switch stackCmd {
	case "apply", "destroy":
		// to support potential positional args in the args list, we append the input=false arg after the first element,
		// which is the target command.
		terragruntOptions.TerraformCliArgs = util.StringListInsert(terragruntOptions.TerraformCliArgs, "-auto-approve", 1)
		stack.syncTerraformCliArgs(terragruntOptions)
	}

	if stackCmd == "plan" {
		// We capture the out stream for each module
		errorStreams := make([]bytes.Buffer, len(stack.Modules))
		for n, module := range stack.Modules {
			module.TerragruntOptions.ErrWriter = &errorStreams[n]
		}
		defer stack.summarizePlanAllErrors(terragruntOptions, errorStreams)
	}

	if terragruntOptions.IgnoreDependencyOrder {
		return RunModulesIgnoreOrder(stack.Modules, terragruntOptions.Parallelism)
	} else if stackCmd == "destroy" {
		return RunModulesReverseOrder(stack.Modules, terragruntOptions.Parallelism)
	} else {
		return RunModules(stack.Modules, terragruntOptions.Parallelism)
	}
}

// We inspect the error streams to give an explicit message if the plan failed because there were references to
// remote states. `terraform plan` will fail if it tries to access remote state from dependencies and the plan
// has never been applied on the dependency.
func (stack *Stack) summarizePlanAllErrors(terragruntOptions *options.TerragruntOptions, errorStreams []bytes.Buffer) {
	for i, errorStream := range errorStreams {
		output := errorStream.String()
		terragruntOptions.Logger.Infoln(output)
		if strings.Contains(output, "Error running plan:") {
			if strings.Contains(output, ": Resource 'data.terraform_remote_state.") {
				var dependenciesMsg string
				if len(stack.Modules[i].Dependencies) > 0 {
					dependenciesMsg = fmt.Sprintf(" contains dependencies to %v and", stack.Modules[i].Config.Dependencies.Paths)
				}
				terragruntOptions.Logger.Infof("%v%v refers to remote state "+
					"you may have to apply your changes in the dependencies prior running terragrunt plan-all.\n",
					stack.Modules[i].Path,
					dependenciesMsg,
				)
			}
		}
	}
}

// Return an error if there is a dependency cycle in the modules of this stack.
func (stack *Stack) CheckForCycles() error {
	return CheckForCycles(stack.Modules)
}

// Find all the Terraform modules in the subfolders of the working directory of the given TerragruntOptions and
// assemble them into a Stack object that can be applied or destroyed in a single command
func FindStackInSubfolders(terragruntOptions *options.TerragruntOptions) (*Stack, error) {
	terragruntConfigFiles, err := config.FindConfigFilesInPath(terragruntOptions.WorkingDir, terragruntOptions)
	if err != nil {
		return nil, err
	}

	howThesePathsWereFound := fmt.Sprintf("Terragrunt config file found in a subdirectory of %s", terragruntOptions.WorkingDir)
	return createStackForTerragruntConfigPaths(terragruntOptions.WorkingDir, terragruntConfigFiles, terragruntOptions, howThesePathsWereFound)
}

// Sync the TerraformCliArgs for each module in the stack to match the provided terragruntOptions struct.
func (stack *Stack) syncTerraformCliArgs(terragruntOptions *options.TerragruntOptions) {
	for _, module := range stack.Modules {
		module.TerragruntOptions.TerraformCliArgs = terragruntOptions.TerraformCliArgs
	}
}

// Find all the Terraform modules in the folders that contain the given Terragrunt config files and assemble those
// modules into a Stack object that can be applied or destroyed in a single command
func createStackForTerragruntConfigPaths(path string, terragruntConfigPaths []string, terragruntOptions *options.TerragruntOptions, howThesePathsWereFound string) (*Stack, error) {
	if len(terragruntConfigPaths) == 0 {
		return nil, errors.WithStackTrace(NoTerraformModulesFound)
	}

	modules, err := ResolveTerraformModules(terragruntConfigPaths, terragruntOptions, howThesePathsWereFound)
	if err != nil {
		return nil, err
	}

	stack := &Stack{Path: path, Modules: modules}
	if err := stack.CheckForCycles(); err != nil {
		return nil, err
	}

	return stack, nil
}

// Custom error types

var NoTerraformModulesFound = fmt.Errorf("Could not find any subfolders with Terragrunt configuration files")

type DependencyCycle []string

func (err DependencyCycle) Error() string {
	return fmt.Sprintf("Found a dependency cycle between modules: %s", strings.Join([]string(err), " -> "))
}
<commit_msg>Fix empty outputs (#1568)<commit_after>package configstack

import (
	"bytes"
	"fmt"
	"sort"
	"strings"

	"github.com/gruntwork-io/terragrunt/config"
	"github.com/gruntwork-io/terragrunt/errors"
	"github.com/gruntwork-io/terragrunt/options"
	"github.com/gruntwork-io/terragrunt/util"
)

// Represents a stack of Terraform modules (i.e. folders with Terraform templates) that you can "spin up" or
// "spin down" in a single command
type Stack struct {
	Path    string
	Modules []*TerraformModule
}

// Render this stack as a human-readable string
func (stack *Stack) String() string {
	modules := []string{}
	for _, module := range stack.Modules {
		modules = append(modules, fmt.Sprintf("  => %s", module.String()))
	}
	sort.Strings(modules)
	return fmt.Sprintf("Stack at %s:\n%s", stack.Path, strings.Join(modules, "\n"))
}

// Graph creates a graphviz representation of the modules
func (stack *Stack) Graph(terragruntOptions *options.TerragruntOptions) {
	WriteDot(terragruntOptions.Writer, terragruntOptions, stack.Modules)
}

func (stack *Stack) Run(terragruntOptions *options.TerragruntOptions) error {
	stackCmd := terragruntOptions.TerraformCommand

	// For any command that needs input, run in non-interactive mode to avoid cominglint stdin across multiple
	// concurrent runs.
	if util.ListContainsElement(config.TERRAFORM_COMMANDS_NEED_INPUT, stackCmd) {
		// to support potential positional args in the args list, we append the input=false arg after the first element,
		// which is the target command.
		terragruntOptions.TerraformCliArgs = util.StringListInsert(terragruntOptions.TerraformCliArgs, "-input=false", 1)
		stack.syncTerraformCliArgs(terragruntOptions)
	}

	// For apply and destroy, run with auto-approve due to the co-mingling of the prompts. This is not ideal, but until
	// we have a better way of handling interactivity with run-all, we take the evil of having a global prompt (managed
	// in cli/cli_app.go) be the gate keeper.
	switch stackCmd {
	case "apply", "destroy":
		// to support potential positional args in the args list, we append the input=false arg after the first element,
		// which is the target command.
		terragruntOptions.TerraformCliArgs = util.StringListInsert(terragruntOptions.TerraformCliArgs, "-auto-approve", 1)
		stack.syncTerraformCliArgs(terragruntOptions)
	}

	if stackCmd == "plan" {
		// We capture the out stream for each module
		errorStreams := make([]bytes.Buffer, len(stack.Modules))
		for n, module := range stack.Modules {
			module.TerragruntOptions.ErrWriter = &errorStreams[n]
		}
		defer stack.summarizePlanAllErrors(terragruntOptions, errorStreams)
	}

	if terragruntOptions.IgnoreDependencyOrder {
		return RunModulesIgnoreOrder(stack.Modules, terragruntOptions.Parallelism)
	} else if stackCmd == "destroy" {
		return RunModulesReverseOrder(stack.Modules, terragruntOptions.Parallelism)
	} else {
		return RunModules(stack.Modules, terragruntOptions.Parallelism)
	}
}

// We inspect the error streams to give an explicit message if the plan failed because there were references to
// remote states. `terraform plan` will fail if it tries to access remote state from dependencies and the plan
// has never been applied on the dependency.
func (stack *Stack) summarizePlanAllErrors(terragruntOptions *options.TerragruntOptions, errorStreams []bytes.Buffer) {
	for i, errorStream := range errorStreams {
		output := errorStream.String()

		if len(output) == 0 {
			// We get empty buffer if stack execution completed without errors, so skip that to avoid logging too much
			continue
		}

		terragruntOptions.Logger.Infoln(output)
		if strings.Contains(output, "Error running plan:") {
			if strings.Contains(output, ": Resource 'data.terraform_remote_state.") {
				var dependenciesMsg string
				if len(stack.Modules[i].Dependencies) > 0 {
					dependenciesMsg = fmt.Sprintf(" contains dependencies to %v and", stack.Modules[i].Config.Dependencies.Paths)
				}
				terragruntOptions.Logger.Infof("%v%v refers to remote state "+
					"you may have to apply your changes in the dependencies prior running terragrunt plan-all.\n",
					stack.Modules[i].Path,
					dependenciesMsg,
				)
			}
		}
	}
}

// Return an error if there is a dependency cycle in the modules of this stack.
func (stack *Stack) CheckForCycles() error {
	return CheckForCycles(stack.Modules)
}

// Find all the Terraform modules in the subfolders of the working directory of the given TerragruntOptions and
// assemble them into a Stack object that can be applied or destroyed in a single command
func FindStackInSubfolders(terragruntOptions *options.TerragruntOptions) (*Stack, error) {
	terragruntConfigFiles, err := config.FindConfigFilesInPath(terragruntOptions.WorkingDir, terragruntOptions)
	if err != nil {
		return nil, err
	}

	howThesePathsWereFound := fmt.Sprintf("Terragrunt config file found in a subdirectory of %s", terragruntOptions.WorkingDir)
	return createStackForTerragruntConfigPaths(terragruntOptions.WorkingDir, terragruntConfigFiles, terragruntOptions, howThesePathsWereFound)
}

// Sync the TerraformCliArgs for each module in the stack to match the provided terragruntOptions struct.
func (stack *Stack) syncTerraformCliArgs(terragruntOptions *options.TerragruntOptions) {
	for _, module := range stack.Modules {
		module.TerragruntOptions.TerraformCliArgs = terragruntOptions.TerraformCliArgs
	}
}

// Find all the Terraform modules in the folders that contain the given Terragrunt config files and assemble those
// modules into a Stack object that can be applied or destroyed in a single command
func createStackForTerragruntConfigPaths(path string, terragruntConfigPaths []string, terragruntOptions *options.TerragruntOptions, howThesePathsWereFound string) (*Stack, error) {
	if len(terragruntConfigPaths) == 0 {
		return nil, errors.WithStackTrace(NoTerraformModulesFound)
	}

	modules, err := ResolveTerraformModules(terragruntConfigPaths, terragruntOptions, howThesePathsWereFound)
	if err != nil {
		return nil, err
	}

	stack := &Stack{Path: path, Modules: modules}
	if err := stack.CheckForCycles(); err != nil {
		return nil, err
	}

	return stack, nil
}

// Custom error types

var NoTerraformModulesFound = fmt.Errorf("Could not find any subfolders with Terragrunt configuration files")

type DependencyCycle []string

func (err DependencyCycle) Error() string {
	return fmt.Sprintf("Found a dependency cycle between modules: %s", strings.Join([]string(err), " -> "))
}
<|endoftext|>


<commit_before>package consultant_test

import (
	"fmt"
	"github.com/hashicorp/consul/api"
	"github.com/hashicorp/consul/testutil"
	"github.com/myENA/consultant"
	"github.com/stretchr/testify/require"
	"github.com/stretchr/testify/suite"
	"strconv"
	"sync"
	"testing"
	"time"
)

const (
	prefix = "test/"
	key1   = "key1"
	val1   = "value 1"
	val1b  = "value 1 after change"
	key2   = "key2"
	val2   = 2
	val2b  = 42
)

func TestConfigurator(t *testing.T) {
	suite.Run(t, &ConfiguratorTestSuite{})
}

// Implement the Configurator interface
type config struct {
	var1 string
	var2 int
	t    *testing.T
	sync.RWMutex
}

func (c *config) Update(_ uint64, data interface{}) {

	var err error

	c.Lock()
	defer c.Unlock()

	switch data.(type) {

	case api.KVPairs:
		kvps := data.(api.KVPairs)
		c.t.Logf("Update received %d KV pairs", len(kvps))
		for _, kvp := range kvps {
			c.t.Logf("key=%s, val=%s", kvp.Key, kvp.Value)
			switch kvp.Key {
			case prefix + key1:
				c.var1 = string(kvp.Value)
				c.t.Logf("c.var1=%s", c.var1)
			case prefix + key2:
				c.var2, err = strconv.Atoi(string(kvp.Value))
				if err != nil {
					c.t.Logf("key %s is not an int", key2)
				}
			}
		}

	case []*api.ServiceEntry:
		// nothing here yet

	default:
		c.t.Log("Typecast failed")
		c.t.Fail()
	}
}

// 1. Test that the consul KV config is transferred correctly
// 2. Verify that an update is reflected in the config object
func (cs *ConfiguratorTestSuite) TestKVInit() {
	var err error

	cs.buildKVTestData()

	config := &config{
		t: cs.T(),
	}

	_, err = cs.client.InitConfigurator(config, prefix)

	require.Nil(cs.T(), err, "InitConfigurator(..., %s) failed: %s", prefix, err)

	// Check that config has what we expect
	require.Equal(cs.T(), val1, config.var1, "the initialized val1 is not what I expected")
	require.Equal(cs.T(), val2, config.var2, "the initialized val2 is not what I expected")

	kv1 := &api.KVPair{Key: prefix + key1, Value: []byte(val1b)}
	_, err = cs.client.KV().Put(kv1, nil)
	require.Nil(cs.T(), err, "Trouble changing the value of %s", key1)
	time.Sleep(time.Second)
	// require.Equal(cs.T(), val1b, config.var1, "var1 is not what i expected after updating in consul")
	cs.T().Logf("var1 after update=%s",config.var1)

	kv2 := &api.KVPair{Key: prefix + key2, Value: []byte(fmt.Sprintf("%d", val2b))}
	_, err = cs.client.KV().Put(kv2, nil)
	require.Nil(cs.T(), err, "Trouble changing the value of %s", key2)
	time.Sleep(time.Second)
	// require.Equal(cs.T(), val2b, config.var2, "var2 is not what i expected after updating in consul")
	cs.T().Logf("var2 after update=%d",config.var2)

	time.Sleep(5*time.Second)

	// report what is actually in the kv prefix now:
	kvps,_,err := cs.client.KV().List(prefix, nil)
	config.Update(0,kvps)
	cs.T().Logf("config after manual update: %+v",config)
}

func (cs *ConfiguratorTestSuite) buildKVTestData() {
	var err error

	kv1 := &api.KVPair{Key: prefix + key1, Value: []byte(val1)}
	_, err = cs.client.KV().Put(kv1, nil)
	require.Nil(cs.T(), err, "Failed storing key1/val1: %s", err)

	kv2 := &api.KVPair{Key: prefix + key2, Value: []byte(fmt.Sprintf("%d", val2))}
	_, err = cs.client.KV().Put(kv2, nil)
	require.Nil(cs.T(), err, "Failed storing key2/val2: %s", err)
}

type ConfiguratorTestSuite struct {
	suite.Suite

	// these values are cyclical, and should be re-defined per test method
	server *testutil.TestServer
	client *consultant.Client
}

// SetupTest is called before each method is run.
func (cs *ConfiguratorTestSuite) SetupTest() {
	cs.server, cs.client = makeServerAndClient(cs.T(), nil)
}

// TearDownTest is called after each method has been run.
func (cs *ConfiguratorTestSuite) TearDownTest() {
	if nil != cs.client {
		cs.client = nil
	}
	if nil != cs.server {
		// TODO: Stop seems to return an error when the process is killed...
		cs.server.Stop()
		cs.server = nil
	}
}

func (cs *ConfiguratorTestSuite) TearDownSuite() {
	cs.TearDownTest()
}

<commit_msg>cleaning up tests<commit_after>package consultant_test

import (
	"fmt"
	"github.com/hashicorp/consul/api"
	"github.com/hashicorp/consul/testutil"
	"github.com/myENA/consultant"
	"github.com/hashicorp/consul/watch"
	"github.com/stretchr/testify/require"
	"github.com/stretchr/testify/suite"
	"strconv"
	"sync"
	"testing"
	"time"
)

const (
	prefix = "test/"
	key1   = "key1"
	val1   = "value 1"
	val1b  = "value 1 after change"
	key2   = "key2"
	val2   = 2
	val2b  = 42
)

func TestConfigurator(t *testing.T) {
	suite.Run(t, &ConfiguratorTestSuite{})
}

// Implement the Configurator interface
type config struct {
	var1 string
	var2 int
	t    *testing.T
	sync.RWMutex
}

func (c *config) Update(_ uint64, data interface{}) {

	var err error

	c.Lock()
	defer c.Unlock()

	switch data.(type) {

	case api.KVPairs:
		kvps := data.(api.KVPairs)
		c.t.Logf("Update received %d KV pairs", len(kvps))
		for _, kvp := range kvps {
			c.t.Logf("key=%s, val=%s", kvp.Key, kvp.Value)
			switch kvp.Key {
			case prefix + key1:
				c.var1 = string(kvp.Value)
				c.t.Logf("c.var1=%s", c.var1)
			case prefix + key2:
				c.var2, err = strconv.Atoi(string(kvp.Value))
				if err != nil {
					c.t.Logf("key %s is not an int", key2)
				}
			}
		}

	case []*api.ServiceEntry:
		// nothing here yet

	default:
		c.t.Log("Typecast failed")
		c.t.Fail()
	}
}

// 1. Test that the consul KV config is transferred correctly
// 2. Verify that an update is reflected in the config object
func (cs *ConfiguratorTestSuite) TestKVInit() {
	var err error

	cs.buildKVTestData()

	config := &config{
		t: cs.T(),
	}

	var wp *watch.Plan
	wp, err = cs.client.InitConfigurator(config, prefix)

	require.Nil(cs.T(), err, "InitConfigurator(..., %s) failed: %s", prefix, err)

	// Check that config has what we expect
	require.Equal(cs.T(), val1, config.var1, "the initialized val1 is not what I expected")
	require.Equal(cs.T(), val2, config.var2, "the initialized val2 is not what I expected")

	kv1 := &api.KVPair{Key: prefix + key1, Value: []byte(val1b)}
	_, err = cs.client.KV().Put(kv1, nil)
	require.Nil(cs.T(), err, "Trouble changing the value of %s", key1)
	time.Sleep(time.Second)
	require.Equal(cs.T(), val1b, config.var1, "var1 is not what i expected after updating in consul")

	kv2 := &api.KVPair{Key: prefix + key2, Value: []byte(fmt.Sprintf("%d", val2b))}
	_, err = cs.client.KV().Put(kv2, nil)
	require.Nil(cs.T(), err, "Trouble changing the value of %s", key2)
	time.Sleep(time.Second)
	require.Equal(cs.T(), val2b, config.var2, "var2 is not what i expected after updating in consul")

	time.Sleep(time.Second)

	// report what is actually in the kv prefix now:
	kvps,_,err := cs.client.KV().List(prefix, nil)
	config.Update(0,kvps)
	cs.T().Logf("config after manual update: %+v",config)

	wp.Stop()
}

func (cs *ConfiguratorTestSuite) buildKVTestData() {
	var err error

	kv1 := &api.KVPair{Key: prefix + key1, Value: []byte(val1)}
	_, err = cs.client.KV().Put(kv1, nil)
	require.Nil(cs.T(), err, "Failed storing key1/val1: %s", err)

	kv2 := &api.KVPair{Key: prefix + key2, Value: []byte(fmt.Sprintf("%d", val2))}
	_, err = cs.client.KV().Put(kv2, nil)
	require.Nil(cs.T(), err, "Failed storing key2/val2: %s", err)
}

type ConfiguratorTestSuite struct {
	suite.Suite

	// these values are cyclical, and should be re-defined per test method
	server *testutil.TestServer
	client *consultant.Client
}

// SetupTest is called before each method is run.
func (cs *ConfiguratorTestSuite) SetupTest() {
	cs.server, cs.client = makeServerAndClient(cs.T(), nil)
}

// TearDownTest is called after each method has been run.
func (cs *ConfiguratorTestSuite) TearDownTest() {
	if nil != cs.client {
		cs.client = nil
	}
	if nil != cs.server {
		// TODO: Stop seems to return an error when the process is killed...
		cs.server.Stop()
		cs.server = nil
	}
}

func (cs *ConfiguratorTestSuite) TearDownSuite() {
	cs.TearDownTest()
}

<|endoftext|>


<commit_before>package network

// Address represents an IP address
type Address struct {
	Addr      string
	PrefixLen int
}

// IPAM represents IP Address Management
type IPAM struct {
	Driver  string
	Options map[string]string //Per network IPAM driver options
	Config  []IPAMConfig
}

// IPAMConfig represents IPAM configurations
type IPAMConfig struct {
	Subnet     string            `json:",omitempty"`
	IPRange    string            `json:",omitempty"`
	Gateway    string            `json:",omitempty"`
	AuxAddress map[string]string `json:"AuxiliaryAddresses,omitempty"`
}

// EndpointIPAMConfig represents IPAM configurations for the endpoint
type EndpointIPAMConfig struct {
	IPv4Address string `json:",omitempty"`
	IPv6Address string `json:",omitempty"`
}

// EndpointSettings stores the network endpoint details
type EndpointSettings struct {
	// Configurations
	IPAMConfig *EndpointIPAMConfig
	Links      []string
	Aliases    []string
	// Operational data
	NetworkID           string
	EndpointID          string
	Gateway             string
	IPAddress           string
	IPPrefixLen         int
	IPv6Gateway         string
	GlobalIPv6Address   string
	GlobalIPv6PrefixLen int
	MacAddress          string
}

// NetworkingConfig represents the container's networking configuration for each of its interfaces
// Carries the networink configs specified in the `docker run` and `docker network connect` commands
type NetworkingConfig struct {
	EndpointsConfig map[string]*EndpointSettings // Endpoint configs for each conencting network
}
<commit_msg>Fix typos in comments<commit_after>package network

// Address represents an IP address
type Address struct {
	Addr      string
	PrefixLen int
}

// IPAM represents IP Address Management
type IPAM struct {
	Driver  string
	Options map[string]string //Per network IPAM driver options
	Config  []IPAMConfig
}

// IPAMConfig represents IPAM configurations
type IPAMConfig struct {
	Subnet     string            `json:",omitempty"`
	IPRange    string            `json:",omitempty"`
	Gateway    string            `json:",omitempty"`
	AuxAddress map[string]string `json:"AuxiliaryAddresses,omitempty"`
}

// EndpointIPAMConfig represents IPAM configurations for the endpoint
type EndpointIPAMConfig struct {
	IPv4Address string `json:",omitempty"`
	IPv6Address string `json:",omitempty"`
}

// EndpointSettings stores the network endpoint details
type EndpointSettings struct {
	// Configurations
	IPAMConfig *EndpointIPAMConfig
	Links      []string
	Aliases    []string
	// Operational data
	NetworkID           string
	EndpointID          string
	Gateway             string
	IPAddress           string
	IPPrefixLen         int
	IPv6Gateway         string
	GlobalIPv6Address   string
	GlobalIPv6PrefixLen int
	MacAddress          string
}

// NetworkingConfig represents the container's networking configuration for each of its interfaces
// Carries the networking configs specified in the `docker run` and `docker network connect` commands
type NetworkingConfig struct {
	EndpointsConfig map[string]*EndpointSettings // Endpoint configs for each connecting network
}
<|endoftext|>


<commit_before>package server

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/kevinburke/rest/resterror"
	"github.com/kevinburke/rickover/test"
)

func newSSAServer() (*SharedSecretAuthorizer, http.Handler) {
	ssa := NewSharedSecretAuthorizer()
	return ssa, Get(Config{Auth: ssa})
}

var empty = json.RawMessage([]byte("{}"))

func Test401UnknownUser(t *testing.T) {
	t.Parallel()
	w := httptest.NewRecorder()
	ejr := &EnqueueJobRequest{
		Data: empty,
	}
	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(ejr)
	req, _ := http.NewRequest("PUT", "/v1/jobs/echo/job_6740b44e-13b9-475d-af06-979627e0e0d6", b)
	req.SetBasicAuth("unknown-user", "foobar")
	_, server := newSSAServer()
	server.ServeHTTP(w, req)
	test.AssertEquals(t, w.Code, http.StatusForbidden)
	var e resterror.Error
	err := json.Unmarshal(w.Body.Bytes(), &e)
	test.AssertNotError(t, err, "")
	test.AssertEquals(t, e.Title, "Username or password are invalid. Please double check your credentials")
	test.AssertEquals(t, e.ID, "forbidden")
}

func Test401UnknownPassword(t *testing.T) {
	t.Parallel()
	w := httptest.NewRecorder()
	ejr := &EnqueueJobRequest{
		Data: empty,
	}
	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(ejr)
	ssa, server := newSSAServer()
	ssa.AddUser("401-unknown-password", "right_password")
	req, _ := http.NewRequest("PUT", "/v1/jobs/echo/job_6740b44e-13b9-475d-af06-979627e0e0d6", b)
	req.SetBasicAuth("401-unknown-password", "wrong_password")
	server.ServeHTTP(w, req)
	test.AssertEquals(t, w.Code, http.StatusForbidden)
	var e resterror.Error
	err := json.Unmarshal(w.Body.Bytes(), &e)
	test.AssertNotError(t, err, "")
	test.AssertEquals(t, e.Title, "Incorrect password for user 401-unknown-password")
	test.AssertEquals(t, e.ID, "incorrect_password")
}

func Test400NoBody(t *testing.T) {
	t.Parallel()
	test.SetUp(t)
	defer test.TearDown(t)
	w := httptest.NewRecorder()
	ejr := &EnqueueJobRequest{
		Data: empty,
	}
	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(ejr)
	ssa, server := newSSAServer()
	ssa.AddUser("test", "password")
	req, _ := http.NewRequest("PUT", "/v1/jobs/echo/job_f17373a6-2cd7-4010-afba-eebc6dc6f9ab", nil)
	req.SetBasicAuth("test", "password")
	server.ServeHTTP(w, req)
	test.AssertEquals(t, w.Code, http.StatusBadRequest)
	var e resterror.Error
	err := json.Unmarshal(w.Body.Bytes(), &e)
	test.AssertNotError(t, err, "")
	test.AssertEquals(t, e.Title, "Missing required field: data")
	test.AssertEquals(t, e.ID, "missing_parameter")
	test.AssertEquals(t, e.Instance, "/v1/jobs/echo/job_f17373a6-2cd7-4010-afba-eebc6dc6f9ab")
}

func Test400EmptyBody(t *testing.T) {
	t.Parallel()
	test.SetUp(t)
	defer test.TearDown(t)
	w := httptest.NewRecorder()
	var v interface{}
	err := json.Unmarshal([]byte("{}"), &v)
	test.AssertNotError(t, err, "")
	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(v)
	ssa, server := newSSAServer()
	ssa.AddUser("test", "password")
	req := httptest.NewRequest("PUT", "/v1/jobs/echo/job_f17373a6-2cd7-4010-afba-eebc6dc6f9ab", b)
	req.SetBasicAuth("test", "password")
	server.ServeHTTP(w, req)
	test.AssertEquals(t, w.Code, http.StatusBadRequest)
	var e resterror.Error
	err = json.Unmarshal(w.Body.Bytes(), &e)
	test.AssertNotError(t, err, "")
	test.AssertEquals(t, e.Title, "Missing required field: data")
	test.AssertEquals(t, e.ID, "missing_parameter")
	test.AssertEquals(t, e.Instance, "/v1/jobs/echo/job_f17373a6-2cd7-4010-afba-eebc6dc6f9ab")
}

func Test400InvalidUUID(t *testing.T) {
	t.Parallel()
	test.SetUp(t)
	defer test.TearDown(t)
	w := httptest.NewRecorder()
	ejr := &EnqueueJobRequest{
		Data: empty,
	}
	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(ejr)
	req, _ := http.NewRequest("PUT", "/v1/jobs/echo/job_zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", b)
	req.SetBasicAuth("test", "password")
	Get(u).ServeHTTP(w, req)
	test.AssertEquals(t, w.Code, http.StatusBadRequest)
	var e resterror.Error
	err := json.Unmarshal(w.Body.Bytes(), &e)
	test.AssertNotError(t, err, "")
	test.AssertEquals(t, e.Title, "incorrect UUID format zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz")
	test.AssertEquals(t, e.ID, "invalid_uuid")
}

func Test400WrongPrefix(t *testing.T) {
	t.Parallel()
	test.SetUp(t)
	defer test.TearDown(t)
	w := httptest.NewRecorder()
	ejr := &EnqueueJobRequest{
		Data: empty,
	}
	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(ejr)
	req, _ := http.NewRequest("PUT", "/v1/jobs/echo/usr_6740b44e-13b9-475d-af06-979627e0e0d6", b)
	req.SetBasicAuth("test", "password")
	Get(u).ServeHTTP(w, req)
	test.AssertEquals(t, w.Code, http.StatusBadRequest)
}

func Test413TooLargeJSON(t *testing.T) {
	t.Parallel()
	test.SetUp(t)
	defer test.TearDown(t)
	w := httptest.NewRecorder()
	// 4 bytes per record - the value and the quotes around it.
	var bigarr [100 * 256]string
	for i := range bigarr {
		bigarr[i] = "a"
	}
	bits, _ := json.Marshal(bigarr)
	ejr := &EnqueueJobRequest{
		Data: json.RawMessage(bits),
	}
	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(ejr)
	test.Assert(t, len(b.Bytes()) > 100*1024, fmt.Sprintf("%d", len(b.Bytes())))
	req, _ := http.NewRequest("PUT", "/v1/jobs/echo/job_6740b44e-13b9-475d-af06-979627e0e0d6", b)
	req.SetBasicAuth("test", "password")
	Get(u).ServeHTTP(w, req)
	test.AssertEquals(t, w.Code, http.StatusRequestEntityTooLarge)
	var e resterror.Error
	err := json.Unmarshal(w.Body.Bytes(), &e)
	test.AssertNotError(t, err, "")
	test.AssertEquals(t, e.Title, "Data parameter is too large (100KB max)")
	test.AssertEquals(t, e.ID, "entity_too_large")
}
<commit_msg>server: fix error message in test<commit_after>package server

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/kevinburke/rest/resterror"
	"github.com/kevinburke/rickover/test"
)

func newSSAServer() (*SharedSecretAuthorizer, http.Handler) {
	ssa := NewSharedSecretAuthorizer()
	return ssa, Get(Config{Auth: ssa})
}

var empty = json.RawMessage([]byte("{}"))

func Test401UnknownUser(t *testing.T) {
	t.Parallel()
	w := httptest.NewRecorder()
	ejr := &EnqueueJobRequest{
		Data: empty,
	}
	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(ejr)
	req, _ := http.NewRequest("PUT", "/v1/jobs/echo/job_6740b44e-13b9-475d-af06-979627e0e0d6", b)
	req.SetBasicAuth("unknown-user", "foobar")
	_, server := newSSAServer()
	server.ServeHTTP(w, req)
	test.AssertEquals(t, w.Code, http.StatusForbidden)
	var e resterror.Error
	err := json.Unmarshal(w.Body.Bytes(), &e)
	test.AssertNotError(t, err, "")
	test.AssertEquals(t, e.Title, "Username or password are invalid. Please double check your credentials")
	test.AssertEquals(t, e.ID, "forbidden")
}

func Test401UnknownPassword(t *testing.T) {
	t.Parallel()
	w := httptest.NewRecorder()
	ejr := &EnqueueJobRequest{
		Data: empty,
	}
	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(ejr)
	ssa, server := newSSAServer()
	ssa.AddUser("401-unknown-password", "right_password")
	req, _ := http.NewRequest("PUT", "/v1/jobs/echo/job_6740b44e-13b9-475d-af06-979627e0e0d6", b)
	req.SetBasicAuth("401-unknown-password", "wrong_password")
	server.ServeHTTP(w, req)
	test.AssertEquals(t, w.Code, http.StatusForbidden)
	var e resterror.Error
	err := json.Unmarshal(w.Body.Bytes(), &e)
	test.AssertNotError(t, err, "")
	test.AssertEquals(t, e.Title, "Incorrect password for user 401-unknown-password")
	test.AssertEquals(t, e.ID, "incorrect_password")
}

func Test400NoBody(t *testing.T) {
	t.Parallel()
	test.SetUp(t)
	defer test.TearDown(t)
	w := httptest.NewRecorder()
	ejr := &EnqueueJobRequest{
		Data: empty,
	}
	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(ejr)
	ssa, server := newSSAServer()
	ssa.AddUser("test", "password")
	req, _ := http.NewRequest("PUT", "/v1/jobs/echo/job_f17373a6-2cd7-4010-afba-eebc6dc6f9ab", nil)
	req.SetBasicAuth("test", "password")
	server.ServeHTTP(w, req)
	test.AssertEquals(t, w.Code, http.StatusBadRequest)
	var e resterror.Error
	err := json.Unmarshal(w.Body.Bytes(), &e)
	test.AssertNotError(t, err, "")
	test.AssertEquals(t, e.Title, "Missing required field: data")
	test.AssertEquals(t, e.ID, "missing_parameter")
	test.AssertEquals(t, e.Instance, "/v1/jobs/echo/job_f17373a6-2cd7-4010-afba-eebc6dc6f9ab")
}

func Test400EmptyBody(t *testing.T) {
	t.Parallel()
	test.SetUp(t)
	defer test.TearDown(t)
	w := httptest.NewRecorder()
	var v interface{}
	err := json.Unmarshal([]byte("{}"), &v)
	test.AssertNotError(t, err, "")
	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(v)
	ssa, server := newSSAServer()
	ssa.AddUser("test", "password")
	req := httptest.NewRequest("PUT", "/v1/jobs/echo/job_f17373a6-2cd7-4010-afba-eebc6dc6f9ab", b)
	req.SetBasicAuth("test", "password")
	server.ServeHTTP(w, req)
	test.AssertEquals(t, w.Code, http.StatusBadRequest)
	var e resterror.Error
	err = json.Unmarshal(w.Body.Bytes(), &e)
	test.AssertNotError(t, err, "")
	test.AssertEquals(t, e.Title, "Missing required field: data")
	test.AssertEquals(t, e.ID, "missing_parameter")
	test.AssertEquals(t, e.Instance, "/v1/jobs/echo/job_f17373a6-2cd7-4010-afba-eebc6dc6f9ab")
}

func Test400InvalidUUID(t *testing.T) {
	t.Parallel()
	test.SetUp(t)
	defer test.TearDown(t)
	w := httptest.NewRecorder()
	ejr := &EnqueueJobRequest{
		Data: empty,
	}
	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(ejr)
	req, _ := http.NewRequest("PUT", "/v1/jobs/echo/job_zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", b)
	req.SetBasicAuth("test", "password")
	Get(u).ServeHTTP(w, req)
	test.AssertEquals(t, w.Code, http.StatusBadRequest)
	var e resterror.Error
	err := json.Unmarshal(w.Body.Bytes(), &e)
	test.AssertNotError(t, err, "")
	test.AssertEquals(t, e.Title, `incorrect UUID format in string "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"`)
	test.AssertEquals(t, e.ID, "invalid_uuid")
}

func Test400WrongPrefix(t *testing.T) {
	t.Parallel()
	test.SetUp(t)
	defer test.TearDown(t)
	w := httptest.NewRecorder()
	ejr := &EnqueueJobRequest{
		Data: empty,
	}
	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(ejr)
	req, _ := http.NewRequest("PUT", "/v1/jobs/echo/usr_6740b44e-13b9-475d-af06-979627e0e0d6", b)
	req.SetBasicAuth("test", "password")
	Get(u).ServeHTTP(w, req)
	test.AssertEquals(t, w.Code, http.StatusBadRequest)
}

func Test413TooLargeJSON(t *testing.T) {
	t.Parallel()
	test.SetUp(t)
	defer test.TearDown(t)
	w := httptest.NewRecorder()
	// 4 bytes per record - the value and the quotes around it.
	var bigarr [100 * 256]string
	for i := range bigarr {
		bigarr[i] = "a"
	}
	bits, _ := json.Marshal(bigarr)
	ejr := &EnqueueJobRequest{
		Data: json.RawMessage(bits),
	}
	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(ejr)
	test.Assert(t, len(b.Bytes()) > 100*1024, fmt.Sprintf("%d", len(b.Bytes())))
	req, _ := http.NewRequest("PUT", "/v1/jobs/echo/job_6740b44e-13b9-475d-af06-979627e0e0d6", b)
	req.SetBasicAuth("test", "password")
	Get(u).ServeHTTP(w, req)
	test.AssertEquals(t, w.Code, http.StatusRequestEntityTooLarge)
	var e resterror.Error
	err := json.Unmarshal(w.Body.Bytes(), &e)
	test.AssertNotError(t, err, "")
	test.AssertEquals(t, e.Title, "Data parameter is too large (100KB max)")
	test.AssertEquals(t, e.ID, "entity_too_large")
}
<|endoftext|>


<commit_before>package consensus

import (
	"encoding/binary"
	"fmt"
	"strings"

	log "github.com/sirupsen/logrus"

	"github.com/bytom/bytom/crypto/ed25519/chainkd"
	"github.com/bytom/bytom/protocol/bc"
)

// consensus variables
const (
	// Max gas that one block contains
	MaxBlockGas    = uint64(10000000)
	VMGasRate      = int64(200)
	StorageGasRate = int64(1)
	MaxGasAmount   = int64(300000)

	// These configs need add to casper config in elegant way
	MaxNumOfValidators = int(10)
	InitBTMSupply      = 169290721678579697
	RewardThreshold    = 0.5
	BlockReward        = uint64(570776255)

	// config parameter for coinbase reward
	CoinbasePendingBlockNumber = uint64(10)
	MinVoteOutputAmount        = uint64(100000000)

	PayToWitnessPubKeyHashDataSize = 20
	PayToWitnessScriptHashDataSize = 32
	BCRPContractHashDataSize       = 32
	CoinbaseArbitrarySizeLimit     = 128

	BCRPRequiredBTMAmount = uint64(100000000)

	BTMAlias = "BTM"
)

type CasperConfig struct {
	// BlockTimeInterval, milliseconds, the block time interval for producing a block
	BlockTimeInterval uint64

	// MaxTimeOffsetMs represent the max number of seconds a block time is allowed to be ahead of the current time
	MaxTimeOffsetMs uint64

	// BlocksOfEpoch represent the block num in one epoch
	BlocksOfEpoch uint64

	// MinValidatorVoteNum is the minimum vote number of become validator
	MinValidatorVoteNum uint64

	// VotePendingBlockNumber is the locked block number of vote utxo
	VotePendingBlockNumber uint64

	FederationXpubs []chainkd.XPub
}

// BTMAssetID is BTM's asset id, the soul asset of Bytom
var BTMAssetID = &bc.AssetID{
	V0: binary.BigEndian.Uint64([]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}),
	V1: binary.BigEndian.Uint64([]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}),
	V2: binary.BigEndian.Uint64([]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}),
	V3: binary.BigEndian.Uint64([]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}),
}

// BTMDefinitionMap is the ....
var BTMDefinitionMap = map[string]interface{}{
	"name":        BTMAlias,
	"symbol":      BTMAlias,
	"decimals":    8,
	"description": `Bytom Official Issue`,
}

// IsBech32SegwitPrefix returns whether the prefix is a known prefix for segwit
// addresses on any default or registered network.  This is used when decoding
// an address string into a specific address type.
func IsBech32SegwitPrefix(prefix string, params *Params) bool {
	prefix = strings.ToLower(prefix)
	return prefix == params.Bech32HRPSegwit+"1"
}

// Params store the config for different network
type Params struct {
	// Name defines a human-readable identifier for the network.
	Name            string
	Bech32HRPSegwit string
	// DefaultPort defines the default peer-to-peer port for the network.
	DefaultPort string

	// DNSSeeds defines a list of DNS seeds for the network that are used
	// as one method to discover peers.
	DNSSeeds []string

	// CasperConfig defines the casper consensus parameters
	CasperConfig
}

// ActiveNetParams is ...
var ActiveNetParams = MainNetParams

// NetParams is the correspondence between chain_id and Params
var NetParams = map[string]Params{
	"mainnet": MainNetParams,
	"wisdom":  TestNetParams,
	"solonet": SoloNetParams,
}

// MainNetParams is the config for production
var MainNetParams = Params{
	Name:            "main",
	Bech32HRPSegwit: "bn",
	DefaultPort:     "46657",
	DNSSeeds:        []string{},
	CasperConfig: CasperConfig{
		BlockTimeInterval:      6000,
		MaxTimeOffsetMs:        3000,
		BlocksOfEpoch:          100,
		MinValidatorVoteNum:    1e14,
		VotePendingBlockNumber: 14400,
		FederationXpubs: []chainkd.XPub{
			xpub("364344fd6e612d10b51745c336c7bf096520265725b1d149b65d12cda6248aa4c98ad68891a503d9eebbf2b7c52188bfd653c2b046f55486ff84e2e3c98ef864"),
			xpub("6e49c57d8bb5ab091d644241e3deedefa1c47bd46ef044ff4b3719f31736afe6caf015ff1f7b2af6c80ff6df87ae2d4bb18f7273f0646ca6df5879707ab756de"),
			xpub("72740630523dddba86213938431938be2b8d1ccac1280b59069cf0b683b0ba3fe7d489e107bbd0f83c9e0b4c841e73aea414796d5078824cf7eb40be09f8db95"),
			xpub("c001cfe8543a8c0ee984ba11da5e26f2aa95ffffd278b9823e77714f18fc4a650e01146449e00a9546a22153566907a6be6e9d1dfe0d72a40fcd0c4131570caa"),
		},
	},
}

// TestNetParams is the config for test-net
var TestNetParams = Params{
	Name:            "test",
	Bech32HRPSegwit: "tn",
	DefaultPort:     "46656",
	DNSSeeds:        []string{},
	CasperConfig: CasperConfig{
		BlockTimeInterval:      6000,
		MaxTimeOffsetMs:        3000,
		BlocksOfEpoch:          100,
		MinValidatorVoteNum:    1e8,
		VotePendingBlockNumber: 10,
		FederationXpubs: []chainkd.XPub{
			xpub("7732fac62320799ff5e4eec1dc4ba7b07dc0e5a647850bf0bc34cb9aca195a05a1118b57d377947d7936156c831c87b700ed945a82cae63aff14905beb39d001"),
			xpub("08543fef8c3ca27483954f80eee6d461c307b6aa564aafaf235a4bd2740debbc71b14af78715c94cbc1d16fa84da97a3eabc5b21f003ab49882e4af7f9f00bbd"),
			xpub("0dd00fe3880c1cb5d5b0b5d03993c004e7fbe3697a47ff60c3bc12950bead964843dfe45b2bab5d01ae32fb23a4b0460049e822d7787a9a15b76d8bb9dfcec74"),
			xpub("b0584ecaefc02d3c367f280e128ec310c9f9198d44cd76b6726cd6c06c002770a1a7dc069ddd06f7a821a176931573d40e63b015ce88b6de01a61205d719567f"),
		},
	},
}

// SoloNetParams is the config for test-net
var SoloNetParams = Params{
	Name:            "solo",
	Bech32HRPSegwit: "sn",
	CasperConfig: CasperConfig{
		BlockTimeInterval:      6000,
		MaxTimeOffsetMs:        24000,
		BlocksOfEpoch:          100,
		MinValidatorVoteNum:    1e8,
		VotePendingBlockNumber: 10,
		FederationXpubs:        []chainkd.XPub{},
	},
}

// InitActiveNetParams load the config by chain ID
func InitActiveNetParams(chainID string) error {
	var exist bool
	if ActiveNetParams, exist = NetParams[chainID]; !exist {
		return fmt.Errorf("chain_id[%v] don't exist", chainID)
	}
	return nil
}

func xpub(str string) (xpub chainkd.XPub) {
	if err := xpub.UnmarshalText([]byte(str)); err != nil {
		log.Panicf("Fail converts a string to xpub")
	}
	return xpub
}
<commit_msg>finalize super node key<commit_after>package consensus

import (
	"encoding/binary"
	"fmt"
	"strings"

	log "github.com/sirupsen/logrus"

	"github.com/bytom/bytom/crypto/ed25519/chainkd"
	"github.com/bytom/bytom/protocol/bc"
)

// consensus variables
const (
	// Max gas that one block contains
	MaxBlockGas    = uint64(10000000)
	VMGasRate      = int64(200)
	StorageGasRate = int64(1)
	MaxGasAmount   = int64(300000)

	// These configs need add to casper config in elegant way
	MaxNumOfValidators = int(10)
	InitBTMSupply      = 169290721678579697
	RewardThreshold    = 0.5
	BlockReward        = uint64(570776255)

	// config parameter for coinbase reward
	CoinbasePendingBlockNumber = uint64(10)
	MinVoteOutputAmount        = uint64(100000000)

	PayToWitnessPubKeyHashDataSize = 20
	PayToWitnessScriptHashDataSize = 32
	BCRPContractHashDataSize       = 32
	CoinbaseArbitrarySizeLimit     = 128

	BCRPRequiredBTMAmount = uint64(100000000)

	BTMAlias = "BTM"
)

type CasperConfig struct {
	// BlockTimeInterval, milliseconds, the block time interval for producing a block
	BlockTimeInterval uint64

	// MaxTimeOffsetMs represent the max number of seconds a block time is allowed to be ahead of the current time
	MaxTimeOffsetMs uint64

	// BlocksOfEpoch represent the block num in one epoch
	BlocksOfEpoch uint64

	// MinValidatorVoteNum is the minimum vote number of become validator
	MinValidatorVoteNum uint64

	// VotePendingBlockNumber is the locked block number of vote utxo
	VotePendingBlockNumber uint64

	FederationXpubs []chainkd.XPub
}

// BTMAssetID is BTM's asset id, the soul asset of Bytom
var BTMAssetID = &bc.AssetID{
	V0: binary.BigEndian.Uint64([]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}),
	V1: binary.BigEndian.Uint64([]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}),
	V2: binary.BigEndian.Uint64([]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}),
	V3: binary.BigEndian.Uint64([]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}),
}

// BTMDefinitionMap is the ....
var BTMDefinitionMap = map[string]interface{}{
	"name":        BTMAlias,
	"symbol":      BTMAlias,
	"decimals":    8,
	"description": `Bytom Official Issue`,
}

// IsBech32SegwitPrefix returns whether the prefix is a known prefix for segwit
// addresses on any default or registered network.  This is used when decoding
// an address string into a specific address type.
func IsBech32SegwitPrefix(prefix string, params *Params) bool {
	prefix = strings.ToLower(prefix)
	return prefix == params.Bech32HRPSegwit+"1"
}

// Params store the config for different network
type Params struct {
	// Name defines a human-readable identifier for the network.
	Name            string
	Bech32HRPSegwit string
	// DefaultPort defines the default peer-to-peer port for the network.
	DefaultPort string

	// DNSSeeds defines a list of DNS seeds for the network that are used
	// as one method to discover peers.
	DNSSeeds []string

	// CasperConfig defines the casper consensus parameters
	CasperConfig
}

// ActiveNetParams is ...
var ActiveNetParams = MainNetParams

// NetParams is the correspondence between chain_id and Params
var NetParams = map[string]Params{
	"mainnet": MainNetParams,
	"wisdom":  TestNetParams,
	"solonet": SoloNetParams,
}

// MainNetParams is the config for production
var MainNetParams = Params{
	Name:            "main",
	Bech32HRPSegwit: "bn",
	DefaultPort:     "46657",
	DNSSeeds:        []string{},
	CasperConfig: CasperConfig{
		BlockTimeInterval:      6000,
		MaxTimeOffsetMs:        3000,
		BlocksOfEpoch:          100,
		MinValidatorVoteNum:    1e14,
		VotePendingBlockNumber: 14400,
		FederationXpubs: []chainkd.XPub{
			xpub("f9003633ccbd8cc37e034f4dbe70d9fae980d437948d8cb908d0cab7909780d74a324b4decb5dfcd43fbc6b896ac066b7e02c733a1537360e933278a101a850c"),
			xpub("d301fee5d4ba7eb5b9d41ca13ec56c19daceb5f6b752d91d49777fd1fc7c45891e5773cafb3b6d6ab764ef2794e8ba953c8bdb9dc77a3af51e979f96885f96b2"),
			xpub("2ba14bdd29fd84c73f67d6025d2a98292dbdd46b90a2af29c8669dd88dacb1cec62a3e9448d8b731a448f0454b0aa367748659d6c01ad7125d395ffda972da54"),
			xpub("1313379b05c38ff2d171d512f23f199f0f068a67d77b9d5b6db040f2da1edc0c35c68a21b068956f448fed6441b9c27294f1ca6aaedc2c580de322f3f0260c1f"),
		},
	},
}

// TestNetParams is the config for test-net
var TestNetParams = Params{
	Name:            "test",
	Bech32HRPSegwit: "tn",
	DefaultPort:     "46656",
	DNSSeeds:        []string{},
	CasperConfig: CasperConfig{
		BlockTimeInterval:      6000,
		MaxTimeOffsetMs:        3000,
		BlocksOfEpoch:          100,
		MinValidatorVoteNum:    1e8,
		VotePendingBlockNumber: 10,
		FederationXpubs: []chainkd.XPub{
			xpub("7732fac62320799ff5e4eec1dc4ba7b07dc0e5a647850bf0bc34cb9aca195a05a1118b57d377947d7936156c831c87b700ed945a82cae63aff14905beb39d001"),
			xpub("08543fef8c3ca27483954f80eee6d461c307b6aa564aafaf235a4bd2740debbc71b14af78715c94cbc1d16fa84da97a3eabc5b21f003ab49882e4af7f9f00bbd"),
			xpub("0dd00fe3880c1cb5d5b0b5d03993c004e7fbe3697a47ff60c3bc12950bead964843dfe45b2bab5d01ae32fb23a4b0460049e822d7787a9a15b76d8bb9dfcec74"),
			xpub("b0584ecaefc02d3c367f280e128ec310c9f9198d44cd76b6726cd6c06c002770a1a7dc069ddd06f7a821a176931573d40e63b015ce88b6de01a61205d719567f"),
		},
	},
}

// SoloNetParams is the config for test-net
var SoloNetParams = Params{
	Name:            "solo",
	Bech32HRPSegwit: "sn",
	CasperConfig: CasperConfig{
		BlockTimeInterval:      6000,
		MaxTimeOffsetMs:        24000,
		BlocksOfEpoch:          100,
		MinValidatorVoteNum:    1e8,
		VotePendingBlockNumber: 10,
		FederationXpubs:        []chainkd.XPub{},
	},
}

// InitActiveNetParams load the config by chain ID
func InitActiveNetParams(chainID string) error {
	var exist bool
	if ActiveNetParams, exist = NetParams[chainID]; !exist {
		return fmt.Errorf("chain_id[%v] don't exist", chainID)
	}
	return nil
}

func xpub(str string) (xpub chainkd.XPub) {
	if err := xpub.UnmarshalText([]byte(str)); err != nil {
		log.Panicf("Fail converts a string to xpub")
	}
	return xpub
}
<|endoftext|>


<commit_before>package mpawsdynamodb

import (
	"flag"
	"log"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface"
	mp "github.com/mackerelio/go-mackerel-plugin-helper"
)

const (
	namespace              = "AWS/DynamoDB"
	metricsTypeAverage     = "Average"
	metricsTypeSum         = "Sum"
	metricsTypeMaximum     = "Maximum"
	metricsTypeMinimum     = "Minimum"
	metricsTypeSampleCount = "SampleCount"
)

// has 1 CloudWatch MetricName and corresponding N Mackerel Metrics
type metricsGroup struct {
	CloudWatchName string
	Metrics        []metric
}

type metric struct {
	MackerelName string
	Type         string
}

// DynamoDBPlugin mackerel plugin for aws kinesis
type DynamoDBPlugin struct {
	TableName string
	Prefix    string

	AccessKeyID     string
	SecretAccessKey string
	Region          string
	CloudWatch      *cloudwatch.CloudWatch
}

// MetricKeyPrefix interface for PluginWithPrefix
func (p DynamoDBPlugin) MetricKeyPrefix() string {
	return p.Prefix
}

// prepare creates CloudWatch instance
func (p *DynamoDBPlugin) prepare() error {
	sess, err := session.NewSession()
	if err != nil {
		return err
	}

	config := aws.NewConfig()
	if p.AccessKeyID != "" && p.SecretAccessKey != "" {
		config = config.WithCredentials(credentials.NewStaticCredentials(p.AccessKeyID, p.SecretAccessKey, ""))
	}
	if p.Region != "" {
		config = config.WithRegion(p.Region)
	}

	p.CloudWatch = cloudwatch.New(sess, config)

	return nil
}

func transformAndAppendDatapoint(dp *cloudwatch.Datapoint, dataType string, label string, stats map[string]interface{}) map[string]interface{} {
	if dp != nil {
		switch dataType {
		case metricsTypeAverage:
			stats[label] = *dp.Average
		case metricsTypeSum:
			stats[label] = *dp.Sum
		case metricsTypeMaximum:
			stats[label] = *dp.Maximum
		case metricsTypeMinimum:
			stats[label] = *dp.Minimum
		case metricsTypeSampleCount:
			stats[label] = *dp.SampleCount
		}
	}
	return stats
}

// fetch metrics which takes "Operation" dimensions querying both ListMetrics and GetMetricsStatistics
func fetchOperationWildcardMetrics(cw cloudwatchiface.CloudWatchAPI, mg metricsGroup, baseDimensions []*cloudwatch.Dimension) (map[string]interface{}, error) {
	// get available dimensions
	dimensionFilters := make([]*cloudwatch.DimensionFilter, len(baseDimensions))
	for i, dimension := range baseDimensions {
		dimensionFilters[i] = &cloudwatch.DimensionFilter{
			Name:  dimension.Name,
			Value: dimension.Value,
		}
	}
	input := &cloudwatch.ListMetricsInput{
		Dimensions: dimensionFilters,
		Namespace:  aws.String(namespace),
		MetricName: aws.String(mg.CloudWatchName),
	}
	// ListMetrics can retrieve up to 500 metrics, but DynamoDB Operations are apparently less than 500
	res, err := cw.ListMetrics(input)
	if err != nil {
		return nil, err
	}

	stats := make(map[string]interface{})

	// get datapoints with retrieved dimensions
	for _, cwMetric := range res.Metrics {
		dimensions := cwMetric.Dimensions
		// extract operation name
		var operation *string
		for _, d := range dimensions {
			if *d.Name == "Operation" {
				operation = d.Value
				break
			}
		}
		if operation == nil {
			log.Printf("Unexpected dimension, skip: %s", dimensions)
			continue
		}

		dp, err := getLastPointFromCloudWatch(cw, mg, dimensions)
		if err != nil {
			return nil, nil
		}
		if dp != nil {
			for _, met := range mg.Metrics {
				label := strings.Replace(met.MackerelName, "#", *operation, 1)
				stats = transformAndAppendDatapoint(dp, met.Type, label, stats)
			}
		}
	}

	return stats, nil
}

// getLastPoint fetches a CloudWatch metric and parse
func getLastPointFromCloudWatch(cw cloudwatchiface.CloudWatchAPI, metric metricsGroup, dimensions []*cloudwatch.Dimension) (*cloudwatch.Datapoint, error) {
	now := time.Now()
	statsInput := make([]*string, len(metric.Metrics))
	for i, typ := range metric.Metrics {
		statsInput[i] = aws.String(typ.Type)
	}
	input := &cloudwatch.GetMetricStatisticsInput{
		// 8 min, since some metrics are aggregated over 5 min
		StartTime:  aws.Time(now.Add(time.Duration(480) * time.Second * -1)),
		EndTime:    aws.Time(now),
		MetricName: aws.String(metric.CloudWatchName),
		Period:     aws.Int64(60),
		Statistics: statsInput,
		Namespace:  aws.String(namespace),
		Dimensions: dimensions,
	}
	response, err := cw.GetMetricStatistics(input)
	if err != nil {
		return nil, err
	}

	datapoints := response.Datapoints
	if len(datapoints) == 0 {
		return nil, nil
	}

	latest := new(time.Time)
	var latestDp *cloudwatch.Datapoint
	for _, dp := range datapoints {
		if dp.Timestamp.Before(*latest) {
			continue
		}

		latest = dp.Timestamp
		latestDp = dp
	}

	return latestDp, nil
}

var defaultMetricsGroup = []metricsGroup{
	{CloudWatchName: "ConditionalCheckFailedRequests", Metrics: []metric{
		{MackerelName: "ConditionalCheckFailedRequests", Type: metricsTypeSum},
	}},
	{CloudWatchName: "ConsumedReadCapacityUnits", Metrics: []metric{
		{MackerelName: "ConsumedReadCapacityUnitsSum", Type: metricsTypeSum},
		{MackerelName: "ConsumedReadCapacityUnitsAverage", Type: metricsTypeAverage},
	}},
	{CloudWatchName: "ConsumedWriteCapacityUnits", Metrics: []metric{
		{MackerelName: "ConsumedWriteCapacityUnitsSum", Type: metricsTypeSum},
		{MackerelName: "ConsumedWriteCapacityUnitsAverage", Type: metricsTypeAverage},
	}},
	{CloudWatchName: "ProvisionedReadCapacityUnits", Metrics: []metric{
		{MackerelName: "ProvisionedReadCapacityUnits", Type: metricsTypeMinimum},
	}},
	{CloudWatchName: "ProvisionedWriteCapacityUnits", Metrics: []metric{
		{MackerelName: "ProvisionedWriteCapacityUnits", Type: metricsTypeMinimum},
	}},
	{CloudWatchName: "SystemErrors", Metrics: []metric{
		{MackerelName: "SystemErrors", Type: metricsTypeSum},
	}},
	{CloudWatchName: "UserErrors", Metrics: []metric{
		{MackerelName: "UserErrors", Type: metricsTypeSum},
	}},
	{CloudWatchName: "WriteThrottleEvents", Metrics: []metric{
		{MackerelName: "WriteThrottleEvents", Type: metricsTypeSum},
	}},
}

var operationalMetricsGroup = []metricsGroup{
	{CloudWatchName: "SuccessfulRequestLatency", Metrics: []metric{
		{MackerelName: "SuccessfulRequests.#", Type: metricsTypeSampleCount},
		{MackerelName: "SuccessfulRequestLatency.#.Minimum", Type: metricsTypeMinimum},
		{MackerelName: "SuccessfulRequestLatency.#.Maximum", Type: metricsTypeMaximum},
		{MackerelName: "SuccessfulRequestLatency.#.Average", Type: metricsTypeAverage},
	}},
	{CloudWatchName: "ThrottledRequests", Metrics: []metric{
		{MackerelName: "ThrottledRequests.#", Type: metricsTypeSampleCount},
	}},
	{CloudWatchName: "SystemErrors", Metrics: []metric{
		{MackerelName: "SystemErrors.#", Type: metricsTypeSampleCount},
	}},
	{CloudWatchName: "UserErrors", Metrics: []metric{
		{MackerelName: "UserErrors.#", Type: metricsTypeSampleCount},
	}},
	{CloudWatchName: "ReturnedItemCount", Metrics: []metric{
		{MackerelName: "ReturnedItemCount.#", Type: metricsTypeAverage},
	}},
}

// FetchMetrics fetch the metrics
func (p DynamoDBPlugin) FetchMetrics() (map[string]interface{}, error) {
	stats := make(map[string]interface{})

	tableDimensions := []*cloudwatch.Dimension{{
		Name:  aws.String("TableName"),
		Value: aws.String(p.TableName),
	}}
	for _, met := range defaultMetricsGroup {
		dp, err := getLastPointFromCloudWatch(p.CloudWatch, met, tableDimensions)
		if err == nil {
			for _, m := range met.Metrics {
				stats = transformAndAppendDatapoint(dp, m.Type, m.MackerelName, stats)
			}
		} else {
			log.Printf("%s: %s", met, err)
		}
	}

	for _, met := range operationalMetricsGroup {
		operationalStats, err := fetchOperationWildcardMetrics(p.CloudWatch, met, tableDimensions)
		if err == nil {
			for name, s := range operationalStats {
				stats[name] = s
			}
		} else {
			log.Printf("%s: %s", met, err)
		}
	}
	return transformMetrics(stats), nil
}

// TransformMetrics converts some of datapoints to post differences of two metrics
func transformMetrics(stats map[string]interface{}) map[string]interface{} {
	// Although stats are interface{}, those values from cloudwatch.Datapoint are guaranteed to be numerical
	if consumedReadCapacitySum, ok := stats["ConsumedReadCapacityUnitsSum"].(float64); ok {
		stats["ConsumedReadCapacityUnitsNormalized"] = consumedReadCapacitySum / 60.0
	}
	if consumedWriteCapacitySum, ok := stats["ConsumedWriteCapacityUnitsSum"].(float64); ok {
		stats["ConsumedWriteCapacityUnitsNormalized"] = consumedWriteCapacitySum / 60.0
	}
	return stats
}

// GraphDefinition of DynamoDBPlugin
func (p DynamoDBPlugin) GraphDefinition() map[string]mp.Graphs {
	labelPrefix := strings.Title(p.Prefix)
	labelPrefix = strings.Replace(labelPrefix, "-", " ", -1)
	labelPrefix = strings.Replace(labelPrefix, "Dynamodb", "DynamoDB", -1)

	var graphdef = map[string]mp.Graphs{
		"ReadCapacity": {
			Label: (labelPrefix + " Read Capacity Units"),
			Unit:  "float",
			Metrics: []mp.Metrics{
				{Name: "ProvisionedReadCapacityUnits", Label: "Provisioned"},
				{Name: "ConsumedReadCapacityUnitsNormalized", Label: "Consumed"},
				{Name: "ConsumedReadCapacityUnitsAverage", Label: "Consumed (Average per request)"},
			},
		},
		"WriteCapacity": {
			Label: (labelPrefix + " Write Capacity Units"),
			Unit:  "float",
			Metrics: []mp.Metrics{
				{Name: "ProvisionedWriteCapacityUnits", Label: "Provisioned"},
				{Name: "ConsumedWriteCapacityUnitsNormalized", Label: "Consumed"},
				{Name: "ConsumedWriteCapacityUnitsAverage", Label: "Consumed (Average per request)"},
			},
		},
		"ThrottledEvents": {
			Label: (labelPrefix + " Throttle Events"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "ReadThrottleEvents", Label: "Read"},
				{Name: "WriteThrottleEvents", Label: "Write"},
			},
		},
		"ConditionalCheckFailedRequests": {
			Label: (labelPrefix + " ConditionalCheckFailedRequests"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "ConditionalCheckFailedRequests", Label: "Counts"},
			},
		},
		"ThrottledRequests": {
			Label: (labelPrefix + " ThrottledRequests"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "*", Label: "%1", Stacked: true},
			},
		},
		"SystemErrors": {
			Label: (labelPrefix + " SystemErrors"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "*", Label: "%1", Stacked: true},
			},
		},
		"UserErrors": {
			Label: (labelPrefix + " UserErrors"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "*", Label: "%1", Stacked: true},
			},
		},
		"ReturnedItemCount": {
			Label: (p.LabelPrefix + " ReturnedItemCount"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "*", Label: "%1"},
			},
		},
		"SuccessfulRequests": {
			Label: (labelPrefix + " SuccessfulRequests"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "*", Label: "%1"},
			},
		},
		"SuccessfulRequestLatency.#": {
			Label: (labelPrefix + " SuccessfulRequestLatency"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "Minimum", Label: "Min"},
				{Name: "Maximum", Label: "Max"},
				{Name: "Average", Label: "Average"},
			},
		},
	}
	return graphdef
}

// Do the plugin
func Do() {
	optAccessKeyID := flag.String("access-key-id", "", "AWS Access Key ID")
	optSecretAccessKey := flag.String("secret-access-key", "", "AWS Secret Access Key")
	optRegion := flag.String("region", "", "AWS Region")
	optTableName := flag.String("table-name", "", "DynamoDB Table Name")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	optPrefix := flag.String("metric-key-prefix", "dynamodb", "Metric key prefix")
	flag.Parse()

	var plugin DynamoDBPlugin

	plugin.AccessKeyID = *optAccessKeyID
	plugin.SecretAccessKey = *optSecretAccessKey
	plugin.Region = *optRegion
	plugin.TableName = *optTableName
	plugin.Prefix = *optPrefix

	err := plugin.prepare()
	if err != nil {
		log.Fatalln(err)
	}

	helper := mp.NewMackerelPlugin(plugin)
	helper.Tempfile = *optTempfile

	helper.Run()
}
<commit_msg>p.LabelPrefix has gone<commit_after>package mpawsdynamodb

import (
	"flag"
	"log"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface"
	mp "github.com/mackerelio/go-mackerel-plugin-helper"
)

const (
	namespace              = "AWS/DynamoDB"
	metricsTypeAverage     = "Average"
	metricsTypeSum         = "Sum"
	metricsTypeMaximum     = "Maximum"
	metricsTypeMinimum     = "Minimum"
	metricsTypeSampleCount = "SampleCount"
)

// has 1 CloudWatch MetricName and corresponding N Mackerel Metrics
type metricsGroup struct {
	CloudWatchName string
	Metrics        []metric
}

type metric struct {
	MackerelName string
	Type         string
}

// DynamoDBPlugin mackerel plugin for aws kinesis
type DynamoDBPlugin struct {
	TableName string
	Prefix    string

	AccessKeyID     string
	SecretAccessKey string
	Region          string
	CloudWatch      *cloudwatch.CloudWatch
}

// MetricKeyPrefix interface for PluginWithPrefix
func (p DynamoDBPlugin) MetricKeyPrefix() string {
	return p.Prefix
}

// prepare creates CloudWatch instance
func (p *DynamoDBPlugin) prepare() error {
	sess, err := session.NewSession()
	if err != nil {
		return err
	}

	config := aws.NewConfig()
	if p.AccessKeyID != "" && p.SecretAccessKey != "" {
		config = config.WithCredentials(credentials.NewStaticCredentials(p.AccessKeyID, p.SecretAccessKey, ""))
	}
	if p.Region != "" {
		config = config.WithRegion(p.Region)
	}

	p.CloudWatch = cloudwatch.New(sess, config)

	return nil
}

func transformAndAppendDatapoint(dp *cloudwatch.Datapoint, dataType string, label string, stats map[string]interface{}) map[string]interface{} {
	if dp != nil {
		switch dataType {
		case metricsTypeAverage:
			stats[label] = *dp.Average
		case metricsTypeSum:
			stats[label] = *dp.Sum
		case metricsTypeMaximum:
			stats[label] = *dp.Maximum
		case metricsTypeMinimum:
			stats[label] = *dp.Minimum
		case metricsTypeSampleCount:
			stats[label] = *dp.SampleCount
		}
	}
	return stats
}

// fetch metrics which takes "Operation" dimensions querying both ListMetrics and GetMetricsStatistics
func fetchOperationWildcardMetrics(cw cloudwatchiface.CloudWatchAPI, mg metricsGroup, baseDimensions []*cloudwatch.Dimension) (map[string]interface{}, error) {
	// get available dimensions
	dimensionFilters := make([]*cloudwatch.DimensionFilter, len(baseDimensions))
	for i, dimension := range baseDimensions {
		dimensionFilters[i] = &cloudwatch.DimensionFilter{
			Name:  dimension.Name,
			Value: dimension.Value,
		}
	}
	input := &cloudwatch.ListMetricsInput{
		Dimensions: dimensionFilters,
		Namespace:  aws.String(namespace),
		MetricName: aws.String(mg.CloudWatchName),
	}
	// ListMetrics can retrieve up to 500 metrics, but DynamoDB Operations are apparently less than 500
	res, err := cw.ListMetrics(input)
	if err != nil {
		return nil, err
	}

	stats := make(map[string]interface{})

	// get datapoints with retrieved dimensions
	for _, cwMetric := range res.Metrics {
		dimensions := cwMetric.Dimensions
		// extract operation name
		var operation *string
		for _, d := range dimensions {
			if *d.Name == "Operation" {
				operation = d.Value
				break
			}
		}
		if operation == nil {
			log.Printf("Unexpected dimension, skip: %s", dimensions)
			continue
		}

		dp, err := getLastPointFromCloudWatch(cw, mg, dimensions)
		if err != nil {
			return nil, nil
		}
		if dp != nil {
			for _, met := range mg.Metrics {
				label := strings.Replace(met.MackerelName, "#", *operation, 1)
				stats = transformAndAppendDatapoint(dp, met.Type, label, stats)
			}
		}
	}

	return stats, nil
}

// getLastPoint fetches a CloudWatch metric and parse
func getLastPointFromCloudWatch(cw cloudwatchiface.CloudWatchAPI, metric metricsGroup, dimensions []*cloudwatch.Dimension) (*cloudwatch.Datapoint, error) {
	now := time.Now()
	statsInput := make([]*string, len(metric.Metrics))
	for i, typ := range metric.Metrics {
		statsInput[i] = aws.String(typ.Type)
	}
	input := &cloudwatch.GetMetricStatisticsInput{
		// 8 min, since some metrics are aggregated over 5 min
		StartTime:  aws.Time(now.Add(time.Duration(480) * time.Second * -1)),
		EndTime:    aws.Time(now),
		MetricName: aws.String(metric.CloudWatchName),
		Period:     aws.Int64(60),
		Statistics: statsInput,
		Namespace:  aws.String(namespace),
		Dimensions: dimensions,
	}
	response, err := cw.GetMetricStatistics(input)
	if err != nil {
		return nil, err
	}

	datapoints := response.Datapoints
	if len(datapoints) == 0 {
		return nil, nil
	}

	latest := new(time.Time)
	var latestDp *cloudwatch.Datapoint
	for _, dp := range datapoints {
		if dp.Timestamp.Before(*latest) {
			continue
		}

		latest = dp.Timestamp
		latestDp = dp
	}

	return latestDp, nil
}

var defaultMetricsGroup = []metricsGroup{
	{CloudWatchName: "ConditionalCheckFailedRequests", Metrics: []metric{
		{MackerelName: "ConditionalCheckFailedRequests", Type: metricsTypeSum},
	}},
	{CloudWatchName: "ConsumedReadCapacityUnits", Metrics: []metric{
		{MackerelName: "ConsumedReadCapacityUnitsSum", Type: metricsTypeSum},
		{MackerelName: "ConsumedReadCapacityUnitsAverage", Type: metricsTypeAverage},
	}},
	{CloudWatchName: "ConsumedWriteCapacityUnits", Metrics: []metric{
		{MackerelName: "ConsumedWriteCapacityUnitsSum", Type: metricsTypeSum},
		{MackerelName: "ConsumedWriteCapacityUnitsAverage", Type: metricsTypeAverage},
	}},
	{CloudWatchName: "ProvisionedReadCapacityUnits", Metrics: []metric{
		{MackerelName: "ProvisionedReadCapacityUnits", Type: metricsTypeMinimum},
	}},
	{CloudWatchName: "ProvisionedWriteCapacityUnits", Metrics: []metric{
		{MackerelName: "ProvisionedWriteCapacityUnits", Type: metricsTypeMinimum},
	}},
	{CloudWatchName: "SystemErrors", Metrics: []metric{
		{MackerelName: "SystemErrors", Type: metricsTypeSum},
	}},
	{CloudWatchName: "UserErrors", Metrics: []metric{
		{MackerelName: "UserErrors", Type: metricsTypeSum},
	}},
	{CloudWatchName: "WriteThrottleEvents", Metrics: []metric{
		{MackerelName: "WriteThrottleEvents", Type: metricsTypeSum},
	}},
}

var operationalMetricsGroup = []metricsGroup{
	{CloudWatchName: "SuccessfulRequestLatency", Metrics: []metric{
		{MackerelName: "SuccessfulRequests.#", Type: metricsTypeSampleCount},
		{MackerelName: "SuccessfulRequestLatency.#.Minimum", Type: metricsTypeMinimum},
		{MackerelName: "SuccessfulRequestLatency.#.Maximum", Type: metricsTypeMaximum},
		{MackerelName: "SuccessfulRequestLatency.#.Average", Type: metricsTypeAverage},
	}},
	{CloudWatchName: "ThrottledRequests", Metrics: []metric{
		{MackerelName: "ThrottledRequests.#", Type: metricsTypeSampleCount},
	}},
	{CloudWatchName: "SystemErrors", Metrics: []metric{
		{MackerelName: "SystemErrors.#", Type: metricsTypeSampleCount},
	}},
	{CloudWatchName: "UserErrors", Metrics: []metric{
		{MackerelName: "UserErrors.#", Type: metricsTypeSampleCount},
	}},
	{CloudWatchName: "ReturnedItemCount", Metrics: []metric{
		{MackerelName: "ReturnedItemCount.#", Type: metricsTypeAverage},
	}},
}

// FetchMetrics fetch the metrics
func (p DynamoDBPlugin) FetchMetrics() (map[string]interface{}, error) {
	stats := make(map[string]interface{})

	tableDimensions := []*cloudwatch.Dimension{{
		Name:  aws.String("TableName"),
		Value: aws.String(p.TableName),
	}}
	for _, met := range defaultMetricsGroup {
		dp, err := getLastPointFromCloudWatch(p.CloudWatch, met, tableDimensions)
		if err == nil {
			for _, m := range met.Metrics {
				stats = transformAndAppendDatapoint(dp, m.Type, m.MackerelName, stats)
			}
		} else {
			log.Printf("%s: %s", met, err)
		}
	}

	for _, met := range operationalMetricsGroup {
		operationalStats, err := fetchOperationWildcardMetrics(p.CloudWatch, met, tableDimensions)
		if err == nil {
			for name, s := range operationalStats {
				stats[name] = s
			}
		} else {
			log.Printf("%s: %s", met, err)
		}
	}
	return transformMetrics(stats), nil
}

// TransformMetrics converts some of datapoints to post differences of two metrics
func transformMetrics(stats map[string]interface{}) map[string]interface{} {
	// Although stats are interface{}, those values from cloudwatch.Datapoint are guaranteed to be numerical
	if consumedReadCapacitySum, ok := stats["ConsumedReadCapacityUnitsSum"].(float64); ok {
		stats["ConsumedReadCapacityUnitsNormalized"] = consumedReadCapacitySum / 60.0
	}
	if consumedWriteCapacitySum, ok := stats["ConsumedWriteCapacityUnitsSum"].(float64); ok {
		stats["ConsumedWriteCapacityUnitsNormalized"] = consumedWriteCapacitySum / 60.0
	}
	return stats
}

// GraphDefinition of DynamoDBPlugin
func (p DynamoDBPlugin) GraphDefinition() map[string]mp.Graphs {
	labelPrefix := strings.Title(p.Prefix)
	labelPrefix = strings.Replace(labelPrefix, "-", " ", -1)
	labelPrefix = strings.Replace(labelPrefix, "Dynamodb", "DynamoDB", -1)

	var graphdef = map[string]mp.Graphs{
		"ReadCapacity": {
			Label: (labelPrefix + " Read Capacity Units"),
			Unit:  "float",
			Metrics: []mp.Metrics{
				{Name: "ProvisionedReadCapacityUnits", Label: "Provisioned"},
				{Name: "ConsumedReadCapacityUnitsNormalized", Label: "Consumed"},
				{Name: "ConsumedReadCapacityUnitsAverage", Label: "Consumed (Average per request)"},
			},
		},
		"WriteCapacity": {
			Label: (labelPrefix + " Write Capacity Units"),
			Unit:  "float",
			Metrics: []mp.Metrics{
				{Name: "ProvisionedWriteCapacityUnits", Label: "Provisioned"},
				{Name: "ConsumedWriteCapacityUnitsNormalized", Label: "Consumed"},
				{Name: "ConsumedWriteCapacityUnitsAverage", Label: "Consumed (Average per request)"},
			},
		},
		"ThrottledEvents": {
			Label: (labelPrefix + " Throttle Events"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "ReadThrottleEvents", Label: "Read"},
				{Name: "WriteThrottleEvents", Label: "Write"},
			},
		},
		"ConditionalCheckFailedRequests": {
			Label: (labelPrefix + " ConditionalCheckFailedRequests"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "ConditionalCheckFailedRequests", Label: "Counts"},
			},
		},
		"ThrottledRequests": {
			Label: (labelPrefix + " ThrottledRequests"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "*", Label: "%1", Stacked: true},
			},
		},
		"SystemErrors": {
			Label: (labelPrefix + " SystemErrors"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "*", Label: "%1", Stacked: true},
			},
		},
		"UserErrors": {
			Label: (labelPrefix + " UserErrors"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "*", Label: "%1", Stacked: true},
			},
		},
		"ReturnedItemCount": {
			Label: (labelPrefix + " ReturnedItemCount"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "*", Label: "%1"},
			},
		},
		"SuccessfulRequests": {
			Label: (labelPrefix + " SuccessfulRequests"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "*", Label: "%1"},
			},
		},
		"SuccessfulRequestLatency.#": {
			Label: (labelPrefix + " SuccessfulRequestLatency"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "Minimum", Label: "Min"},
				{Name: "Maximum", Label: "Max"},
				{Name: "Average", Label: "Average"},
			},
		},
	}
	return graphdef
}

// Do the plugin
func Do() {
	optAccessKeyID := flag.String("access-key-id", "", "AWS Access Key ID")
	optSecretAccessKey := flag.String("secret-access-key", "", "AWS Secret Access Key")
	optRegion := flag.String("region", "", "AWS Region")
	optTableName := flag.String("table-name", "", "DynamoDB Table Name")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	optPrefix := flag.String("metric-key-prefix", "dynamodb", "Metric key prefix")
	flag.Parse()

	var plugin DynamoDBPlugin

	plugin.AccessKeyID = *optAccessKeyID
	plugin.SecretAccessKey = *optSecretAccessKey
	plugin.Region = *optRegion
	plugin.TableName = *optTableName
	plugin.Prefix = *optPrefix

	err := plugin.prepare()
	if err != nil {
		log.Fatalln(err)
	}

	helper := mp.NewMackerelPlugin(plugin)
	helper.Tempfile = *optTempfile

	helper.Run()
}
<|endoftext|>