<commit_before>package main

//go:generate go-bindata -pkg $GOPACKAGE -o assets.go -prefix ../../ui/build ../../ui/build/

import (
	"log"
	"strings"

	"github.com/gin-gonic/gin"
)

var DBPath = "incr.db"

// curl http://www.google-analytics.com/__utm.gif | xxd -i
var minimalGIF = []byte{
	0x47, 0x49, 0x46, 0x38, 0x39, 0x61, 0x01, 0x00, 0x01, 0x00, 0x80, 0xff,
	0x00, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x2c, 0x00, 0x00, 0x00, 0x00,
	0x01, 0x00, 0x01, 0x00, 0x00, 0x02, 0x02, 0x44, 0x01, 0x00, 0x3b,
}

func corsHandler(c *gin.Context) {
	c.Writer.Header().Add("Access-Control-Allow-Origin",
		c.Request.Header.Get("Origin"))
	c.Writer.Header().Add("Access-Control-Allow-Credentials", "true")
	c.Writer.Header().Add("Access-Control-Allow-Headers",
		c.Request.Header.Get("Access-Control-Request-Headers"))
	c.Writer.Header().Add("Access-Control-Allow-Methods",
		c.Request.Header.Get("Access-Control-Request-Method"))
	if c.Request.Method == "OPTIONS" {
		c.AbortWithStatus(200)
	} else {
		c.Next()
	}
}

func incr(c *gin.Context, s Store, gif bool) {
	s.Incr(c.Param("ns"), strings.TrimSuffix(c.Param("counter"), ".gif"))
	if gif {
		c.Data(200, "image/gif", minimalGIF)
	} else {
		c.AbortWithStatus(200)
	}
}

func main() {
	s, err := NewStore(DBPath)
	if err != nil {
		log.Fatal(err)
	}

	r := gin.Default()
	r.Use(corsHandler)
	r.GET("/api/:ns", func(c *gin.Context) {
		if list, err := s.List(c.Param("ns")); err != nil {
			c.AbortWithStatus(500)
		} else {
			c.JSON(200, list)
		}
	})
	r.GET("/api/:ns/:counter", func(c *gin.Context) {
		if strings.HasSuffix(c.Param("counter"), ".gif") {
			incr(c, s, true)
		} else {
			if counter, err := s.Query(c.Param("ns"), c.Param("counter")); err != nil {
				log.Println(err)
				c.AbortWithStatus(500)
			} else {
				result := gin.H{"now": counter.Atime}
				for i, bucket := range Buckets {
					result[bucket.Name] = counter.Values[i]
				}
				c.JSON(200, result)
			}
		}
	})
	r.POST("/api/:ns/:counter", func(c *gin.Context) {
		incr(c, s, false)
	})
	r.NoRoute(func(c *gin.Context) {
		log.Println(c.Request.URL.Path)
		switch c.Request.URL.Path {
		case "/":
			fallthrough
		case "/index.html":
			c.Data(200, "text/html", MustAsset("index.html"))
		case "/bundle.js":
			c.Data(200, "application/javascript", MustAsset("bundle.js"))
		}
	})
	r.Run() // listen and server on 0.0.0.0:8080
}
<commit_msg>added db path env config variable<commit_after>package main

//go:generate go-bindata -pkg $GOPACKAGE -o assets.go -prefix ../../ui/build ../../ui/build/

import (
	"log"
	"os"
	"strings"

	"github.com/gin-gonic/gin"
)

var DBPath = "incr.db"

// curl http://www.google-analytics.com/__utm.gif | xxd -i
var minimalGIF = []byte{
	0x47, 0x49, 0x46, 0x38, 0x39, 0x61, 0x01, 0x00, 0x01, 0x00, 0x80, 0xff,
	0x00, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x2c, 0x00, 0x00, 0x00, 0x00,
	0x01, 0x00, 0x01, 0x00, 0x00, 0x02, 0x02, 0x44, 0x01, 0x00, 0x3b,
}

func corsHandler(c *gin.Context) {
	c.Writer.Header().Add("Access-Control-Allow-Origin",
		c.Request.Header.Get("Origin"))
	c.Writer.Header().Add("Access-Control-Allow-Credentials", "true")
	c.Writer.Header().Add("Access-Control-Allow-Headers",
		c.Request.Header.Get("Access-Control-Request-Headers"))
	c.Writer.Header().Add("Access-Control-Allow-Methods",
		c.Request.Header.Get("Access-Control-Request-Method"))
	if c.Request.Method == "OPTIONS" {
		c.AbortWithStatus(200)
	} else {
		c.Next()
	}
}

func incr(c *gin.Context, s Store, gif bool) {
	s.Incr(c.Param("ns"), strings.TrimSuffix(c.Param("counter"), ".gif"))
	if gif {
		c.Data(200, "image/gif", minimalGIF)
	} else {
		c.AbortWithStatus(200)
	}
}

func main() {
	if db := os.Getenv("INCRDB"); db != "" {
		DBPath = db
	}

	s, err := NewStore(DBPath)
	if err != nil {
		log.Fatal(err)
	}

	r := gin.Default()
	r.Use(corsHandler)
	r.GET("/api/:ns", func(c *gin.Context) {
		if list, err := s.List(c.Param("ns")); err != nil {
			c.AbortWithStatus(500)
		} else {
			c.JSON(200, list)
		}
	})
	r.GET("/api/:ns/:counter", func(c *gin.Context) {
		if strings.HasSuffix(c.Param("counter"), ".gif") {
			incr(c, s, true)
		} else {
			if counter, err := s.Query(c.Param("ns"), c.Param("counter")); err != nil {
				log.Println(err)
				c.AbortWithStatus(500)
			} else {
				result := gin.H{"now": counter.Atime}
				for i, bucket := range Buckets {
					result[bucket.Name] = counter.Values[i]
				}
				c.JSON(200, result)
			}
		}
	})
	r.POST("/api/:ns/:counter", func(c *gin.Context) {
		incr(c, s, false)
	})
	r.NoRoute(func(c *gin.Context) {
		log.Println(c.Request.URL.Path)
		switch c.Request.URL.Path {
		case "/":
			fallthrough
		case "/index.html":
			c.Data(200, "text/html", MustAsset("index.html"))
		case "/bundle.js":
			c.Data(200, "application/javascript", MustAsset("bundle.js"))
		}
	})
	r.Run() // listen and server on 0.0.0.0:8080
}
<|endoftext|>


<commit_before>// Copyright 2015 The rkt Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package networking

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"

	"github.com/appc/spec/schema/types"
	"github.com/hashicorp/errwrap"

	"github.com/containernetworking/cni/pkg/ns"

	"github.com/coreos/rkt/common"
	"github.com/coreos/rkt/networking/netinfo"
)

const (
	// Suffix to LocalConfigDir path, where users place their net configs
	UserNetPathSuffix = "net.d"

	// Default net path relative to stage1 root
	DefaultNetPath           = "etc/rkt/net.d/99-default.conf"
	DefaultRestrictedNetPath = "etc/rkt/net.d/99-default-restricted.conf"
)

// "base" struct that's populated from the beginning
// describing the environment in which the pod
// is running in
type podEnv struct {
	podRoot      string
	podID        types.UUID
	netsLoadList common.NetList
	localConfig  string
	podNS        ns.NetNS
}

type activeNet struct {
	confBytes []byte
	conf      *NetConf
	runtime   *netinfo.NetInfo
}

// Loads nets specified by user and default one from stage1
func (e *podEnv) loadNets() ([]activeNet, error) {
	nets, err := loadUserNets(e.localConfig, e.netsLoadList)
	if err != nil {
		return nil, err
	}

	if e.netsLoadList.None() {
		return nets, nil
	}

	if !netExists(nets, "default") && !netExists(nets, "default-restricted") {
		var defaultNet string
		if e.netsLoadList.Specific("default") || e.netsLoadList.All() {
			defaultNet = DefaultNetPath
		} else {
			defaultNet = DefaultRestrictedNetPath
		}
		defPath := path.Join(common.Stage1RootfsPath(e.podRoot), defaultNet)
		n, err := loadNet(defPath)
		if err != nil {
			return nil, err
		}
		nets = append(nets, *n)
	}

	missing := missingNets(e.netsLoadList, nets)
	if len(missing) > 0 {
		return nil, fmt.Errorf("networks not found: %v", strings.Join(missing, ", "))
	}

	return nets, nil
}

func (e *podEnv) podNSFilePath() string {
	return filepath.Join(e.podRoot, "netns")
}

func (e *podEnv) podNSPathLoad() (string, error) {
	podNSPath, err := ioutil.ReadFile(e.podNSFilePath())
	if err != nil {
		return "", err
	}

	return string(podNSPath), nil
}

func (e *podEnv) podNSLoad() (ns.NetNS, error) {
	podNSPath, err := e.podNSPathLoad()
	if err != nil {
		return nil, err
	}

	podNS, err := ns.GetNS(podNSPath)
	if err != nil {
		return nil, err
	}
	return podNS, nil
}

func (e *podEnv) podNSPathSave() error {
	podNSFile, err := os.OpenFile(e.podNSFilePath(), os.O_WRONLY|os.O_CREATE, 0)
	if err != nil {
		return err
	}
	defer podNSFile.Close()

	if _, err = io.WriteString(podNSFile, e.podNS.Path()); err != nil {
		return err
	}

	return nil
}

func (e *podEnv) netDir() string {
	return filepath.Join(e.podRoot, "net")
}

func (e *podEnv) setupNets(nets []activeNet) error {
	err := os.MkdirAll(e.netDir(), 0755)
	if err != nil {
		return err
	}

	i := 0
	defer func() {
		if err != nil {
			e.teardownNets(nets[:i])
		}
	}()

	n := activeNet{}
	for i, n = range nets {
		stderr.Printf("loading network %v with type %v", n.conf.Name, n.conf.Type)

		n.runtime.IfName = fmt.Sprintf(IfNamePattern, i)
		if n.runtime.ConfPath, err = copyFileToDir(n.runtime.ConfPath, e.netDir()); err != nil {
			return errwrap.Wrap(fmt.Errorf("error copying %q to %q", n.runtime.ConfPath, e.netDir()), err)
		}

		n.runtime.IP, n.runtime.HostIP, err = e.netPluginAdd(&n, e.podNS.Path())
		if err != nil {
			return errwrap.Wrap(fmt.Errorf("error adding network %q", n.conf.Name), err)
		}
	}
	return nil
}

func (e *podEnv) teardownNets(nets []activeNet) {

	for i := len(nets) - 1; i >= 0; i-- {
		stderr.Printf("teardown - executing net-plugin %v", nets[i].conf.Type)

		podNSpath := ""
		if e.podNS != nil {
			podNSpath = e.podNS.Path()
		}

		err := e.netPluginDel(&nets[i], podNSpath)
		if err != nil {
			stderr.PrintE(fmt.Sprintf("error deleting %q", nets[i].conf.Name), err)
		}

		// Delete the conf file to signal that the network was
		// torn down (or at least attempted to)
		if err = os.Remove(nets[i].runtime.ConfPath); err != nil {
			stderr.PrintE(fmt.Sprintf("error deleting %q", nets[i].runtime.ConfPath), err)
		}
	}
}

func listFiles(dir string) ([]string, error) {
	dirents, err := ioutil.ReadDir(dir)
	switch {
	case err == nil:
	case os.IsNotExist(err):
		return nil, nil
	default:
		return nil, err
	}

	var files []string
	for _, dent := range dirents {
		if dent.IsDir() {
			continue
		}

		files = append(files, dent.Name())
	}

	return files, nil
}

func netExists(nets []activeNet, name string) bool {
	for _, n := range nets {
		if n.conf.Name == name {
			return true
		}
	}
	return false
}

func loadNet(filepath string) (*activeNet, error) {
	bytes, err := ioutil.ReadFile(filepath)
	if err != nil {
		return nil, err
	}

	n := &NetConf{}
	if err = json.Unmarshal(bytes, n); err != nil {
		return nil, errwrap.Wrap(fmt.Errorf("error loading %v", filepath), err)
	}

	return &activeNet{
		confBytes: bytes,
		conf:      n,
		runtime: &netinfo.NetInfo{
			NetName:  n.Name,
			ConfPath: filepath,
		},
	}, nil
}

func copyFileToDir(src, dstdir string) (string, error) {
	dst := filepath.Join(dstdir, filepath.Base(src))

	s, err := os.Open(src)
	if err != nil {
		return "", err
	}
	defer s.Close()

	d, err := os.Create(dst)
	if err != nil {
		return "", err
	}
	defer d.Close()

	_, err = io.Copy(d, s)
	return dst, err
}

func loadUserNets(localConfig string, netsLoadList common.NetList) ([]activeNet, error) {
	if netsLoadList.None() {
		stderr.Printf("networking namespace with loopback only")
		return nil, nil
	}

	userNetPath := filepath.Join(localConfig, UserNetPathSuffix)
	stderr.Printf("loading networks from %v", userNetPath)

	files, err := listFiles(userNetPath)
	if err != nil {
		return nil, err
	}
	sort.Strings(files)
	nets := make([]activeNet, 0, len(files))

	for _, filename := range files {
		filepath := filepath.Join(userNetPath, filename)

		if !strings.HasSuffix(filepath, ".conf") {
			continue
		}

		n, err := loadNet(filepath)
		if err != nil {
			return nil, err
		}

		if !(netsLoadList.All() || netsLoadList.Specific(n.conf.Name)) {
			continue
		}

		if n.conf.Name == "default" ||
			n.conf.Name == "default-restricted" {
			stderr.Printf(`overriding %q network with %v`, n.conf.Name, filename)
		}

		if netExists(nets, n.conf.Name) {
			stderr.Printf("%q network already defined, ignoring %v", n.conf.Name, filename)
			continue
		}

		n.runtime.Args = netsLoadList.SpecificArgs(n.conf.Name)

		nets = append(nets, *n)
	}

	return nets, nil
}

func missingNets(defined common.NetList, loaded []activeNet) []string {
	diff := make(map[string]struct{})
	for _, n := range defined.StringsOnlyNames() {
		if n != "all" {
			diff[n] = struct{}{}
		}
	}

	for _, an := range loaded {
		delete(diff, an.conf.Name)
	}

	var missing []string
	for n := range diff {
		missing = append(missing, n)
	}
	return missing
}
<commit_msg>networking: use error types to check NS loading<commit_after>// Copyright 2015 The rkt Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package networking

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"

	"github.com/appc/spec/schema/types"
	"github.com/hashicorp/errwrap"

	"github.com/containernetworking/cni/pkg/ns"

	"github.com/coreos/rkt/common"
	"github.com/coreos/rkt/networking/netinfo"
)

const (
	// Suffix to LocalConfigDir path, where users place their net configs
	UserNetPathSuffix = "net.d"

	// Default net path relative to stage1 root
	DefaultNetPath           = "etc/rkt/net.d/99-default.conf"
	DefaultRestrictedNetPath = "etc/rkt/net.d/99-default-restricted.conf"
)

// "base" struct that's populated from the beginning
// describing the environment in which the pod
// is running in
type podEnv struct {
	podRoot      string
	podID        types.UUID
	netsLoadList common.NetList
	localConfig  string
	podNS        ns.NetNS
}

type activeNet struct {
	confBytes []byte
	conf      *NetConf
	runtime   *netinfo.NetInfo
}

// Loads nets specified by user and default one from stage1
func (e *podEnv) loadNets() ([]activeNet, error) {
	nets, err := loadUserNets(e.localConfig, e.netsLoadList)
	if err != nil {
		return nil, err
	}

	if e.netsLoadList.None() {
		return nets, nil
	}

	if !netExists(nets, "default") && !netExists(nets, "default-restricted") {
		var defaultNet string
		if e.netsLoadList.Specific("default") || e.netsLoadList.All() {
			defaultNet = DefaultNetPath
		} else {
			defaultNet = DefaultRestrictedNetPath
		}
		defPath := path.Join(common.Stage1RootfsPath(e.podRoot), defaultNet)
		n, err := loadNet(defPath)
		if err != nil {
			return nil, err
		}
		nets = append(nets, *n)
	}

	missing := missingNets(e.netsLoadList, nets)
	if len(missing) > 0 {
		return nil, fmt.Errorf("networks not found: %v", strings.Join(missing, ", "))
	}

	return nets, nil
}

func (e *podEnv) podNSFilePath() string {
	return filepath.Join(e.podRoot, "netns")
}

func (e *podEnv) podNSPathLoad() (string, error) {
	podNSPath, err := ioutil.ReadFile(e.podNSFilePath())
	if err != nil {
		return "", err
	}

	return string(podNSPath), nil
}

func podNSerrorOK(podNSPath string, err error) bool {
	switch err.(type) {
	case ns.NSPathNotExistErr:
		return true
	case ns.NSPathNotNSErr:
		return true

	default:
		if os.IsNotExist(err) {
			return true
		}
		return false
	}
}

func (e *podEnv) podNSLoad() (ns.NetNS, error) {
	podNSPath, err := e.podNSPathLoad()
	if err != nil && !podNSerrorOK(podNSPath, err) {
		return nil, err
	} else {
		podNS, err := ns.GetNS(podNSPath)
		if err != nil && !podNSerrorOK(podNSPath, err) {
			return nil, err
		}
		return podNS, nil
	}
}

func (e *podEnv) podNSPathSave() error {
	podNSFile, err := os.OpenFile(e.podNSFilePath(), os.O_WRONLY|os.O_CREATE, 0)
	if err != nil {
		return err
	}
	defer podNSFile.Close()

	if _, err = io.WriteString(podNSFile, e.podNS.Path()); err != nil {
		return err
	}

	return nil
}

func (e *podEnv) netDir() string {
	return filepath.Join(e.podRoot, "net")
}

func (e *podEnv) setupNets(nets []activeNet) error {
	err := os.MkdirAll(e.netDir(), 0755)
	if err != nil {
		return err
	}

	i := 0
	defer func() {
		if err != nil {
			e.teardownNets(nets[:i])
		}
	}()

	n := activeNet{}
	for i, n = range nets {
		stderr.Printf("loading network %v with type %v", n.conf.Name, n.conf.Type)

		n.runtime.IfName = fmt.Sprintf(IfNamePattern, i)
		if n.runtime.ConfPath, err = copyFileToDir(n.runtime.ConfPath, e.netDir()); err != nil {
			return errwrap.Wrap(fmt.Errorf("error copying %q to %q", n.runtime.ConfPath, e.netDir()), err)
		}

		n.runtime.IP, n.runtime.HostIP, err = e.netPluginAdd(&n, e.podNS.Path())
		if err != nil {
			return errwrap.Wrap(fmt.Errorf("error adding network %q", n.conf.Name), err)
		}
	}
	return nil
}

func (e *podEnv) teardownNets(nets []activeNet) {

	for i := len(nets) - 1; i >= 0; i-- {
		stderr.Printf("teardown - executing net-plugin %v", nets[i].conf.Type)

		podNSpath := ""
		if e.podNS != nil {
			podNSpath = e.podNS.Path()
		}

		err := e.netPluginDel(&nets[i], podNSpath)
		if err != nil {
			stderr.PrintE(fmt.Sprintf("error deleting %q", nets[i].conf.Name), err)
		}

		// Delete the conf file to signal that the network was
		// torn down (or at least attempted to)
		if err = os.Remove(nets[i].runtime.ConfPath); err != nil {
			stderr.PrintE(fmt.Sprintf("error deleting %q", nets[i].runtime.ConfPath), err)
		}
	}
}

func listFiles(dir string) ([]string, error) {
	dirents, err := ioutil.ReadDir(dir)
	switch {
	case err == nil:
	case os.IsNotExist(err):
		return nil, nil
	default:
		return nil, err
	}

	var files []string
	for _, dent := range dirents {
		if dent.IsDir() {
			continue
		}

		files = append(files, dent.Name())
	}

	return files, nil
}

func netExists(nets []activeNet, name string) bool {
	for _, n := range nets {
		if n.conf.Name == name {
			return true
		}
	}
	return false
}

func loadNet(filepath string) (*activeNet, error) {
	bytes, err := ioutil.ReadFile(filepath)
	if err != nil {
		return nil, err
	}

	n := &NetConf{}
	if err = json.Unmarshal(bytes, n); err != nil {
		return nil, errwrap.Wrap(fmt.Errorf("error loading %v", filepath), err)
	}

	return &activeNet{
		confBytes: bytes,
		conf:      n,
		runtime: &netinfo.NetInfo{
			NetName:  n.Name,
			ConfPath: filepath,
		},
	}, nil
}

func copyFileToDir(src, dstdir string) (string, error) {
	dst := filepath.Join(dstdir, filepath.Base(src))

	s, err := os.Open(src)
	if err != nil {
		return "", err
	}
	defer s.Close()

	d, err := os.Create(dst)
	if err != nil {
		return "", err
	}
	defer d.Close()

	_, err = io.Copy(d, s)
	return dst, err
}

func loadUserNets(localConfig string, netsLoadList common.NetList) ([]activeNet, error) {
	if netsLoadList.None() {
		stderr.Printf("networking namespace with loopback only")
		return nil, nil
	}

	userNetPath := filepath.Join(localConfig, UserNetPathSuffix)
	stderr.Printf("loading networks from %v", userNetPath)

	files, err := listFiles(userNetPath)
	if err != nil {
		return nil, err
	}
	sort.Strings(files)
	nets := make([]activeNet, 0, len(files))

	for _, filename := range files {
		filepath := filepath.Join(userNetPath, filename)

		if !strings.HasSuffix(filepath, ".conf") {
			continue
		}

		n, err := loadNet(filepath)
		if err != nil {
			return nil, err
		}

		if !(netsLoadList.All() || netsLoadList.Specific(n.conf.Name)) {
			continue
		}

		if n.conf.Name == "default" ||
			n.conf.Name == "default-restricted" {
			stderr.Printf(`overriding %q network with %v`, n.conf.Name, filename)
		}

		if netExists(nets, n.conf.Name) {
			stderr.Printf("%q network already defined, ignoring %v", n.conf.Name, filename)
			continue
		}

		n.runtime.Args = netsLoadList.SpecificArgs(n.conf.Name)

		nets = append(nets, *n)
	}

	return nets, nil
}

func missingNets(defined common.NetList, loaded []activeNet) []string {
	diff := make(map[string]struct{})
	for _, n := range defined.StringsOnlyNames() {
		if n != "all" {
			diff[n] = struct{}{}
		}
	}

	for _, an := range loaded {
		delete(diff, an.conf.Name)
	}

	var missing []string
	for n := range diff {
		missing = append(missing, n)
	}
	return missing
}
<|endoftext|>


<commit_before>package ip2region

import (
	"fmt"
	"strings"
	"sync"

	"github.com/admpub/ip2region/binding/golang/ip2region"
	"github.com/webx-top/echo"
)

var (
	region   *ip2region.Ip2Region
	dictFile string
	once     sync.Once
)

func init() {
	dictFile = echo.Wd() + echo.FilePathSeparator + `data` + echo.FilePathSeparator + `ip2region` + echo.FilePathSeparator + `ip2region.db`

}

func SetDictFile(f string) {
	dictFile = f
}

func Initialize() (err error) {
	if region == nil {
		region, err = ip2region.New(dictFile)
	}
	return
}

func IPInfo(ip string) (info ip2region.IpInfo, err error) {
	if len(ip) == 0 {
		return
	}
	once.Do(func() {
		err = Initialize()
	})
	if err != nil {
		return
	}
	info, err = region.MemorySearch(ip)
	return
}

func Stringify(info ip2region.IpInfo) string {
	var (
		formats []string
		args    []interface{}
	)
	if len(info.Country) > 0 && info.Country != `0` {
		formats = append(formats, `"国家":%q`)
		args = append(args, info.Country)
	}
	if len(info.Region) > 0 && info.Region != `0` {
		formats = append(formats, `"地区":%q`)
		args = append(args, info.Region)
	}
	if len(info.Province) > 0 && info.Province != `0` {
		formats = append(formats, `"省份":%q`)
		args = append(args, info.Province)
	}
	if len(info.City) > 0 && info.City != `0` {
		formats = append(formats, `"城市":%q`)
		args = append(args, info.City)
	}
	if len(info.ISP) > 0 && info.ISP != `0` {
		formats = append(formats, `"线路":%q`)
		args = append(args, info.ISP)
	}
	return fmt.Sprintf(`{`+strings.Join(formats, `,`)+`}`, args...)
}
<commit_msg>update<commit_after>package ip2region

import (
	"fmt"
	"strings"
	"sync"

	"github.com/admpub/ip2region/binding/golang/ip2region"
	"github.com/webx-top/echo"
)

var (
	region   *ip2region.Ip2Region
	dictFile string
	once     sync.Once
)

func init() {
	dictFile = echo.Wd() + echo.FilePathSeparator + `data` + echo.FilePathSeparator + `ip2region` + echo.FilePathSeparator + `ip2region.db`

}

func SetDictFile(f string) {
	dictFile = f
}

func Initialize() (err error) {
	if region == nil {
		region, err = ip2region.New(dictFile)
	}
	return
}

func IsInitialized() bool {
	return region != nil
}

func IPInfo(ip string) (info ip2region.IpInfo, err error) {
	if len(ip) == 0 {
		return
	}
	once.Do(func() {
		err = Initialize()
	})
	if err != nil {
		return
	}
	info, err = region.MemorySearch(ip)
	return
}

func Stringify(info ip2region.IpInfo) string {
	var (
		formats []string
		args    []interface{}
	)
	if len(info.Country) > 0 && info.Country != `0` {
		formats = append(formats, `"国家":%q`)
		args = append(args, info.Country)
	}
	if len(info.Region) > 0 && info.Region != `0` {
		formats = append(formats, `"地区":%q`)
		args = append(args, info.Region)
	}
	if len(info.Province) > 0 && info.Province != `0` {
		formats = append(formats, `"省份":%q`)
		args = append(args, info.Province)
	}
	if len(info.City) > 0 && info.City != `0` {
		formats = append(formats, `"城市":%q`)
		args = append(args, info.City)
	}
	if len(info.ISP) > 0 && info.ISP != `0` {
		formats = append(formats, `"线路":%q`)
		args = append(args, info.ISP)
	}
	return fmt.Sprintf(`{`+strings.Join(formats, `,`)+`}`, args...)
}
<|endoftext|>


<commit_before>// Leaving this file in integration/v7/global for now, since it can be used as the test file for `cf7 org-quota` command
package global

import (
	"code.cloudfoundry.org/cli/integration/helpers"
	"code.cloudfoundry.org/cli/integration/v7/global"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("quota command", func() {
	var (
		quotaName string
	)

	BeforeEach(func() {
		quotaName = helpers.QuotaName()
	})

	Describe("help", func() {
		When("--help flag is set", func() {
			It("Displays command usage to output", func() {
				session := helpers.CF("quota", "--help")
				Eventually(session).Should(Say("NAME:"))
				Eventually(session).Should(Say("quota - Show quota info"))
				Eventually(session).Should(Say("USAGE:"))
				Eventually(session).Should(Say("cf quota QUOTA"))
				Eventually(session).Should(Say("SEE ALSO:"))
				Eventually(session).Should(Say("org, quotas"))
				Eventually(session).Should(Exit(0))
			})
		})
	})

	When("the environment is not setup correctly", func() {
		It("fails with the appropriate errors", func() {
			helpers.CheckEnvironmentTargetedCorrectly(false, false, ReadOnlyOrg, "org", "org-name")
		})
	})

	When("the environment is set up correctly", func() {
		BeforeEach(func() {
			helpers.LoginCF()
		})

		When("the quota does not exist", func() {
			It("displays quota not found and exits 1", func() {
				session := helpers.CF("quota", quotaName)
				userName, _ := helpers.GetCredentials()
				Eventually(session).Should(Say(`Getting quota %s info as %s\.\.\.`, quotaName, userName))
				Eventually(session).Should(Say("FAILED"))
				Eventually(session).Should(Say("Quota %s not found", quotaName))
				Eventually(session).Should(Exit(1))
			})
		})

		When("the quota exists", func() {
			When("no flags are used", func() {
				It("displays a table with quota names and their values and exits 0", func() {
					session := helpers.CF("quota", "default")
					userName, _ := helpers.GetCredentials()
					Eventually(session).Should(Say(`Getting quota %s info as %s\.\.\.`, "default", userName))
					Eventually(session).Should(Say("OK"))

					Eventually(session).Should(Say(`Total Memory\s+100G`))
					Eventually(session).Should(Say(`Instance Memory\s+unlimited`))
					Eventually(session).Should(Say(`Routes\s+1000`))
					Eventually(session).Should(Say(`Services\s+unlimited`))
					Eventually(session).Should(Say(`Paid service plans\s+allowed`))
					Eventually(session).Should(Say(`App instance limit\s+unlimited`))
					Eventually(session).Should(Say(`Reserved Route Ports\s+100`))

					Eventually(session).Should(Exit(0))
				})
			})
		})
	})
})
<commit_msg>Remove accidental v7 import in v6 integration test<commit_after>package global

import (
	"code.cloudfoundry.org/cli/integration/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("quota command", func() {
	var (
		quotaName string
	)

	BeforeEach(func() {
		quotaName = helpers.QuotaName()
	})

	Describe("help", func() {
		When("--help flag is set", func() {
			It("Displays command usage to output", func() {
				session := helpers.CF("quota", "--help")
				Eventually(session).Should(Say("NAME:"))
				Eventually(session).Should(Say("quota - Show quota info"))
				Eventually(session).Should(Say("USAGE:"))
				Eventually(session).Should(Say("cf quota QUOTA"))
				Eventually(session).Should(Say("SEE ALSO:"))
				Eventually(session).Should(Say("org, quotas"))
				Eventually(session).Should(Exit(0))
			})
		})
	})

	When("the environment is not setup correctly", func() {
		It("fails with the appropriate errors", func() {
			helpers.CheckEnvironmentTargetedCorrectly(false, false, ReadOnlyOrg, "org", "org-name")
		})
	})

	When("the environment is set up correctly", func() {
		BeforeEach(func() {
			helpers.LoginCF()
		})

		When("the quota does not exist", func() {
			It("displays quota not found and exits 1", func() {
				session := helpers.CF("quota", quotaName)
				userName, _ := helpers.GetCredentials()
				Eventually(session).Should(Say(`Getting quota %s info as %s\.\.\.`, quotaName, userName))
				Eventually(session).Should(Say("FAILED"))
				Eventually(session).Should(Say("Quota %s not found", quotaName))
				Eventually(session).Should(Exit(1))
			})
		})

		When("the quota exists", func() {
			When("no flags are used", func() {
				It("displays a table with quota names and their values and exits 0", func() {
					session := helpers.CF("quota", "default")
					userName, _ := helpers.GetCredentials()
					Eventually(session).Should(Say(`Getting quota %s info as %s\.\.\.`, "default", userName))
					Eventually(session).Should(Say("OK"))

					Eventually(session).Should(Say(`Total Memory\s+100G`))
					Eventually(session).Should(Say(`Instance Memory\s+unlimited`))
					Eventually(session).Should(Say(`Routes\s+1000`))
					Eventually(session).Should(Say(`Services\s+unlimited`))
					Eventually(session).Should(Say(`Paid service plans\s+allowed`))
					Eventually(session).Should(Say(`App instance limit\s+unlimited`))
					Eventually(session).Should(Say(`Reserved Route Ports\s+100`))

					Eventually(session).Should(Exit(0))
				})
			})
		})
	})
})
<|endoftext|>


<commit_before>package httptracker

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"strconv"
	"time"

	"github.com/cenkalti/rain/internal/logger"
	"github.com/cenkalti/rain/internal/tracker"
	"github.com/zeebo/bencode"
)

type HTTPTracker struct {
	rawURL            string
	url               *url.URL
	log               logger.Logger
	http              *http.Client
	transport         *http.Transport
	trackerID         string
	userAgent         string
	maxResponseLength int64
}

var _ tracker.Tracker = (*HTTPTracker)(nil)

func New(rawURL string, u *url.URL, timeout time.Duration, t *http.Transport, userAgent string, maxResponseLength int64) *HTTPTracker {
	return &HTTPTracker{
		rawURL:            rawURL,
		url:               u,
		log:               logger.New("tracker " + u.String()),
		transport:         t,
		userAgent:         userAgent,
		maxResponseLength: maxResponseLength,
		http: &http.Client{
			Timeout:   timeout,
			Transport: t,
		},
	}
}

func (t *HTTPTracker) URL() string {
	return t.rawURL
}

func (t *HTTPTracker) Announce(ctx context.Context, req tracker.AnnounceRequest) (*tracker.AnnounceResponse, error) {
	q := t.url.Query()
	q.Set("info_hash", string(req.Torrent.InfoHash[:]))
	q.Set("peer_id", string(req.Torrent.PeerID[:]))
	q.Set("port", strconv.FormatUint(uint64(req.Torrent.Port), 10))
	q.Set("uploaded", strconv.FormatInt(req.Torrent.BytesUploaded, 10))
	q.Set("downloaded", strconv.FormatInt(req.Torrent.BytesDownloaded, 10))
	q.Set("left", strconv.FormatInt(req.Torrent.BytesLeft, 10))
	q.Set("compact", "1")
	q.Set("no_peer_id", "1")
	q.Set("numwant", strconv.Itoa(req.NumWant))
	if req.Event != tracker.EventNone {
		q.Set("event", req.Event.String())
	}
	if t.trackerID != "" {
		q.Set("trackerid", t.trackerID)
	}

	u := t.url
	u.RawQuery = q.Encode()
	t.log.Debugf("making request to: %q", u.String())

	httpReq := &http.Request{
		Method:     http.MethodGet,
		URL:        u,
		Proto:      "HTTP/1.1",
		ProtoMajor: 1,
		ProtoMinor: 1,
		Header:     make(http.Header),
		Host:       u.Host,
	}
	httpReq = httpReq.WithContext(ctx)

	httpReq.Header.Set("User-Agent", t.userAgent)

	doReq := func() ([]byte, error) {
		resp, err := t.http.Do(httpReq)
		if err != nil {
			return nil, err
		}
		defer resp.Body.Close()
		if resp.StatusCode != 200 {
			data, _ := ioutil.ReadAll(resp.Body)
			return nil, fmt.Errorf("status not 200 OK (status: %d body: %q)", resp.StatusCode, string(data))
		}
		if resp.ContentLength > t.maxResponseLength {
			return nil, fmt.Errorf("tracker respsonse too large: %d", resp.ContentLength)
		}
		r := io.LimitReader(resp.Body, t.maxResponseLength)
		return ioutil.ReadAll(r)
	}

	body, err := doReq()
	if uerr, ok := err.(*url.Error); ok && uerr.Err == context.Canceled {
		return nil, context.Canceled
	}

	var response announceResponse
	err = bencode.DecodeBytes(body, &response)
	if err != nil {
		return nil, err
	}

	if response.WarningMessage != "" {
		t.log.Warning(response.WarningMessage)
	}
	if response.FailureReason != "" {
		return nil, tracker.Error(response.FailureReason)
	}

	if response.TrackerID != "" {
		t.trackerID = response.TrackerID
	}

	// Peers may be in binary or dictionary model.
	var peers []*net.TCPAddr
	if len(response.Peers) > 0 {
		if response.Peers[0] == 'l' {
			peers, err = parsePeersDictionary(response.Peers)
		} else {
			var b []byte
			err = bencode.DecodeBytes(response.Peers, &b)
			if err != nil {
				return nil, err
			}
			peers, err = tracker.DecodePeersCompact(b)
		}
	}
	if err != nil {
		return nil, err
	}

	// Filter external IP
	if len(response.ExternalIP) != 0 {
		for i, p := range peers {
			if bytes.Equal(p.IP[:], response.ExternalIP) {
				peers[i], peers = peers[len(peers)-1], peers[:len(peers)-1]
				break
			}
		}
	}

	return &tracker.AnnounceResponse{
		Interval:    time.Duration(response.Interval) * time.Second,
		MinInterval: time.Duration(response.MinInterval) * time.Second,
		Leechers:    response.Incomplete,
		Seeders:     response.Complete,
		Peers:       peers,
	}, nil
}

func parsePeersDictionary(b bencode.RawMessage) ([]*net.TCPAddr, error) {
	var peers []struct {
		IP   string `bencode:"ip"`
		Port uint16 `bencode:"port"`
	}
	err := bencode.DecodeBytes(b, &peers)
	if err != nil {
		return nil, err
	}

	addrs := make([]*net.TCPAddr, len(peers))
	for i, p := range peers {
		pe := &net.TCPAddr{IP: net.ParseIP(p.IP), Port: int(p.Port)}
		addrs[i] = pe
	}
	return addrs, err
}
<commit_msg>fix race in http tracker announce<commit_after>package httptracker

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"strconv"
	"time"

	"github.com/cenkalti/rain/internal/logger"
	"github.com/cenkalti/rain/internal/tracker"
	"github.com/zeebo/bencode"
)

type HTTPTracker struct {
	rawURL            string
	url               *url.URL
	log               logger.Logger
	http              *http.Client
	transport         *http.Transport
	trackerID         string
	userAgent         string
	maxResponseLength int64
}

var _ tracker.Tracker = (*HTTPTracker)(nil)

func New(rawURL string, u *url.URL, timeout time.Duration, t *http.Transport, userAgent string, maxResponseLength int64) *HTTPTracker {
	return &HTTPTracker{
		rawURL:            rawURL,
		url:               u,
		log:               logger.New("tracker " + u.String()),
		transport:         t,
		userAgent:         userAgent,
		maxResponseLength: maxResponseLength,
		http: &http.Client{
			Timeout:   timeout,
			Transport: t,
		},
	}
}

func (t *HTTPTracker) URL() string {
	return t.rawURL
}

func (t *HTTPTracker) Announce(ctx context.Context, req tracker.AnnounceRequest) (*tracker.AnnounceResponse, error) {
	u := *t.url

	q := u.Query()
	q.Set("info_hash", string(req.Torrent.InfoHash[:]))
	q.Set("peer_id", string(req.Torrent.PeerID[:]))
	q.Set("port", strconv.FormatUint(uint64(req.Torrent.Port), 10))
	q.Set("uploaded", strconv.FormatInt(req.Torrent.BytesUploaded, 10))
	q.Set("downloaded", strconv.FormatInt(req.Torrent.BytesDownloaded, 10))
	q.Set("left", strconv.FormatInt(req.Torrent.BytesLeft, 10))
	q.Set("compact", "1")
	q.Set("no_peer_id", "1")
	q.Set("numwant", strconv.Itoa(req.NumWant))
	if req.Event != tracker.EventNone {
		q.Set("event", req.Event.String())
	}
	if t.trackerID != "" {
		q.Set("trackerid", t.trackerID)
	}

	u.RawQuery = q.Encode()
	t.log.Debugf("making request to: %q", u.String())

	httpReq := &http.Request{
		Method:     http.MethodGet,
		URL:        &u,
		Proto:      "HTTP/1.1",
		ProtoMajor: 1,
		ProtoMinor: 1,
		Header:     make(http.Header),
		Host:       u.Host,
	}
	httpReq = httpReq.WithContext(ctx)

	httpReq.Header.Set("User-Agent", t.userAgent)

	doReq := func() ([]byte, error) {
		resp, err := t.http.Do(httpReq)
		if err != nil {
			return nil, err
		}
		defer resp.Body.Close()
		if resp.StatusCode != 200 {
			data, _ := ioutil.ReadAll(resp.Body)
			return nil, fmt.Errorf("status not 200 OK (status: %d body: %q)", resp.StatusCode, string(data))
		}
		if resp.ContentLength > t.maxResponseLength {
			return nil, fmt.Errorf("tracker respsonse too large: %d", resp.ContentLength)
		}
		r := io.LimitReader(resp.Body, t.maxResponseLength)
		return ioutil.ReadAll(r)
	}

	body, err := doReq()
	if uerr, ok := err.(*url.Error); ok && uerr.Err == context.Canceled {
		return nil, context.Canceled
	}

	var response announceResponse
	err = bencode.DecodeBytes(body, &response)
	if err != nil {
		return nil, err
	}

	if response.WarningMessage != "" {
		t.log.Warning(response.WarningMessage)
	}
	if response.FailureReason != "" {
		return nil, tracker.Error(response.FailureReason)
	}

	if response.TrackerID != "" {
		t.trackerID = response.TrackerID
	}

	// Peers may be in binary or dictionary model.
	var peers []*net.TCPAddr
	if len(response.Peers) > 0 {
		if response.Peers[0] == 'l' {
			peers, err = parsePeersDictionary(response.Peers)
		} else {
			var b []byte
			err = bencode.DecodeBytes(response.Peers, &b)
			if err != nil {
				return nil, err
			}
			peers, err = tracker.DecodePeersCompact(b)
		}
	}
	if err != nil {
		return nil, err
	}

	// Filter external IP
	if len(response.ExternalIP) != 0 {
		for i, p := range peers {
			if bytes.Equal(p.IP[:], response.ExternalIP) {
				peers[i], peers = peers[len(peers)-1], peers[:len(peers)-1]
				break
			}
		}
	}

	return &tracker.AnnounceResponse{
		Interval:    time.Duration(response.Interval) * time.Second,
		MinInterval: time.Duration(response.MinInterval) * time.Second,
		Leechers:    response.Incomplete,
		Seeders:     response.Complete,
		Peers:       peers,
	}, nil
}

func parsePeersDictionary(b bencode.RawMessage) ([]*net.TCPAddr, error) {
	var peers []struct {
		IP   string `bencode:"ip"`
		Port uint16 `bencode:"port"`
	}
	err := bencode.DecodeBytes(b, &peers)
	if err != nil {
		return nil, err
	}

	addrs := make([]*net.TCPAddr, len(peers))
	for i, p := range peers {
		pe := &net.TCPAddr{IP: net.ParseIP(p.IP), Port: int(p.Port)}
		addrs[i] = pe
	}
	return addrs, err
}
<|endoftext|>


<commit_before>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Test that SIGSETXID runs on signal stack, since it's likely to
// overflow if it runs on the Go stack.

package cgotest

/*
#include <sys/types.h>
#include <unistd.h>
*/
import "C"

import (
	"runtime"
	"sync/atomic"
	"testing"

	"cgotest/issue9400"
)

func test9400(t *testing.T) {
	// We synchronize through a shared variable, so we need two procs
	defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(2))

	// Start signaller
	atomic.StoreInt32(&issue9400.Baton, 0)
	go func() {
		// Wait for RewindAndSetgid
		for atomic.LoadInt32(&issue9400.Baton) == 0 {
			runtime.Gosched()
		}
		// Broadcast SIGSETXID
		runtime.LockOSThread()
		C.setgid(0)
		// Indicate that signalling is done
		atomic.StoreInt32(&issue9400.Baton, 0)
	}()

	// Grow the stack and put down a test pattern
	const pattern = 0x123456789abcdef
	var big [1024]uint64 // len must match assembly
	for i := range big {
		big[i] = pattern
	}

	// Temporarily rewind the stack and trigger SIGSETXID
	issue9400.RewindAndSetgid()

	// Check test pattern
	for i := range big {
		if big[i] != pattern {
			t.Fatalf("entry %d of test pattern is wrong; %#x != %#x", i, big[i], uint64(pattern))
		}
	}
}
<commit_msg>misc/cgo/test: reduce likeliness of hang in Test9400<commit_after>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Test that SIGSETXID runs on signal stack, since it's likely to
// overflow if it runs on the Go stack.

package cgotest

/*
#include <sys/types.h>
#include <unistd.h>
*/
import "C"

import (
	"runtime"
	"runtime/debug"
	"sync/atomic"
	"testing"

	"cgotest/issue9400"
)

func test9400(t *testing.T) {
	// We synchronize through a shared variable, so we need two procs
	defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(2))

	// Start signaller
	atomic.StoreInt32(&issue9400.Baton, 0)
	go func() {
		// Wait for RewindAndSetgid
		for atomic.LoadInt32(&issue9400.Baton) == 0 {
			runtime.Gosched()
		}
		// Broadcast SIGSETXID
		runtime.LockOSThread()
		C.setgid(0)
		// Indicate that signalling is done
		atomic.StoreInt32(&issue9400.Baton, 0)
	}()

	// Grow the stack and put down a test pattern
	const pattern = 0x123456789abcdef
	var big [1024]uint64 // len must match assembly
	for i := range big {
		big[i] = pattern
	}

	// Disable GC for the duration of the test.
	// This avoids a potential GC deadlock when spinning in uninterruptable ASM below #49695.
	defer debug.SetGCPercent(debug.SetGCPercent(-1))

	// Temporarily rewind the stack and trigger SIGSETXID
	issue9400.RewindAndSetgid()

	// Check test pattern
	for i := range big {
		if big[i] != pattern {
			t.Fatalf("entry %d of test pattern is wrong; %#x != %#x", i, big[i], uint64(pattern))
		}
	}
}
<|endoftext|>


<commit_before>package github

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/sdk/cdsclient"
	"github.com/ovh/cds/sdk/log"

	"github.com/ovh/cds/sdk"
)

//Github http var
var (
	RateLimitLimit     int
	RateLimitRemaining = 5000
	RateLimitReset     int

	httpClient = cdsclient.NewHTTPClient(time.Second*30, false)
)

func (g *githubConsumer) postForm(path string, data url.Values, headers map[string][]string) (int, []byte, error) {
	body := strings.NewReader(data.Encode())

	req, err := http.NewRequest(http.MethodPost, URL+path, body)
	if err != nil {
		return 0, nil, err
	}

	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	req.Header.Set("User-Agent", "CDS-gh_client_id="+g.ClientID)
	for k, h := range headers {
		for i := range h {
			req.Header.Add(k, h[i])
		}
	}

	res, err := httpClient.Do(req)
	if err != nil {
		return 0, nil, err
	}
	defer res.Body.Close()
	resBody, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return res.StatusCode, nil, err
	}

	if res.StatusCode > 400 {
		ghErr := &ghError{}
		if err := json.Unmarshal(resBody, ghErr); err == nil {
			return res.StatusCode, resBody, ghErr
		}
	}

	return res.StatusCode, resBody, nil
}

func (c *githubClient) setETag(path string, headers http.Header) {
	etag := headers.Get("ETag")

	r, _ := regexp.Compile(".*\"(.*)\".*")
	s := r.FindStringSubmatch(etag)
	if len(s) == 2 {
		etag = s[1]
	}

	if etag != "" {
		//Put etag for this path in cache for 15 minutes
		c.Cache.SetWithTTL(cache.Key("vcs", "github", "etag", c.OAuthToken, strings.Replace(path, "https://", "", -1)), etag, 15*60)
	}
}

func (c *githubClient) getETag(path string) string {
	var s string
	c.Cache.Get(cache.Key("vcs", "github", "etag", c.OAuthToken, strings.Replace(path, "https://", "", -1)), &s)
	return s
}

func getNextPage(headers http.Header) string {
	linkHeader := headers.Get("Link")
	if linkHeader != "" {
		links := strings.Split(linkHeader, ",")
		for _, link := range links {
			if strings.Contains(link, "rel=\"next\"") {
				r, _ := regexp.Compile("<(.*)>.*")
				s := r.FindStringSubmatch(link)
				if len(s) == 2 {
					return s[1]
				}
				break
			}
		}
	}
	return ""
}

type getArgFunc func(c *githubClient, req *http.Request, path string)

func withETag(c *githubClient, req *http.Request, path string) {
	etag := c.getETag(path)
	if etag != "" {
		req.Header.Add("If-None-Match", fmt.Sprintf("W/\"%s\"", etag))
	}
}
func withoutETag(c *githubClient, req *http.Request, path string) {}

type postOptions struct {
	skipDefaultBaseURL bool
	asUser             bool
}

func (c *githubClient) post(path string, bodyType string, body io.Reader, opts *postOptions) (*http.Response, error) {
	if opts == nil {
		opts = new(postOptions)
	}
	if !opts.skipDefaultBaseURL && !strings.HasPrefix(path, APIURL) {
		path = APIURL + path
	}

	req, err := http.NewRequest(http.MethodPost, path, body)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Content-Type", bodyType)
	req.Header.Set("User-Agent", "CDS-gh_client_id="+c.ClientID)
	req.Header.Add("Accept", "application/json")
	if opts.asUser && c.token != "" {
		req.SetBasicAuth(c.username, c.token)
	} else {
		req.Header.Add("Authorization", fmt.Sprintf("token %s", c.OAuthToken))
	}

	log.Debug("Github API>> Request URL %s", req.URL.String())

	return httpClient.Do(req)
}

func (c *githubClient) patch(path string, opts *postOptions) (*http.Response, error) {
	if opts == nil {
		opts = new(postOptions)
	}
	if !opts.skipDefaultBaseURL && !strings.HasPrefix(path, APIURL) {
		path = APIURL + path
	}

	req, err := http.NewRequest(http.MethodPatch, path, nil)
	if err != nil {
		return nil, err
	}

	req.Header.Set("User-Agent", "CDS-gh_client_id="+c.ClientID)
	req.Header.Add("Accept", "application/json")
	if opts.asUser && c.token != "" {
		req.SetBasicAuth(c.username, c.token)
	} else {
		req.Header.Add("Authorization", fmt.Sprintf("token %s", c.OAuthToken))
	}

	log.Debug("Github API>> Request URL %s", req.URL.String())

	return httpClient.Do(req)
}

func (c *githubClient) put(path string, bodyType string, body io.Reader, opts *postOptions) (*http.Response, error) {
	if opts == nil {
		opts = new(postOptions)
	}
	if !opts.skipDefaultBaseURL && !strings.HasPrefix(path, APIURL) {
		path = APIURL + path
	}

	req, err := http.NewRequest(http.MethodPut, path, body)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Content-Type", bodyType)
	req.Header.Set("User-Agent", "CDS-gh_client_id="+c.ClientID)
	req.Header.Add("Accept", "application/json")
	if opts.asUser && c.token != "" {
		req.SetBasicAuth(c.username, c.token)
	} else {
		req.Header.Add("Authorization", fmt.Sprintf("token %s", c.OAuthToken))
	}

	log.Debug("Github API>> Request URL %s", req.URL.String())

	return httpClient.Do(req)
}

func (c *githubClient) get(path string, opts ...getArgFunc) (int, []byte, http.Header, error) {
	if isRateLimitReached() {
		return 0, nil, nil, ErrorRateLimit
	}

	if !strings.HasPrefix(path, APIURL) {
		path = APIURL + path
	}

	callURL, err := url.ParseRequestURI(path)
	if err != nil {
		return 0, nil, nil, err
	}

	req, err := http.NewRequest(http.MethodGet, callURL.String(), nil)
	if err != nil {
		return 0, nil, nil, err
	}

	req.Header.Set("User-Agent", "CDS-gh_client_id="+c.ClientID)
	req.Header.Add("Accept", "application/json")
	req.Header.Add("Authorization", fmt.Sprintf("token %s", c.OAuthToken))

	if opts == nil {
		withETag(c, req, path)
	} else {
		for _, o := range opts {
			o(c, req, path)
		}
	}

	log.Debug("Github API>> Request URL %s", req.URL.String())

	res, err := httpClient.Do(req)
	if err != nil {
		return 0, nil, nil, err
	}
	defer res.Body.Close()

	switch res.StatusCode {
	case http.StatusNotModified:
		return res.StatusCode, nil, res.Header, nil
	case http.StatusMovedPermanently, http.StatusTemporaryRedirect, http.StatusFound:
		location := res.Header.Get("Location")
		if location != "" {
			log.Debug("Github API>> Response Follow redirect :%s", location)
			return c.get(location)
		}
	case http.StatusUnauthorized:
		return res.StatusCode, nil, nil, ErrorUnauthorized
	}

	resBody, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return res.StatusCode, nil, nil, err
	}

	c.setETag(path, res.Header)

	rateLimitLimit := res.Header.Get("X-RateLimit-Limit")
	rateLimitRemaining := res.Header.Get("X-RateLimit-Remaining")
	rateLimitReset := res.Header.Get("X-RateLimit-Reset")

	if rateLimitLimit != "" && rateLimitRemaining != "" && rateLimitReset != "" {
		RateLimitLimit, _ = strconv.Atoi(rateLimitLimit)
		RateLimitRemaining, _ = strconv.Atoi(rateLimitRemaining)
		RateLimitReset, _ = strconv.Atoi(rateLimitReset)
	}

	return res.StatusCode, resBody, res.Header, nil
}

func (c *githubClient) delete(path string) error {
	if isRateLimitReached() {
		return ErrorRateLimit
	}

	if !strings.HasPrefix(path, APIURL) {
		path = APIURL + path
	}

	req, err := http.NewRequest(http.MethodDelete, path, nil)
	if err != nil {
		return err
	}

	req.Header.Set("User-Agent", "CDS-gh_client_id="+c.ClientID)
	req.Header.Add("Authorization", fmt.Sprintf("token %s", c.OAuthToken))
	log.Debug("Github API>> Request URL %s", req.URL.String())

	res, err := httpClient.Do(req)
	if err != nil {
		return sdk.WrapError(err, "Cannot do delete request")
	}

	rateLimitLimit := res.Header.Get("X-RateLimit-Limit")
	rateLimitRemaining := res.Header.Get("X-RateLimit-Remaining")
	rateLimitReset := res.Header.Get("X-RateLimit-Reset")

	if rateLimitLimit != "" && rateLimitRemaining != "" && rateLimitReset != "" {
		RateLimitRemaining, _ = strconv.Atoi(rateLimitRemaining)
		RateLimitReset, _ = strconv.Atoi(rateLimitReset)
	}

	if res.StatusCode != 204 {
		return fmt.Errorf("github>delete wrong status code %d on url %s", res.StatusCode, path)
	}
	return nil
}
<commit_msg>fix(vcs): initialize github ratelimit values (#4063)<commit_after>package github

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/sdk/cdsclient"
	"github.com/ovh/cds/sdk/log"

	"github.com/ovh/cds/sdk"
)

//Github http var
var (
	RateLimitLimit     = 5000
	RateLimitRemaining = 5000
	RateLimitReset     = int(time.Now().Unix())

	httpClient = cdsclient.NewHTTPClient(time.Second*30, false)
)

func (g *githubConsumer) postForm(path string, data url.Values, headers map[string][]string) (int, []byte, error) {
	body := strings.NewReader(data.Encode())

	req, err := http.NewRequest(http.MethodPost, URL+path, body)
	if err != nil {
		return 0, nil, err
	}

	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	req.Header.Set("User-Agent", "CDS-gh_client_id="+g.ClientID)
	for k, h := range headers {
		for i := range h {
			req.Header.Add(k, h[i])
		}
	}

	res, err := httpClient.Do(req)
	if err != nil {
		return 0, nil, err
	}
	defer res.Body.Close()
	resBody, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return res.StatusCode, nil, err
	}

	if res.StatusCode > 400 {
		ghErr := &ghError{}
		if err := json.Unmarshal(resBody, ghErr); err == nil {
			return res.StatusCode, resBody, ghErr
		}
	}

	return res.StatusCode, resBody, nil
}

func (c *githubClient) setETag(path string, headers http.Header) {
	etag := headers.Get("ETag")

	r, _ := regexp.Compile(".*\"(.*)\".*")
	s := r.FindStringSubmatch(etag)
	if len(s) == 2 {
		etag = s[1]
	}

	if etag != "" {
		//Put etag for this path in cache for 15 minutes
		c.Cache.SetWithTTL(cache.Key("vcs", "github", "etag", c.OAuthToken, strings.Replace(path, "https://", "", -1)), etag, 15*60)
	}
}

func (c *githubClient) getETag(path string) string {
	var s string
	c.Cache.Get(cache.Key("vcs", "github", "etag", c.OAuthToken, strings.Replace(path, "https://", "", -1)), &s)
	return s
}

func getNextPage(headers http.Header) string {
	linkHeader := headers.Get("Link")
	if linkHeader != "" {
		links := strings.Split(linkHeader, ",")
		for _, link := range links {
			if strings.Contains(link, "rel=\"next\"") {
				r, _ := regexp.Compile("<(.*)>.*")
				s := r.FindStringSubmatch(link)
				if len(s) == 2 {
					return s[1]
				}
				break
			}
		}
	}
	return ""
}

type getArgFunc func(c *githubClient, req *http.Request, path string)

func withETag(c *githubClient, req *http.Request, path string) {
	etag := c.getETag(path)
	if etag != "" {
		req.Header.Add("If-None-Match", fmt.Sprintf("W/\"%s\"", etag))
	}
}
func withoutETag(c *githubClient, req *http.Request, path string) {}

type postOptions struct {
	skipDefaultBaseURL bool
	asUser             bool
}

func (c *githubClient) post(path string, bodyType string, body io.Reader, opts *postOptions) (*http.Response, error) {
	if opts == nil {
		opts = new(postOptions)
	}
	if !opts.skipDefaultBaseURL && !strings.HasPrefix(path, APIURL) {
		path = APIURL + path
	}

	req, err := http.NewRequest(http.MethodPost, path, body)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Content-Type", bodyType)
	req.Header.Set("User-Agent", "CDS-gh_client_id="+c.ClientID)
	req.Header.Add("Accept", "application/json")
	if opts.asUser && c.token != "" {
		req.SetBasicAuth(c.username, c.token)
	} else {
		req.Header.Add("Authorization", fmt.Sprintf("token %s", c.OAuthToken))
	}

	log.Debug("Github API>> Request URL %s", req.URL.String())

	return httpClient.Do(req)
}

func (c *githubClient) patch(path string, opts *postOptions) (*http.Response, error) {
	if opts == nil {
		opts = new(postOptions)
	}
	if !opts.skipDefaultBaseURL && !strings.HasPrefix(path, APIURL) {
		path = APIURL + path
	}

	req, err := http.NewRequest(http.MethodPatch, path, nil)
	if err != nil {
		return nil, err
	}

	req.Header.Set("User-Agent", "CDS-gh_client_id="+c.ClientID)
	req.Header.Add("Accept", "application/json")
	if opts.asUser && c.token != "" {
		req.SetBasicAuth(c.username, c.token)
	} else {
		req.Header.Add("Authorization", fmt.Sprintf("token %s", c.OAuthToken))
	}

	log.Debug("Github API>> Request URL %s", req.URL.String())

	return httpClient.Do(req)
}

func (c *githubClient) put(path string, bodyType string, body io.Reader, opts *postOptions) (*http.Response, error) {
	if opts == nil {
		opts = new(postOptions)
	}
	if !opts.skipDefaultBaseURL && !strings.HasPrefix(path, APIURL) {
		path = APIURL + path
	}

	req, err := http.NewRequest(http.MethodPut, path, body)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Content-Type", bodyType)
	req.Header.Set("User-Agent", "CDS-gh_client_id="+c.ClientID)
	req.Header.Add("Accept", "application/json")
	if opts.asUser && c.token != "" {
		req.SetBasicAuth(c.username, c.token)
	} else {
		req.Header.Add("Authorization", fmt.Sprintf("token %s", c.OAuthToken))
	}

	log.Debug("Github API>> Request URL %s", req.URL.String())

	return httpClient.Do(req)
}

func (c *githubClient) get(path string, opts ...getArgFunc) (int, []byte, http.Header, error) {
	if isRateLimitReached() {
		return 0, nil, nil, ErrorRateLimit
	}

	if !strings.HasPrefix(path, APIURL) {
		path = APIURL + path
	}

	callURL, err := url.ParseRequestURI(path)
	if err != nil {
		return 0, nil, nil, err
	}

	req, err := http.NewRequest(http.MethodGet, callURL.String(), nil)
	if err != nil {
		return 0, nil, nil, err
	}

	req.Header.Set("User-Agent", "CDS-gh_client_id="+c.ClientID)
	req.Header.Add("Accept", "application/json")
	req.Header.Add("Authorization", fmt.Sprintf("token %s", c.OAuthToken))

	if opts == nil {
		withETag(c, req, path)
	} else {
		for _, o := range opts {
			o(c, req, path)
		}
	}

	log.Debug("Github API>> Request URL %s", req.URL.String())

	res, err := httpClient.Do(req)
	if err != nil {
		return 0, nil, nil, err
	}
	defer res.Body.Close()

	switch res.StatusCode {
	case http.StatusNotModified:
		return res.StatusCode, nil, res.Header, nil
	case http.StatusMovedPermanently, http.StatusTemporaryRedirect, http.StatusFound:
		location := res.Header.Get("Location")
		if location != "" {
			log.Debug("Github API>> Response Follow redirect :%s", location)
			return c.get(location)
		}
	case http.StatusUnauthorized:
		return res.StatusCode, nil, nil, ErrorUnauthorized
	}

	resBody, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return res.StatusCode, nil, nil, err
	}

	c.setETag(path, res.Header)

	rateLimitLimit := res.Header.Get("X-RateLimit-Limit")
	rateLimitRemaining := res.Header.Get("X-RateLimit-Remaining")
	rateLimitReset := res.Header.Get("X-RateLimit-Reset")

	if rateLimitLimit != "" && rateLimitRemaining != "" && rateLimitReset != "" {
		RateLimitLimit, _ = strconv.Atoi(rateLimitLimit)
		RateLimitRemaining, _ = strconv.Atoi(rateLimitRemaining)
		RateLimitReset, _ = strconv.Atoi(rateLimitReset)
	}

	return res.StatusCode, resBody, res.Header, nil
}

func (c *githubClient) delete(path string) error {
	if isRateLimitReached() {
		return ErrorRateLimit
	}

	if !strings.HasPrefix(path, APIURL) {
		path = APIURL + path
	}

	req, err := http.NewRequest(http.MethodDelete, path, nil)
	if err != nil {
		return err
	}

	req.Header.Set("User-Agent", "CDS-gh_client_id="+c.ClientID)
	req.Header.Add("Authorization", fmt.Sprintf("token %s", c.OAuthToken))
	log.Debug("Github API>> Request URL %s", req.URL.String())

	res, err := httpClient.Do(req)
	if err != nil {
		return sdk.WrapError(err, "Cannot do delete request")
	}

	rateLimitLimit := res.Header.Get("X-RateLimit-Limit")
	rateLimitRemaining := res.Header.Get("X-RateLimit-Remaining")
	rateLimitReset := res.Header.Get("X-RateLimit-Reset")

	if rateLimitLimit != "" && rateLimitRemaining != "" && rateLimitReset != "" {
		RateLimitRemaining, _ = strconv.Atoi(rateLimitRemaining)
		RateLimitReset, _ = strconv.Atoi(rateLimitReset)
	}

	if res.StatusCode != 204 {
		return fmt.Errorf("github>delete wrong status code %d on url %s", res.StatusCode, path)
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

// +genclient
// +genclient:noStatus
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

type CompositeController struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata"`
	Spec              CompositeControllerSpec   `json:"spec"`
	Status            CompositeControllerStatus `json:"status,omitempty"`
}

type CompositeControllerSpec struct {
	ParentResource   ResourceRule             `json:"parentResource"`
	ChildResources   []ResourcesRule          `json:"childResources,omitempty"`
	ClientConfig     ClientConfig             `json:"clientConfig,omitempty"`
	Hooks            CompositeControllerHooks `json:"hooks,omitempty"`
	GenerateSelector bool                     `json:"generateSelector,omitempty"`
}

type ResourceRule struct {
	APIVersion string `json:"apiVersion"`
	Resource   string `json:"resource"`
}

type ResourcesRule struct {
	APIVersion string   `json:"apiVersion`
	Resources  []string `json:"resources"`
}

type ClientConfig struct {
	Service ServiceReference `json:"service,omitempty"`
}

type ServiceReference struct {
	Name      string `json:"name"`
	Namespace string `json:"namespace"`
}

type CompositeControllerHooks struct {
	Sync CompositeControllerSyncHook `json:"sync,omitempty"`
}

type CompositeControllerSyncHook struct {
	Path string `json:"path"`
}

type CompositeControllerStatus struct {
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

type CompositeControllerList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata"`
	Items           []CompositeController `json:"items"`
}

// +genclient
// +genclient:noStatus
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

type InitializerController struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata"`
	Spec              InitializerControllerSpec   `json:"spec"`
	Status            InitializerControllerStatus `json:"status,omitempty"`
}

type InitializerControllerSpec struct {
	InitializerName        string                     `json:"initializerName"`
	UninitializedResources []ResourcesRule            `json:"uninitializedResources,omitempty"`
	ClientConfig           ClientConfig               `json:"clientConfig,omitempty"`
	Hooks                  InitializerControllerHooks `json:"hooks,omitempty"`
}

type InitializerControllerHooks struct {
	Init InitializerControllerInitHook `json:"init,omitempty"`
}

type InitializerControllerInitHook struct {
	Path string `json:"path"`
}

type InitializerControllerStatus struct {
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

type InitializerControllerList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata"`
	Items           []InitializerController `json:"items"`
}
<commit_msg>types.go: Fix typo in json struct tag.<commit_after>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

// +genclient
// +genclient:noStatus
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

type CompositeController struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata"`
	Spec              CompositeControllerSpec   `json:"spec"`
	Status            CompositeControllerStatus `json:"status,omitempty"`
}

type CompositeControllerSpec struct {
	ParentResource   ResourceRule             `json:"parentResource"`
	ChildResources   []ResourcesRule          `json:"childResources,omitempty"`
	ClientConfig     ClientConfig             `json:"clientConfig,omitempty"`
	Hooks            CompositeControllerHooks `json:"hooks,omitempty"`
	GenerateSelector bool                     `json:"generateSelector,omitempty"`
}

type ResourceRule struct {
	APIVersion string `json:"apiVersion"`
	Resource   string `json:"resource"`
}

type ResourcesRule struct {
	APIVersion string   `json:"apiVersion"`
	Resources  []string `json:"resources"`
}

type ClientConfig struct {
	Service ServiceReference `json:"service,omitempty"`
}

type ServiceReference struct {
	Name      string `json:"name"`
	Namespace string `json:"namespace"`
}

type CompositeControllerHooks struct {
	Sync CompositeControllerSyncHook `json:"sync,omitempty"`
}

type CompositeControllerSyncHook struct {
	Path string `json:"path"`
}

type CompositeControllerStatus struct {
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

type CompositeControllerList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata"`
	Items           []CompositeController `json:"items"`
}

// +genclient
// +genclient:noStatus
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

type InitializerController struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata"`
	Spec              InitializerControllerSpec   `json:"spec"`
	Status            InitializerControllerStatus `json:"status,omitempty"`
}

type InitializerControllerSpec struct {
	InitializerName        string                     `json:"initializerName"`
	UninitializedResources []ResourcesRule            `json:"uninitializedResources,omitempty"`
	ClientConfig           ClientConfig               `json:"clientConfig,omitempty"`
	Hooks                  InitializerControllerHooks `json:"hooks,omitempty"`
}

type InitializerControllerHooks struct {
	Init InitializerControllerInitHook `json:"init,omitempty"`
}

type InitializerControllerInitHook struct {
	Path string `json:"path"`
}

type InitializerControllerStatus struct {
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

type InitializerControllerList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata"`
	Items           []InitializerController `json:"items"`
}
<|endoftext|>


<commit_before>package terraform

import (
	"fmt"

	"github.com/hashicorp/go-version"
)

// The main version number that is being run at the moment.
const Version = "0.9.6"

// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
const VersionPrerelease = "dev"

// SemVersion is an instance of version.Version. This has the secondary
// benefit of verifying during tests and init time that our version is a
// proper semantic version, which should always be the case.
var SemVersion = version.Must(version.NewVersion(Version))

// VersionHeader is the header name used to send the current terraform version
// in http requests.
const VersionHeader = "Terraform-Version"

func VersionString() string {
	if VersionPrerelease != "" {
		return fmt.Sprintf("%s-%s", Version, VersionPrerelease)
	}
	return Version
}
<commit_msg>core: Use environment variables to set VersionPrerelease at compile time<commit_after>package terraform

import (
	"fmt"

	"github.com/hashicorp/go-version"
)

// The main version number that is being run at the moment.
const Version = "0.9.6"

// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
var VersionPrerelease string

// SemVersion is an instance of version.Version. This has the secondary
// benefit of verifying during tests and init time that our version is a
// proper semantic version, which should always be the case.
var SemVersion = version.Must(version.NewVersion(Version))

// VersionHeader is the header name used to send the current terraform version
// in http requests.
const VersionHeader = "Terraform-Version"

func VersionString() string {
	if VersionPrerelease != "" {
		return fmt.Sprintf("%s-%s", Version, VersionPrerelease)
	}
	return Version
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"os"
	"testing"

	"github.com/go-ole/go-ole"
	"github.com/go-ole/go-ole/oleutil"
	"github.com/stretchr/testify/assert"
)

func TestGetStateFile(t *testing.T) {
	sPath := getStateFile("/var/lib", "C:/Windows/hoge")
	assert.Equal(t, sPath, "/var/lib/C/Windows/hoge", "drive letter should be cared")

	sPath = getStateFile("/var/lib", "/linux/hoge")
	assert.Equal(t, sPath, "/var/lib/linux/hoge", "drive letter should be cared")
}

func TestWriteLastOffset(t *testing.T) {
	f := ".tmp/fuga/piyo"
	err := writeLastOffset(f, 15)
	assert.Equal(t, err, nil, "err should be nil")

	recordNumber, err := getLastOffset(f)
	assert.Equal(t, err, nil, "err should be nil")
	assert.Equal(t, recordNumber, int64(15))
}

func raiseEvent(t *testing.T, typ int, msg string) {
	ole.CoInitialize(0)
	defer ole.CoUninitialize()

	unk, err := oleutil.CreateObject("Wscript.Shell")
	if err != nil {
		t.Fatal(err)
	}
	disp, err := unk.QueryInterface(ole.IID_IDispatch)
	if err != nil {
		t.Fatal(err)
	}
	oleutil.MustCallMethod(disp, "LogEvent", typ, msg)
}

func TestRun(t *testing.T) {
	dir, err := ioutil.TempDir("", "check-event-log-test")
	if err != nil {
		t.Errorf("something went wrong")
	}
	defer os.RemoveAll(dir)

	opts, _ := parseArgs([]string{"-s", dir, "--log", "Application"})
	opts.prepare()

	stateFile := getStateFile(opts.StateDir, "Application")

	recordNumber, _ := getLastOffset(stateFile)
	lastNumber := recordNumber
	assert.Equal(t, int64(0), recordNumber, "something went wrong")

	testEmpty := func() {
		w, c, errLines, err := opts.searchLog("Application")
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(0), w, "something went wrong")
		assert.Equal(t, int64(0), c, "something went wrong")
		assert.Equal(t, "", errLines, "something went wrong")

		recordNumber, _ = getLastOffset(stateFile)
		assert.NotEqual(t, 0, recordNumber, "something went wrong")
	}
	testEmpty()

	lastNumber = recordNumber

	testInfo := func() {
		raiseEvent(t, 0, "check-event-log: something info occured")
		w, c, errLines, err := opts.searchLog("Application")
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(0), w, "something went wrong")
		assert.Equal(t, int64(0), c, "something went wrong")
		assert.Equal(t, "", errLines, "something went wrong")

		recordNumber, _ = getLastOffset(stateFile)
		assert.NotEqual(t, lastNumber, recordNumber, "something went wrong")
	}
	testInfo()

	lastNumber = recordNumber

	testWarning := func() {
		raiseEvent(t, 2, "check-event-log: something warning occured")
		w, c, errLines, err := opts.searchLog("Application")
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(1), w, "something went wrong")
		assert.Equal(t, int64(0), c, "something went wrong")
		assert.Equal(t, "", errLines, "something went wrong")

		recordNumber, _ = getLastOffset(stateFile)
		assert.NotEqual(t, lastNumber, recordNumber, "something went wrong")
	}
	testWarning()

	lastNumber = recordNumber

	testError := func() {
		raiseEvent(t, 1, "check-event-log: something error occured")
		w, c, errLines, err := opts.searchLog("Application")
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(0), w, "something went wrong")
		assert.Equal(t, int64(1), c, "something went wrong")
		assert.Equal(t, "", errLines, "something went wrong")

		recordNumber, _ = getLastOffset(stateFile)
		assert.NotEqual(t, lastNumber, recordNumber, "something went wrong")
	}
	testError()

	lastNumber = recordNumber

	opts, _ = parseArgs([]string{"-s", dir, "--log", "Application", "-r"})
	opts.prepare()

	testReturn := func() {
		raiseEvent(t, 1, "check-event-log: something error occured")
		raiseEvent(t, 2, "check-event-log: something warning occured")
		w, c, errLines, err := opts.searchLog("Application")
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(1), w, "something went wrong")
		assert.Equal(t, int64(1), c, "something went wrong")
		assert.Equal(t, "WSH:check-event-log: something error occured\nWSH:check-event-log: something warning occured\n", errLines, "something went wrong")

		recordNumber, _ = getLastOffset(stateFile)
		assert.NotEqual(t, lastNumber, recordNumber, "something went wrong")
	}
	testReturn()
}

func TestSourcePattern(t *testing.T) {
	dir, err := ioutil.TempDir("", "check-event-log-test")
	if err != nil {
		t.Errorf("something went wrong")
	}
	defer os.RemoveAll(dir)

	opts, _ := parseArgs([]string{"-s", dir, "--log", "Application"})
	opts.prepare()

	stateFile := getStateFile(opts.StateDir, "Application")

	recordNumber, _ := getLastOffset(stateFile)
	lastNumber := recordNumber
	assert.Equal(t, int64(0), recordNumber, "something went wrong")

	testEmpty := func() {
		w, c, errLines, err := opts.searchLog("Application")
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(0), w, "something went wrong")
		assert.Equal(t, int64(0), c, "something went wrong")
		assert.Equal(t, "", errLines, "something went wrong")

		recordNumber, _ = getLastOffset(stateFile)
		assert.NotEqual(t, 0, recordNumber, "something went wrong")
	}
	testEmpty()

	lastNumber = recordNumber

	opts, _ = parseArgs([]string{"-s", dir, "--log", "Application", "--message-pattern", "テストエラーが(発生しました|起きました)"})
	opts.prepare()

	testMessagePattern := func() {
		raiseEvent(t, 1, "check-event-log: テストエラーが発生しました")
		w, c, errLines, err := opts.searchLog("Application")
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(0), w, "something went wrong")
		assert.Equal(t, int64(1), c, "something went wrong")
		assert.Equal(t, "", errLines, "something went wrong")

		recordNumber, _ = getLastOffset(stateFile)
		assert.NotEqual(t, lastNumber, recordNumber, "something went wrong")
	}
	testMessagePattern()

	opts, _ = parseArgs([]string{"-s", dir, "--log", "Application", "--source-pattern", "[Ww][Ss][Hh]"})
	opts.prepare()

	testSourcePattern := func() {
		raiseEvent(t, 2, "check-event-log: テストエラーが発生しました")
		w, c, errLines, err := opts.searchLog("Application")
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(1), w, "something went wrong")
		assert.Equal(t, int64(0), c, "something went wrong")
		assert.Equal(t, "", errLines, "something went wrong")

		recordNumber, _ = getLastOffset(stateFile)
		assert.NotEqual(t, lastNumber, recordNumber, "something went wrong")
	}
	testSourcePattern()
}
<commit_msg>add test<commit_after>package main

import (
	"io/ioutil"
	"os"
	"testing"

	"github.com/go-ole/go-ole"
	"github.com/go-ole/go-ole/oleutil"
	"github.com/stretchr/testify/assert"
)

func TestGetStateFile(t *testing.T) {
	sPath := getStateFile("/var/lib", "C:/Windows/hoge")
	assert.Equal(t, sPath, "/var/lib/C/Windows/hoge", "drive letter should be cared")

	sPath = getStateFile("/var/lib", "/linux/hoge")
	assert.Equal(t, sPath, "/var/lib/linux/hoge", "drive letter should be cared")
}

func TestWriteLastOffset(t *testing.T) {
	f := ".tmp/fuga/piyo"
	err := writeLastOffset(f, 15)
	assert.Equal(t, err, nil, "err should be nil")

	recordNumber, err := getLastOffset(f)
	assert.Equal(t, err, nil, "err should be nil")
	assert.Equal(t, recordNumber, int64(15))
}

func raiseEvent(t *testing.T, typ int, msg string) {
	ole.CoInitialize(0)
	defer ole.CoUninitialize()

	unk, err := oleutil.CreateObject("Wscript.Shell")
	if err != nil {
		t.Fatal(err)
	}
	disp, err := unk.QueryInterface(ole.IID_IDispatch)
	if err != nil {
		t.Fatal(err)
	}
	oleutil.MustCallMethod(disp, "LogEvent", typ, msg)
}

func TestRun(t *testing.T) {
	dir, err := ioutil.TempDir("", "check-event-log-test")
	if err != nil {
		t.Errorf("something went wrong")
	}
	defer os.RemoveAll(dir)

	opts, _ := parseArgs([]string{"-s", dir, "--log", "Application"})
	opts.prepare()

	stateFile := getStateFile(opts.StateDir, "Application")

	recordNumber, _ := getLastOffset(stateFile)
	lastNumber := recordNumber
	assert.Equal(t, int64(0), recordNumber, "something went wrong")

	testEmpty := func() {
		w, c, errLines, err := opts.searchLog("Application")
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(0), w, "something went wrong")
		assert.Equal(t, int64(0), c, "something went wrong")
		assert.Equal(t, "", errLines, "something went wrong")

		recordNumber, _ = getLastOffset(stateFile)
		assert.NotEqual(t, 0, recordNumber, "something went wrong")
	}
	testEmpty()

	lastNumber = recordNumber

	testInfo := func() {
		raiseEvent(t, 0, "check-event-log: something info occured")
		w, c, errLines, err := opts.searchLog("Application")
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(0), w, "something went wrong")
		assert.Equal(t, int64(0), c, "something went wrong")
		assert.Equal(t, "", errLines, "something went wrong")

		recordNumber, _ = getLastOffset(stateFile)
		assert.NotEqual(t, lastNumber, recordNumber, "something went wrong")
	}
	testInfo()

	lastNumber = recordNumber

	testWarning := func() {
		raiseEvent(t, 2, "check-event-log: something warning occured")
		w, c, errLines, err := opts.searchLog("Application")
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(1), w, "something went wrong")
		assert.Equal(t, int64(0), c, "something went wrong")
		assert.Equal(t, "", errLines, "something went wrong")

		recordNumber, _ = getLastOffset(stateFile)
		assert.NotEqual(t, lastNumber, recordNumber, "something went wrong")
	}
	testWarning()

	lastNumber = recordNumber

	testError := func() {
		raiseEvent(t, 1, "check-event-log: something error occured")
		w, c, errLines, err := opts.searchLog("Application")
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(0), w, "something went wrong")
		assert.Equal(t, int64(1), c, "something went wrong")
		assert.Equal(t, "", errLines, "something went wrong")

		recordNumber, _ = getLastOffset(stateFile)
		assert.NotEqual(t, lastNumber, recordNumber, "something went wrong")
	}
	testError()

	lastNumber = recordNumber

	opts, _ = parseArgs([]string{"-s", dir, "--log", "Application", "-r"})
	opts.prepare()

	testReturn := func() {
		raiseEvent(t, 1, "check-event-log: something error occured")
		raiseEvent(t, 2, "check-event-log: something warning occured")
		w, c, errLines, err := opts.searchLog("Application")
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(1), w, "something went wrong")
		assert.Equal(t, int64(1), c, "something went wrong")
		assert.Equal(t, "WSH:check-event-log: something error occured\nWSH:check-event-log: something warning occured\n", errLines, "something went wrong")

		recordNumber, _ = getLastOffset(stateFile)
		assert.NotEqual(t, lastNumber, recordNumber, "something went wrong")
	}
	testReturn()
}

func TestSourcePattern(t *testing.T) {
	dir, err := ioutil.TempDir("", "check-event-log-test")
	if err != nil {
		t.Errorf("something went wrong")
	}
	defer os.RemoveAll(dir)

	opts, _ := parseArgs([]string{"-s", dir, "--log", "Application"})
	opts.prepare()

	stateFile := getStateFile(opts.StateDir, "Application")

	recordNumber, _ := getLastOffset(stateFile)
	lastNumber := recordNumber
	assert.Equal(t, int64(0), recordNumber, "something went wrong")

	testEmpty := func() {
		w, c, errLines, err := opts.searchLog("Application")
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(0), w, "something went wrong")
		assert.Equal(t, int64(0), c, "something went wrong")
		assert.Equal(t, "", errLines, "something went wrong")

		recordNumber, _ = getLastOffset(stateFile)
		assert.NotEqual(t, 0, recordNumber, "something went wrong")
	}
	testEmpty()

	lastNumber = recordNumber

	opts, _ = parseArgs([]string{"-s", dir, "--log", "Application", "--message-pattern", "テストエラーが(発生しました|起きました)"})
	opts.prepare()

	testMessagePattern := func() {
		raiseEvent(t, 1, "check-event-log: テストエラーが発生しました")
		w, c, errLines, err := opts.searchLog("Application")
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(0), w, "something went wrong")
		assert.Equal(t, int64(1), c, "something went wrong")
		assert.Equal(t, "", errLines, "something went wrong")

		recordNumber, _ = getLastOffset(stateFile)
		assert.NotEqual(t, lastNumber, recordNumber, "something went wrong")
	}
	testMessagePattern()

	opts, _ = parseArgs([]string{"-s", dir, "--log", "Application", "--source-pattern", "[Ww][Ss][Hh]"})
	opts.prepare()

	testSourcePattern := func() {
		raiseEvent(t, 2, "check-event-log: テストエラーが発生しました")
		w, c, errLines, err := opts.searchLog("Application")
		assert.Equal(t, err, nil, "err should be nil")
		assert.Equal(t, int64(1), w, "something went wrong")
		assert.Equal(t, int64(0), c, "something went wrong")
		assert.Equal(t, "", errLines, "something went wrong")

		recordNumber, _ = getLastOffset(stateFile)
		assert.NotEqual(t, lastNumber, recordNumber, "something went wrong")
	}
	testSourcePattern()
}

func TestFailFirst(t *testing.T) {
	dir, err := ioutil.TempDir("", "check-event-log-test")
	if err != nil {
		t.Errorf("something went wrong")
	}
	defer os.RemoveAll(dir)

	opts, _ := parseArgs([]string{"-s", dir, "--log", "Application", "--fail-first", "--warning-over", "0", "--critical-over", "0"})
	opts.prepare()

	testFailFirst := func() {
		w, c, errLines, err := opts.searchLog("Application")
		assert.Equal(t, err, nil, "err should be nil")
		assert.NotEqual(t, int64(0), w, "something went wrong")
		assert.NotEqual(t, int64(0), c, "something went wrong")
		assert.Equal(t, "", errLines, "something went wrong")
	}
	testFailFirst()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math/rand"
	"time"

	"github.com/vbauerster/mpb"
)

func main() {

	name := "Single bar:"
	// Star mpb's rendering goroutine.
	// If you don't plan to cancel, feed with nil
	// otherwise provide context.Context, see cancel example
	p := mpb.New(nil)
	bar := p.AddBar(100).PrependName(name, 0).AppendPercentage()

	for i := 0; i < 100; i++ {
		time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()
	fmt.Println("stop")
}
<commit_msg>use custom bar format<commit_after>package main

import (
	"fmt"
	"math/rand"
	"time"

	"github.com/vbauerster/mpb"
)

func main() {

	name := "Single bar:"
	// Star mpb's rendering goroutine.
	// If you don't plan to cancel, feed with nil
	// otherwise provide context.Context, see cancel example
	p := mpb.New(nil)
	// Set custom format, the default one is "[=>-]"
	p.Format("╢▌▌░╟")

	bar := p.AddBar(100).PrependName(name, 0).AppendPercentage()

	for i := 0; i < 100; i++ {
		time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond)
		bar.Incr(1)
	}

	p.Stop()
	fmt.Println("stop")
}
<|endoftext|>


<commit_before>package mpawselasticsearch

import (
	"errors"
	"flag"
	"log"
	"os"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/ec2metadata"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	mp "github.com/mackerelio/go-mackerel-plugin"
)

var graphdef = map[string]mp.Graphs{
	"es.Nodes": {
		Label: "AWS ES Nodes",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "Nodes", Label: "Nodes"},
		},
	},
	"es.CPUUtilization": {
		Label: "AWS ES CPU Utilization",
		Unit:  "percentage",
		Metrics: []mp.Metrics{
			{Name: "CPUUtilization", Label: "CPUUtilization"},
		},
	},
	"es.JVMMemoryPressure": {
		Label: "AWS ES JVMMemoryPressure",
		Unit:  "percentage",
		Metrics: []mp.Metrics{
			{Name: "JVMMemoryPressure", Label: "JVMMemoryPressure"},
		},
	},
	"es.FreeStorageSpace": {
		Label: "AWS ES Free Storage Space",
		Unit:  "bytes",
		Metrics: []mp.Metrics{
			{Name: "FreeStorageSpace", Label: "FreeStorageSpace"},
		},
	},
	"es.SearchableDocuments": {
		Label: "AWS ES SearchableDocuments",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "SearchableDocuments", Label: "SearchableDocuments"},
		},
	},
	"es.DeletedDocuments": {
		Label: "AWS ES DeletedDocuments",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "DeletedDocuments", Label: "DeletedDocuments"},
		},
	},
	"es.IOPS": {
		Label: "AWS ES IOPS",
		Unit:  "iops",
		Metrics: []mp.Metrics{
			{Name: "ReadIOPS", Label: "ReadIOPS"},
			{Name: "WriteIOPS", Label: "WriteIOPS"},
		},
	},
	"es.Throughput": {
		Label: "AWS ES Throughput",
		Unit:  "bytes/sec",
		Metrics: []mp.Metrics{
			{Name: "ReadThroughput", Label: "ReadThroughput"},
			{Name: "WriteThroughput", Label: "WriteThroughput"},
		},
	},
	"es.DiskQueueDepth": {
		Label: "AWS ES DiskQueueDepth",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "DiskQueueDepth", Label: "DiskQueueDepth"},
		},
	},
	"es.AutomatedSnapshotFailure": {
		Label: "AWS ES AutomatedSnapshotFailure",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "AutomatedSnapshotFailure", Label: "AutomatedSnapshotFailure"},
		},
	},
	"es.MasterCPUUtilization": {
		Label: "AWS ES MasterCPUUtilization",
		Unit:  "percentage",
		Metrics: []mp.Metrics{
			{Name: "MasterCPUUtilization", Label: "MasterCPUUtilization"},
		},
	},
	"es.Latency": {
		Label: "AWS ES Latency",
		Unit:  "float",
		Metrics: []mp.Metrics{
			{Name: "ReadLatency", Label: "ReadLatency"},
			{Name: "WriteLatency", Label: "WriteLatency"},
		},
	},
	"es.MasterJVMMemoryPressure": {
		Label: "AWS ES MasterJVMMemoryPressure",
		Unit:  "percentage",
		Metrics: []mp.Metrics{
			{Name: "MasterJVMMemoryPressure", Label: "MasterJVMMemoryPressure"},
		},
	},
	"es.MasterFreeStorageSpace": {
		Label: "AWS ES MasterFreeStorageSpace",
		Unit:  "bytes",
		Metrics: []mp.Metrics{
			{Name: "MasterFreeStorageSpace", Label: "MasterFreeStorageSpace"},
		},
	},
	"es.ClusterStatus": {
		Label: "AWS ES ClusterStatus",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "ClusterStatus.green", Label: "green"},
			{Name: "ClusterStatus.yellow", Label: "yellow"},
			{Name: "ClusterStatus.red", Label: "red"},
		},
	},
}

// ESPlugin mackerel plugin for aws elasticsearch
type ESPlugin struct {
	Region          string
	AccessKeyID     string
	SecretAccessKey string
	Domain          string
	ClientID        string
	CloudWatch      *cloudwatch.CloudWatch
}

const esNameSpace = "AWS/ES"

func (p *ESPlugin) prepare() error {
	sess, err := session.NewSession()
	if err != nil {
		return err
	}

	config := aws.NewConfig()
	if p.AccessKeyID != "" && p.SecretAccessKey != "" {
		config = config.WithCredentials(credentials.NewStaticCredentials(p.AccessKeyID, p.SecretAccessKey, ""))
	}
	if p.Region != "" {
		config = config.WithRegion(p.Region)
	}

	p.CloudWatch = cloudwatch.New(sess, config)
	return nil
}

func (p ESPlugin) getLastPoint(dimensions []*cloudwatch.Dimension, metricName *string) (float64, error) {
	now := time.Now()

	response, err := p.CloudWatch.GetMetricStatistics(&cloudwatch.GetMetricStatisticsInput{
		Dimensions: dimensions,
		StartTime:  aws.Time(now.Add(time.Duration(180) * time.Second * -1)),
		EndTime:    aws.Time(now),
		MetricName: metricName,
		Period:     aws.Int64(60),
		Statistics: []*string{aws.String("Average")},
		Namespace:  aws.String(esNameSpace),
	})

	if err != nil {
		return 0, err
	}

	datapoints := response.Datapoints
	if len(datapoints) == 0 {
		return 0, errors.New("fetched no datapoints")
	}

	latest := new(time.Time)
	var latestVal float64
	for _, dp := range datapoints {
		if dp.Timestamp.Before(*latest) {
			continue
		}

		latest = dp.Timestamp
		latestVal = *dp.Average
	}

	return latestVal, nil
}

// FetchMetrics interface for mackerelplugin
func (p ESPlugin) FetchMetrics() (map[string]float64, error) {
	dimensionFilters := []*cloudwatch.DimensionFilter{
		{
			Name:  aws.String("DomainName"),
			Value: aws.String(p.Domain),
		},
		{
			Name:  aws.String("ClientId"),
			Value: aws.String(p.ClientID),
		},
	}

	ret, err := p.CloudWatch.ListMetrics(&cloudwatch.ListMetricsInput{
		Namespace:  aws.String(esNameSpace),
		Dimensions: dimensionFilters,
	})
	if err != nil {
		log.Printf("%s", err)
	}

	stat := make(map[string]float64)

	dimensions := []*cloudwatch.Dimension{
		{
			Name:  aws.String("DomainName"),
			Value: aws.String(p.Domain),
		},
		{
			Name:  aws.String("ClientId"),
			Value: aws.String(p.ClientID),
		},
	}
	for _, met := range ret.Metrics {
		v, err := p.getLastPoint(dimensions, met.MetricName)
		if err == nil {
			if *met.MetricName == "MasterFreeStorageSpace" || *met.MetricName == "FreeStorageSpace" {
				// MBytes -> Bytes
				v = v * 1024 * 1024
			}
			stat[*met.MetricName] = v
		} else {
			log.Printf("%s: %s", met.MetricName, err)
		}
	}

	return stat, nil
}

// GraphDefinition interface for mackerelplugin
func (p ESPlugin) GraphDefinition() map[string]mp.Graphs {
	return graphdef
}

// Do the plugin
func Do() {
	optRegion := flag.String("region", "", "AWS Region")
	optAccessKeyID := flag.String("access-key-id", "", "AWS Access Key ID")
	optSecretAccessKey := flag.String("secret-access-key", "", "AWS Secret Access Key")
	optClientID := flag.String("client-id", "", "AWS Client ID")
	optDomain := flag.String("domain", "", "ES domain name")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	flag.Parse()

	var es ESPlugin

	if *optRegion == "" {
		ec2metadata := ec2metadata.New(session.New())
		if ec2metadata.Available() {
			es.Region, _ = ec2metadata.Region()
		}
	} else {
		es.Region = *optRegion
	}

	es.Region = *optRegion
	es.Domain = *optDomain
	es.ClientID = *optClientID
	es.AccessKeyID = *optAccessKeyID
	es.SecretAccessKey = *optSecretAccessKey

	err := es.prepare()
	if err != nil {
		log.Fatalln(err)
	}

	helper := mp.NewMackerelPlugin(es)
	helper.Tempfile = *optTempfile

	helper.Run()
}
<commit_msg>fix import again<commit_after>package mpawselasticsearch

import (
	"errors"
	"flag"
	"log"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/ec2metadata"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	mp "github.com/mackerelio/go-mackerel-plugin"
)

var graphdef = map[string]mp.Graphs{
	"es.Nodes": {
		Label: "AWS ES Nodes",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "Nodes", Label: "Nodes"},
		},
	},
	"es.CPUUtilization": {
		Label: "AWS ES CPU Utilization",
		Unit:  "percentage",
		Metrics: []mp.Metrics{
			{Name: "CPUUtilization", Label: "CPUUtilization"},
		},
	},
	"es.JVMMemoryPressure": {
		Label: "AWS ES JVMMemoryPressure",
		Unit:  "percentage",
		Metrics: []mp.Metrics{
			{Name: "JVMMemoryPressure", Label: "JVMMemoryPressure"},
		},
	},
	"es.FreeStorageSpace": {
		Label: "AWS ES Free Storage Space",
		Unit:  "bytes",
		Metrics: []mp.Metrics{
			{Name: "FreeStorageSpace", Label: "FreeStorageSpace"},
		},
	},
	"es.SearchableDocuments": {
		Label: "AWS ES SearchableDocuments",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "SearchableDocuments", Label: "SearchableDocuments"},
		},
	},
	"es.DeletedDocuments": {
		Label: "AWS ES DeletedDocuments",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "DeletedDocuments", Label: "DeletedDocuments"},
		},
	},
	"es.IOPS": {
		Label: "AWS ES IOPS",
		Unit:  "iops",
		Metrics: []mp.Metrics{
			{Name: "ReadIOPS", Label: "ReadIOPS"},
			{Name: "WriteIOPS", Label: "WriteIOPS"},
		},
	},
	"es.Throughput": {
		Label: "AWS ES Throughput",
		Unit:  "bytes/sec",
		Metrics: []mp.Metrics{
			{Name: "ReadThroughput", Label: "ReadThroughput"},
			{Name: "WriteThroughput", Label: "WriteThroughput"},
		},
	},
	"es.DiskQueueDepth": {
		Label: "AWS ES DiskQueueDepth",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "DiskQueueDepth", Label: "DiskQueueDepth"},
		},
	},
	"es.AutomatedSnapshotFailure": {
		Label: "AWS ES AutomatedSnapshotFailure",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "AutomatedSnapshotFailure", Label: "AutomatedSnapshotFailure"},
		},
	},
	"es.MasterCPUUtilization": {
		Label: "AWS ES MasterCPUUtilization",
		Unit:  "percentage",
		Metrics: []mp.Metrics{
			{Name: "MasterCPUUtilization", Label: "MasterCPUUtilization"},
		},
	},
	"es.Latency": {
		Label: "AWS ES Latency",
		Unit:  "float",
		Metrics: []mp.Metrics{
			{Name: "ReadLatency", Label: "ReadLatency"},
			{Name: "WriteLatency", Label: "WriteLatency"},
		},
	},
	"es.MasterJVMMemoryPressure": {
		Label: "AWS ES MasterJVMMemoryPressure",
		Unit:  "percentage",
		Metrics: []mp.Metrics{
			{Name: "MasterJVMMemoryPressure", Label: "MasterJVMMemoryPressure"},
		},
	},
	"es.MasterFreeStorageSpace": {
		Label: "AWS ES MasterFreeStorageSpace",
		Unit:  "bytes",
		Metrics: []mp.Metrics{
			{Name: "MasterFreeStorageSpace", Label: "MasterFreeStorageSpace"},
		},
	},
	"es.ClusterStatus": {
		Label: "AWS ES ClusterStatus",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "ClusterStatus.green", Label: "green"},
			{Name: "ClusterStatus.yellow", Label: "yellow"},
			{Name: "ClusterStatus.red", Label: "red"},
		},
	},
}

// ESPlugin mackerel plugin for aws elasticsearch
type ESPlugin struct {
	Region          string
	AccessKeyID     string
	SecretAccessKey string
	Domain          string
	ClientID        string
	CloudWatch      *cloudwatch.CloudWatch
}

const esNameSpace = "AWS/ES"

func (p *ESPlugin) prepare() error {
	sess, err := session.NewSession()
	if err != nil {
		return err
	}

	config := aws.NewConfig()
	if p.AccessKeyID != "" && p.SecretAccessKey != "" {
		config = config.WithCredentials(credentials.NewStaticCredentials(p.AccessKeyID, p.SecretAccessKey, ""))
	}
	if p.Region != "" {
		config = config.WithRegion(p.Region)
	}

	p.CloudWatch = cloudwatch.New(sess, config)
	return nil
}

func (p ESPlugin) getLastPoint(dimensions []*cloudwatch.Dimension, metricName *string) (float64, error) {
	now := time.Now()

	response, err := p.CloudWatch.GetMetricStatistics(&cloudwatch.GetMetricStatisticsInput{
		Dimensions: dimensions,
		StartTime:  aws.Time(now.Add(time.Duration(180) * time.Second * -1)),
		EndTime:    aws.Time(now),
		MetricName: metricName,
		Period:     aws.Int64(60),
		Statistics: []*string{aws.String("Average")},
		Namespace:  aws.String(esNameSpace),
	})

	if err != nil {
		return 0, err
	}

	datapoints := response.Datapoints
	if len(datapoints) == 0 {
		return 0, errors.New("fetched no datapoints")
	}

	latest := new(time.Time)
	var latestVal float64
	for _, dp := range datapoints {
		if dp.Timestamp.Before(*latest) {
			continue
		}

		latest = dp.Timestamp
		latestVal = *dp.Average
	}

	return latestVal, nil
}

// FetchMetrics interface for mackerelplugin
func (p ESPlugin) FetchMetrics() (map[string]float64, error) {
	dimensionFilters := []*cloudwatch.DimensionFilter{
		{
			Name:  aws.String("DomainName"),
			Value: aws.String(p.Domain),
		},
		{
			Name:  aws.String("ClientId"),
			Value: aws.String(p.ClientID),
		},
	}

	ret, err := p.CloudWatch.ListMetrics(&cloudwatch.ListMetricsInput{
		Namespace:  aws.String(esNameSpace),
		Dimensions: dimensionFilters,
	})
	if err != nil {
		log.Printf("%s", err)
	}

	stat := make(map[string]float64)

	dimensions := []*cloudwatch.Dimension{
		{
			Name:  aws.String("DomainName"),
			Value: aws.String(p.Domain),
		},
		{
			Name:  aws.String("ClientId"),
			Value: aws.String(p.ClientID),
		},
	}
	for _, met := range ret.Metrics {
		v, err := p.getLastPoint(dimensions, met.MetricName)
		if err == nil {
			if *met.MetricName == "MasterFreeStorageSpace" || *met.MetricName == "FreeStorageSpace" {
				// MBytes -> Bytes
				v = v * 1024 * 1024
			}
			stat[*met.MetricName] = v
		} else {
			log.Printf("%s: %s", met.MetricName, err)
		}
	}

	return stat, nil
}

// GraphDefinition interface for mackerelplugin
func (p ESPlugin) GraphDefinition() map[string]mp.Graphs {
	return graphdef
}

// Do the plugin
func Do() {
	optRegion := flag.String("region", "", "AWS Region")
	optAccessKeyID := flag.String("access-key-id", "", "AWS Access Key ID")
	optSecretAccessKey := flag.String("secret-access-key", "", "AWS Secret Access Key")
	optClientID := flag.String("client-id", "", "AWS Client ID")
	optDomain := flag.String("domain", "", "ES domain name")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	flag.Parse()

	var es ESPlugin

	if *optRegion == "" {
		ec2metadata := ec2metadata.New(session.New())
		if ec2metadata.Available() {
			es.Region, _ = ec2metadata.Region()
		}
	} else {
		es.Region = *optRegion
	}

	es.Region = *optRegion
	es.Domain = *optDomain
	es.ClientID = *optClientID
	es.AccessKeyID = *optAccessKeyID
	es.SecretAccessKey = *optSecretAccessKey

	err := es.prepare()
	if err != nil {
		log.Fatalln(err)
	}

	helper := mp.NewMackerelPlugin(es)
	helper.Tempfile = *optTempfile

	helper.Run()
}
<|endoftext|>


<commit_before>package test

import "testing"

// func TestConnectTCP(t *testing.T) {
// 	s := NewTCPServerHelper(t)
// 	defer s.Close()
// 	c := s.GetTCPClient()
// 	defer c.Close()
// }

func TestConnectTLS(t *testing.T) {
	sh := NewTLSServerHelper(t)
	defer sh.Close()
	ch := sh.GetTLSClient(true)
	defer ch.Close()

	// todo: solve multiple close error messages first

	// var wg sync.WaitGroup
	// msg := []byte("test message")
	//
	// sh.Server.MiddlewareIn(middleware.Echo)
	// ch.Client.MiddlewareIn(func(ctx *client.Ctx) {
	// 	defer wg.Done()
	// 	if !reflect.DeepEqual(ctx.Msg, msg) {
	// 		t.Fatalf("expected: '%s', got: '%s'", msg, ctx.Msg)
	// 	}
	// 	ctx.Next()
	// })
	//
	// wg.Add(1)
	// ch.Client.Send(msg) // todo: use fail-safe ClientHelper.Send instead
	// wg.Wait()
}
<commit_msg>add test todo about debug env<commit_after>package test

import "testing"

// func TestConnectTCP(t *testing.T) {
// 	s := NewTCPServerHelper(t)
// 	defer s.Close()
// 	c := s.GetTCPClient()
// 	defer c.Close()
// }

func TestConnectTLS(t *testing.T) {
	sh := NewTLSServerHelper(t)
	defer sh.Close()
	ch := sh.GetTLSClient(true)
	defer ch.Close()

	// todo: solve multiple close error messages first
	// todo: enable debug mode both on client & server if debug env var is defined during test launch

	// var wg sync.WaitGroup
	// msg := []byte("test message")
	//
	// sh.Server.MiddlewareIn(middleware.Echo)
	// ch.Client.MiddlewareIn(func(ctx *client.Ctx) {
	// 	defer wg.Done()
	// 	if !reflect.DeepEqual(ctx.Msg, msg) {
	// 		t.Fatalf("expected: '%s', got: '%s'", msg, ctx.Msg)
	// 	}
	// 	ctx.Next()
	// })
	//
	// wg.Add(1)
	// ch.Client.Send(msg) // todo: use fail-safe ClientHelper.Send instead
	// wg.Wait()
}
<|endoftext|>