<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package alicloud

import (
	"encoding/json"
	"path/filepath"
	"time"

	"fmt"
	"github.com/denverdino/aliyungo/common"
	"github.com/denverdino/aliyungo/ecs"
	"github.com/denverdino/aliyungo/metadata"
	"github.com/denverdino/aliyungo/pvtz"
	"github.com/denverdino/aliyungo/slb"
	"github.com/go-cmd/cmd"
	"github.com/golang/glog"
	"k8s.io/apimachinery/pkg/util/wait"
	"strings"
)

// ROLE_NAME default kubernetes master role name
var ROLE_NAME = "KubernetesMasterRole"

// ASSUME_ROLE_NAME managed kubernetes role name
var ASSUME_ROLE_NAME = "AliyunCSManagedKubernetesRole"

// TOKEN_RESYNC_PERIOD default token sync period
var TOKEN_RESYNC_PERIOD = 10 * time.Minute

// ClientMgr client manager for aliyun sdk
type ClientMgr struct {
	stop <-chan struct{}

	token TokenAuth

	meta         IMetaData
	routes       *RoutesClient
	loadbalancer *LoadBalancerClient
	privateZone  *PrivateZoneClient
	instance     *InstanceClient
}

// NewClientMgr return a new client manager
func NewClientMgr(key, secret string) (*ClientMgr, error) {
	m := NewMetaData()
	region, err := m.Region()
	if err != nil {
		return nil, fmt.Errorf("can not determin region: %s", err.Error())
	}
	vpcid, err := m.VpcID()
	if err != nil {
		return nil, fmt.Errorf("can not determin vpcid: %s", err.Error())
	}
	ecsclient := ecs.NewECSClientWithSecurityToken(key, secret, "", common.Region(region))
	mgr := &ClientMgr{
		stop: make(<-chan struct{}, 1),
		meta: m,
		instance: &InstanceClient{
			c: ecsclient,
		},
		loadbalancer: &LoadBalancerClient{
			vpcid: vpcid,
			ins:   ecsclient,
			c:     slb.NewSLBClientWithSecurityToken(key, secret, "", common.Region(region)),
		},
		privateZone: &PrivateZoneClient{
			c: pvtz.NewPVTZClientWithSecurityToken(key, secret, "", common.Region("cn-hangzhou")),
		},
		routes: &RoutesClient{
			client: ecs.NewVPCClientWithSecurityToken(key, secret, "", common.Region(region)),
			region: region,
		},
	}

	if key == "" || secret == "" {
		glog.Infof("alicloud: use ramrole token mode without ak.")
		mgr.token = &RamRoleToken{meta: m}
	} else {
		inittoken := &Token{
			AccessKey:    key,
			AccessSecret: secret,
			UID:          cfg.Global.UID,
		}
		if inittoken.UID == "" {
			glog.Infof("alicloud: ak mode to authenticate user. without token and role assume")
			mgr.token = &AkAuthToken{ak: inittoken}
		} else {
			glog.Infof("alicloud: service account auth mode")
			mgr.token = &ServiceToken{svcak: inittoken}
		}
	}
	return mgr, nil
}

func (mgr *ClientMgr) Start(settoken func(mgr *ClientMgr, token *Token) error) error {
	initialized := false
	tokenfunc := func() {
		// refresh client token periodically
		token, err := mgr.token.NextToken()
		if err != nil {
			glog.Errorf("token retrieve: %s", err.Error())
			return
		}
		err = settoken(mgr, token)
		if err != nil {
			glog.Errorf("set token: %s", err.Error())
			return
		}
		initialized = true
	}

	go wait.Until(
		tokenfunc,
		time.Duration(TOKEN_RESYNC_PERIOD),
		mgr.stop,
	)
	return wait.ExponentialBackoff(
		wait.Backoff{
			Steps:    7,
			Duration: 1 * time.Second,
			Jitter:   1,
			Factor:   2,
		}, func() (done bool, err error) {
			tokenfunc()
			glog.Infof("wait for token ready")
			return initialized, nil
		},
	)
}

func RefreshToken(mgr *ClientMgr, token *Token) error {
	ecsclient := mgr.instance.c.(*ecs.Client)
	slbclient := mgr.loadbalancer.c.(*slb.Client)
	pvtzclient := mgr.privateZone.c.(*pvtz.Client)
	vpcclient := mgr.routes.client.(*ecs.Client)
	ecsclient.WithSecurityToken(token.Token).
		WithAccessKeyId(token.AccessKey).
		WithAccessKeySecret(token.AccessSecret)
	slbclient.WithSecurityToken(token.Token).
		WithAccessKeyId(token.AccessKey).
		WithAccessKeySecret(token.AccessSecret)
	pvtzclient.WithSecurityToken(token.Token).
		WithAccessKeyId(token.AccessKey).
		WithAccessKeySecret(token.AccessSecret)
	vpcclient.WithSecurityToken(token.Token).
		WithAccessKeyId(token.AccessKey).
		WithAccessKeySecret(token.AccessSecret)

	ecsclient.SetUserAgent(KUBERNETES_ALICLOUD_IDENTITY)
	slbclient.SetUserAgent(KUBERNETES_ALICLOUD_IDENTITY)
	pvtzclient.SetUserAgent(KUBERNETES_ALICLOUD_IDENTITY)
	vpcclient.SetUserAgent(KUBERNETES_ALICLOUD_IDENTITY)
	return nil
}

// Instances return instance client
func (mgr *ClientMgr) Instances() *InstanceClient { return mgr.instance }

// Routes return routes client
func (mgr *ClientMgr) Routes() *RoutesClient { return mgr.routes }

// LoadBalancers return loadbalancer client
func (mgr *ClientMgr) LoadBalancers() *LoadBalancerClient { return mgr.loadbalancer }

// PrivateZones return PrivateZones client
func (mgr *ClientMgr) PrivateZones() *PrivateZoneClient { return mgr.privateZone }

// MetaData return MetaData client
func (mgr *ClientMgr) MetaData() IMetaData { return mgr.meta }

// Token base token info
type Token struct {
	AccessSecret string `json:"accessSecret,omitempty"`
	UID          string `json:"uid,omitempty"`
	Token        string `json:"token,omitempty"`
	AccessKey    string `json:"accesskey,omitempty"`
}

// TokenAuth is an interface of token auth method
type TokenAuth interface {
	NextToken() (*Token, error)
}

// AkAuthToken implement ak auth
type AkAuthToken struct{ ak *Token }

func (f *AkAuthToken) NextToken() (*Token, error) { return f.ak, nil }

type RamRoleToken struct {
	meta IMetaData
}

func (f *RamRoleToken) NextToken() (*Token, error) {
	roleName, err := f.meta.RoleName()
	if err != nil {
		return nil, fmt.Errorf("role name: %s", err.Error())
	}
	// use instance ram file way.
	role, err := f.meta.RamRoleToken(roleName)
	if err != nil {
		return nil, fmt.Errorf("ramrole token retrieve: %s", err.Error())
	}
	return &Token{
		AccessKey:    role.AccessKeyId,
		AccessSecret: role.AccessKeySecret,
		Token:        role.SecurityToken,
	}, nil
}

// ServiceToken is an implemention of service account auth
type ServiceToken struct {
	svcak    *Token
	execpath string
}

func (f *ServiceToken) NextToken() (*Token, error) {
	status := <-cmd.NewCmd(
		filepath.Join(f.execpath, "servicetoken"),
		fmt.Sprintf("--uid=%s", f.svcak.UID),
		fmt.Sprintf("--key=%s", f.svcak.AccessKey),
		fmt.Sprintf("--secret=%s", f.svcak.AccessSecret),
	).Start()
	if status.Error != nil {
		return nil, fmt.Errorf("invoke servicetoken: %s", status.Error.Error())
	}
	token := &Token{}
	err := json.Unmarshal(
		[]byte(strings.Join(status.Stdout, "")),
		token,
	)
	if err == nil {
		return token, nil
	}
	return nil, fmt.Errorf("unmarshal token: %s", err.Error())
}

// IMetaData metadata interface
type IMetaData interface {
	HostName() (string, error)
	ImageID() (string, error)
	InstanceID() (string, error)
	Mac() (string, error)
	NetworkType() (string, error)
	OwnerAccountID() (string, error)
	PrivateIPv4() (string, error)
	Region() (string, error)
	SerialNumber() (string, error)
	SourceAddress() (string, error)
	VpcCIDRBlock() (string, error)
	VpcID() (string, error)
	VswitchCIDRBlock() (string, error)
	Zone() (string, error)
	NTPConfigServers() ([]string, error)
	RoleName() (string, error)
	RamRoleToken(role string) (metadata.RoleAuth, error)
	VswitchID() (string, error)
}

// NewMetaData return new metadata
func NewMetaData() IMetaData {
	if cfg.Global.VpcID != "" &&
		cfg.Global.VswitchID != "" {
		glog.V(2).Infof("use mocked metadata server.")
		return &fakeMetaData{base: metadata.NewMetaData(nil)}
	}
	return metadata.NewMetaData(nil)
}

type fakeMetaData struct {
	base IMetaData
}

func (m *fakeMetaData) HostName() (string, error) {

	return "", fmt.Errorf("unimplemented")
}

func (m *fakeMetaData) ImageID() (string, error) {

	return "", fmt.Errorf("unimplemented")
}

func (m *fakeMetaData) InstanceID() (string, error) {

	return "fakedInstanceid", nil
}

func (m *fakeMetaData) Mac() (string, error) {

	return "", fmt.Errorf("unimplemented")
}

func (m *fakeMetaData) NetworkType() (string, error) {

	return "", fmt.Errorf("unimplemented")
}

func (m *fakeMetaData) OwnerAccountID() (string, error) {

	return "", fmt.Errorf("unimplemented")
}

func (m *fakeMetaData) PrivateIPv4() (string, error) {

	return "", fmt.Errorf("unimplemented")
}

func (m *fakeMetaData) Region() (string, error) {
	if cfg.Global.Region != "" {
		return cfg.Global.Region, nil
	}
	return m.base.Region()
}

func (m *fakeMetaData) SerialNumber() (string, error) {

	return "", fmt.Errorf("unimplemented")
}

func (m *fakeMetaData) SourceAddress() (string, error) {

	return "", fmt.Errorf("unimplemented")

}

func (m *fakeMetaData) VpcCIDRBlock() (string, error) {

	return "", fmt.Errorf("unimplemented")
}

func (m *fakeMetaData) VpcID() (string, error) {
	if cfg.Global.VpcID != "" {
		return cfg.Global.VpcID, nil
	}
	return m.base.VpcID()
}

func (m *fakeMetaData) VswitchCIDRBlock() (string, error) {

	return "", fmt.Errorf("unimplemented")
}

// zone1:vswitchid1,zone2:vswitch2
func (m *fakeMetaData) VswitchID() (string, error) {

	if cfg.Global.VswitchID == "" {
		// get vswitch id from meta server
		return m.base.VswitchID()
	}
	zlist := strings.Split(cfg.Global.VswitchID, ",")
	if len(zlist) == 1 {
		vSwitchs := strings.Split(cfg.Global.VswitchID, ":")
		if len(vSwitchs) == 2 {
			glog.Infof("only one vswitchid mode, %s", vSwitchs[1])
			return vSwitchs[1], nil
		}
		glog.Infof("simple vswitchid mode, %s", cfg.Global.VswitchID)
		return cfg.Global.VswitchID, nil
	}
	mzone, err := m.Zone()
	if err != nil {
		return "", fmt.Errorf("retrieve vswitchid error for %s", err.Error())
	}
	for _, zone := range zlist {
		vs := strings.Split(zone, ":")
		if len(vs) != 2 {
			return "", fmt.Errorf("cloud-config vswitch format error: %s", cfg.Global.VswitchID)
		}
		if vs[0] == mzone {
			return vs[1], nil
		}
	}
	glog.Infof("zone[%s] match failed, fallback with simple vswitch id mode, [%s]", mzone, cfg.Global.VswitchID)
	return cfg.Global.VswitchID, nil
}

func (m *fakeMetaData) EIPv4() (string, error) {

	return "", fmt.Errorf("unimplemented")
}

func (m *fakeMetaData) DNSNameServers() ([]string, error) {

	return []string{""}, fmt.Errorf("unimplemented")
}

func (m *fakeMetaData) NTPConfigServers() ([]string, error) {

	return []string{""}, fmt.Errorf("unimplemented")
}

func (m *fakeMetaData) Zone() (string, error) {
	if cfg.Global.ZoneID != "" {
		return cfg.Global.ZoneID, nil
	}
	return m.base.Zone()
}

func (m *fakeMetaData) RoleName() (string, error) {

	return m.base.RoleName()
}

func (m *fakeMetaData) RamRoleToken(role string) (metadata.RoleAuth, error) {

	return m.base.RamRoleToken(role)
}
<commit_msg>allow zone field to be empty when reading cloud config<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package alicloud

import (
	"encoding/json"
	"path/filepath"
	"time"

	"fmt"
	"github.com/denverdino/aliyungo/common"
	"github.com/denverdino/aliyungo/ecs"
	"github.com/denverdino/aliyungo/metadata"
	"github.com/denverdino/aliyungo/pvtz"
	"github.com/denverdino/aliyungo/slb"
	"github.com/go-cmd/cmd"
	"github.com/golang/glog"
	"k8s.io/apimachinery/pkg/util/wait"
	"strings"
)

// ROLE_NAME default kubernetes master role name
var ROLE_NAME = "KubernetesMasterRole"

// ASSUME_ROLE_NAME managed kubernetes role name
var ASSUME_ROLE_NAME = "AliyunCSManagedKubernetesRole"

// TOKEN_RESYNC_PERIOD default token sync period
var TOKEN_RESYNC_PERIOD = 10 * time.Minute

// ClientMgr client manager for aliyun sdk
type ClientMgr struct {
	stop <-chan struct{}

	token TokenAuth

	meta         IMetaData
	routes       *RoutesClient
	loadbalancer *LoadBalancerClient
	privateZone  *PrivateZoneClient
	instance     *InstanceClient
}

// NewClientMgr return a new client manager
func NewClientMgr(key, secret string) (*ClientMgr, error) {
	m := NewMetaData()
	region, err := m.Region()
	if err != nil {
		return nil, fmt.Errorf("can not determin region: %s", err.Error())
	}
	vpcid, err := m.VpcID()
	if err != nil {
		return nil, fmt.Errorf("can not determin vpcid: %s", err.Error())
	}
	ecsclient := ecs.NewECSClientWithSecurityToken(key, secret, "", common.Region(region))
	mgr := &ClientMgr{
		stop: make(<-chan struct{}, 1),
		meta: m,
		instance: &InstanceClient{
			c: ecsclient,
		},
		loadbalancer: &LoadBalancerClient{
			vpcid: vpcid,
			ins:   ecsclient,
			c:     slb.NewSLBClientWithSecurityToken(key, secret, "", common.Region(region)),
		},
		privateZone: &PrivateZoneClient{
			c: pvtz.NewPVTZClientWithSecurityToken(key, secret, "", common.Region("cn-hangzhou")),
		},
		routes: &RoutesClient{
			client: ecs.NewVPCClientWithSecurityToken(key, secret, "", common.Region(region)),
			region: region,
		},
	}

	if key == "" || secret == "" {
		glog.Infof("alicloud: use ramrole token mode without ak.")
		mgr.token = &RamRoleToken{meta: m}
	} else {
		inittoken := &Token{
			AccessKey:    key,
			AccessSecret: secret,
			UID:          cfg.Global.UID,
		}
		if inittoken.UID == "" {
			glog.Infof("alicloud: ak mode to authenticate user. without token and role assume")
			mgr.token = &AkAuthToken{ak: inittoken}
		} else {
			glog.Infof("alicloud: service account auth mode")
			mgr.token = &ServiceToken{svcak: inittoken}
		}
	}
	return mgr, nil
}

func (mgr *ClientMgr) Start(settoken func(mgr *ClientMgr, token *Token) error) error {
	initialized := false
	tokenfunc := func() {
		// refresh client token periodically
		token, err := mgr.token.NextToken()
		if err != nil {
			glog.Errorf("token retrieve: %s", err.Error())
			return
		}
		err = settoken(mgr, token)
		if err != nil {
			glog.Errorf("set token: %s", err.Error())
			return
		}
		initialized = true
	}

	go wait.Until(
		tokenfunc,
		time.Duration(TOKEN_RESYNC_PERIOD),
		mgr.stop,
	)
	return wait.ExponentialBackoff(
		wait.Backoff{
			Steps:    7,
			Duration: 1 * time.Second,
			Jitter:   1,
			Factor:   2,
		}, func() (done bool, err error) {
			tokenfunc()
			glog.Infof("wait for token ready")
			return initialized, nil
		},
	)
}

func RefreshToken(mgr *ClientMgr, token *Token) error {
	ecsclient := mgr.instance.c.(*ecs.Client)
	slbclient := mgr.loadbalancer.c.(*slb.Client)
	pvtzclient := mgr.privateZone.c.(*pvtz.Client)
	vpcclient := mgr.routes.client.(*ecs.Client)
	ecsclient.WithSecurityToken(token.Token).
		WithAccessKeyId(token.AccessKey).
		WithAccessKeySecret(token.AccessSecret)
	slbclient.WithSecurityToken(token.Token).
		WithAccessKeyId(token.AccessKey).
		WithAccessKeySecret(token.AccessSecret)
	pvtzclient.WithSecurityToken(token.Token).
		WithAccessKeyId(token.AccessKey).
		WithAccessKeySecret(token.AccessSecret)
	vpcclient.WithSecurityToken(token.Token).
		WithAccessKeyId(token.AccessKey).
		WithAccessKeySecret(token.AccessSecret)

	ecsclient.SetUserAgent(KUBERNETES_ALICLOUD_IDENTITY)
	slbclient.SetUserAgent(KUBERNETES_ALICLOUD_IDENTITY)
	pvtzclient.SetUserAgent(KUBERNETES_ALICLOUD_IDENTITY)
	vpcclient.SetUserAgent(KUBERNETES_ALICLOUD_IDENTITY)
	return nil
}

// Instances return instance client
func (mgr *ClientMgr) Instances() *InstanceClient { return mgr.instance }

// Routes return routes client
func (mgr *ClientMgr) Routes() *RoutesClient { return mgr.routes }

// LoadBalancers return loadbalancer client
func (mgr *ClientMgr) LoadBalancers() *LoadBalancerClient { return mgr.loadbalancer }

// PrivateZones return PrivateZones client
func (mgr *ClientMgr) PrivateZones() *PrivateZoneClient { return mgr.privateZone }

// MetaData return MetaData client
func (mgr *ClientMgr) MetaData() IMetaData { return mgr.meta }

// Token base token info
type Token struct {
	AccessSecret string `json:"accessSecret,omitempty"`
	UID          string `json:"uid,omitempty"`
	Token        string `json:"token,omitempty"`
	AccessKey    string `json:"accesskey,omitempty"`
}

// TokenAuth is an interface of token auth method
type TokenAuth interface {
	NextToken() (*Token, error)
}

// AkAuthToken implement ak auth
type AkAuthToken struct{ ak *Token }

func (f *AkAuthToken) NextToken() (*Token, error) { return f.ak, nil }

type RamRoleToken struct {
	meta IMetaData
}

func (f *RamRoleToken) NextToken() (*Token, error) {
	roleName, err := f.meta.RoleName()
	if err != nil {
		return nil, fmt.Errorf("role name: %s", err.Error())
	}
	// use instance ram file way.
	role, err := f.meta.RamRoleToken(roleName)
	if err != nil {
		return nil, fmt.Errorf("ramrole token retrieve: %s", err.Error())
	}
	return &Token{
		AccessKey:    role.AccessKeyId,
		AccessSecret: role.AccessKeySecret,
		Token:        role.SecurityToken,
	}, nil
}

// ServiceToken is an implemention of service account auth
type ServiceToken struct {
	svcak    *Token
	execpath string
}

func (f *ServiceToken) NextToken() (*Token, error) {
	status := <-cmd.NewCmd(
		filepath.Join(f.execpath, "servicetoken"),
		fmt.Sprintf("--uid=%s", f.svcak.UID),
		fmt.Sprintf("--key=%s", f.svcak.AccessKey),
		fmt.Sprintf("--secret=%s", f.svcak.AccessSecret),
	).Start()
	if status.Error != nil {
		return nil, fmt.Errorf("invoke servicetoken: %s", status.Error.Error())
	}
	token := &Token{}
	err := json.Unmarshal(
		[]byte(strings.Join(status.Stdout, "")),
		token,
	)
	if err == nil {
		return token, nil
	}
	return nil, fmt.Errorf("unmarshal token: %s", err.Error())
}

// IMetaData metadata interface
type IMetaData interface {
	HostName() (string, error)
	ImageID() (string, error)
	InstanceID() (string, error)
	Mac() (string, error)
	NetworkType() (string, error)
	OwnerAccountID() (string, error)
	PrivateIPv4() (string, error)
	Region() (string, error)
	SerialNumber() (string, error)
	SourceAddress() (string, error)
	VpcCIDRBlock() (string, error)
	VpcID() (string, error)
	VswitchCIDRBlock() (string, error)
	Zone() (string, error)
	NTPConfigServers() ([]string, error)
	RoleName() (string, error)
	RamRoleToken(role string) (metadata.RoleAuth, error)
	VswitchID() (string, error)
}

// NewMetaData return new metadata
func NewMetaData() IMetaData {
	if cfg.Global.VpcID != "" &&
		cfg.Global.VswitchID != "" {
		glog.V(2).Infof("use mocked metadata server.")
		return &fakeMetaData{base: metadata.NewMetaData(nil)}
	}
	return metadata.NewMetaData(nil)
}

type fakeMetaData struct {
	base IMetaData
}

func (m *fakeMetaData) HostName() (string, error) {

	return "", fmt.Errorf("unimplemented")
}

func (m *fakeMetaData) ImageID() (string, error) {

	return "", fmt.Errorf("unimplemented")
}

func (m *fakeMetaData) InstanceID() (string, error) {

	return "fakedInstanceid", nil
}

func (m *fakeMetaData) Mac() (string, error) {

	return "", fmt.Errorf("unimplemented")
}

func (m *fakeMetaData) NetworkType() (string, error) {

	return "", fmt.Errorf("unimplemented")
}

func (m *fakeMetaData) OwnerAccountID() (string, error) {

	return "", fmt.Errorf("unimplemented")
}

func (m *fakeMetaData) PrivateIPv4() (string, error) {

	return "", fmt.Errorf("unimplemented")
}

func (m *fakeMetaData) Region() (string, error) {
	if cfg.Global.Region != "" {
		return cfg.Global.Region, nil
	}
	return m.base.Region()
}

func (m *fakeMetaData) SerialNumber() (string, error) {

	return "", fmt.Errorf("unimplemented")
}

func (m *fakeMetaData) SourceAddress() (string, error) {

	return "", fmt.Errorf("unimplemented")

}

func (m *fakeMetaData) VpcCIDRBlock() (string, error) {

	return "", fmt.Errorf("unimplemented")
}

func (m *fakeMetaData) VpcID() (string, error) {
	if cfg.Global.VpcID != "" {
		return cfg.Global.VpcID, nil
	}
	return m.base.VpcID()
}

func (m *fakeMetaData) VswitchCIDRBlock() (string, error) {

	return "", fmt.Errorf("unimplemented")
}

// zone1:vswitchid1,zone2:vswitch2
func (m *fakeMetaData) VswitchID() (string, error) {

	if cfg.Global.VswitchID == "" {
		// get vswitch id from meta server
		return m.base.VswitchID()
	}
	zlist := strings.Split(cfg.Global.VswitchID, ",")
	if len(zlist) == 1 {
		vSwitchs := strings.Split(cfg.Global.VswitchID, ":")
		if len(vSwitchs) == 2 {
			glog.Infof("only one vswitchid mode, %s", vSwitchs[1])
			return vSwitchs[1], nil
		}
		glog.Infof("simple vswitchid mode, %s", cfg.Global.VswitchID)
		return cfg.Global.VswitchID, nil
	}
	mzone, err := m.Zone()
	if err != nil {
		return "", fmt.Errorf("retrieve vswitchid error for %s", err.Error())
	}
	for _, zone := range zlist {
		vs := strings.Split(zone, ":")
		if len(vs) != 2 {
			return "", fmt.Errorf("cloud-config vswitch format error: %s", cfg.Global.VswitchID)
		}
		if vs[0] == "" || vs[0] == mzone {
			return vs[1], nil
		}
	}
	glog.Infof("zone[%s] match failed, fallback with simple vswitch id mode, [%s]", mzone, cfg.Global.VswitchID)
	return cfg.Global.VswitchID, nil
}

func (m *fakeMetaData) EIPv4() (string, error) {

	return "", fmt.Errorf("unimplemented")
}

func (m *fakeMetaData) DNSNameServers() ([]string, error) {

	return []string{""}, fmt.Errorf("unimplemented")
}

func (m *fakeMetaData) NTPConfigServers() ([]string, error) {

	return []string{""}, fmt.Errorf("unimplemented")
}

func (m *fakeMetaData) Zone() (string, error) {
	if cfg.Global.ZoneID != "" {
		return cfg.Global.ZoneID, nil
	}
	return m.base.Zone()
}

func (m *fakeMetaData) RoleName() (string, error) {

	return m.base.RoleName()
}

func (m *fakeMetaData) RamRoleToken(role string) (metadata.RoleAuth, error) {

	return m.base.RamRoleToken(role)
}
<|endoftext|>


<commit_before>package margelet

import (
	"fmt"
	"gopkg.in/telegram-bot-api.v4"
	"log"
	"runtime/debug"
	"strings"
)

func handleUpdate(margelet *Margelet, update tgbotapi.Update) {
	defer func() {
		if err := recover(); err != nil {
			log.Println(string(debug.Stack()))

			var panicMessage string

			if margelet.verbose {
				panicMessage = fmt.Sprintf("Panic occured: %v", err)
			} else {
				panicMessage = "Panic occured!"
			}

			margelet.QuickSend(update.Message.Chat.ID, panicMessage)
		}
	}()

	switch {
	case update.Message != nil:
		message := update.Message
		margelet.ChatRepository.Add(message.Chat.ID)

		// If we have active session in this chat with this user, handle it first
		if command := margelet.SessionRepository.Command(message.Chat.ID, message.From.ID); len(command) > 0 {
			margelet.HandleSession(message, command)
		} else {
			if message.IsCommand() {
				handleCommand(margelet, message)
			} else {
				handleMessage(margelet, message)
			}
		}
	case update.InlineQuery != nil:
		handleInline(margelet, update.InlineQuery)
	case update.CallbackQuery != nil:
		handleCallback(margelet, update.CallbackQuery)
	}
}

func handleInline(margelet *Margelet, query *tgbotapi.InlineQuery) {
	handler := margelet.InlineHandler

	if handler != nil {
		handler.HandleInline(margelet, query)
	}
}

func handleCallback(margelet *Margelet, query *tgbotapi.CallbackQuery) {
	handler := margelet.CallbackHandler

	if handler != nil {
		handler.HandleCallback(margelet, query)
	}
}

func handleCommand(margelet *Margelet, message *tgbotapi.Message) {
	if authHandler, ok := margelet.CommandHandlers[strings.TrimSpace(message.Command())]; ok {
		if err := authHandler.Allow(message); err != nil {
			margelet.QuickSend(message.Chat.ID, "Authorization error: "+err.Error())
			return
		}
		err := authHandler.handler.HandleCommand(margelet, message)

		if err != nil {
			margelet.QuickSend(message.Chat.ID, "Error occured: "+err.Error())
		}
		return
	}

	if authHandler, ok := margelet.SessionHandlers[strings.TrimSpace(message.Command())]; ok {
		margelet.SessionRepository.Create(message.Chat.ID, message.From.ID, strings.TrimSpace(message.Command()))
		handleSession(margelet, message, authHandler)
		return
	}
}

func handleMessage(margelet *Margelet, message *tgbotapi.Message) {
	for _, handler := range margelet.MessageHandlers {
		err := handler.HandleMessage(margelet, message)

		if err != nil {
			margelet.QuickSend(message.Chat.ID, "Error occured: "+err.Error())
		}
	}
}

func handleSession(margelet *Margelet, message *tgbotapi.Message, authHandler authorizedSessionHandler) {
	if err := authHandler.Allow(message); err != nil {
		margelet.QuickSend(message.Chat.ID, "Authorization error: "+err.Error())
		return
	}
	if strings.TrimSpace(message.Command()) == "cancel" {
		authHandler.handler.CancelSession(margelet, message, margelet.SessionRepository.Dialog(message.Chat.ID, message.From.ID))
		margelet.SessionRepository.Remove(message.Chat.ID, message.From.ID)
		return
	}

	finish, err := authHandler.handler.HandleSession(margelet, message, margelet.SessionRepository.Dialog(message.Chat.ID, message.From.ID))

	if finish {
		margelet.SessionRepository.Remove(message.Chat.ID, message.From.ID)
		return
	}

	if err == nil {
		margelet.SessionRepository.Add(message.Chat.ID, message.From.ID, message)
	}

}
<commit_msg>some logging added<commit_after>package margelet

import (
	"fmt"
	"gopkg.in/telegram-bot-api.v4"
	"log"
	"runtime/debug"
	"strings"
)

func handleUpdate(margelet *Margelet, update tgbotapi.Update) {
	defer func() {
		if err := recover(); err != nil {
			log.Println(string(debug.Stack()))

			var panicMessage string

			if margelet.verbose {
				panicMessage = fmt.Sprintf("Panic occured: %v", err)
			} else {
				panicMessage = "Panic occured!"
			}

			margelet.QuickSend(update.Message.Chat.ID, panicMessage)
		}
	}()

	switch {
	case update.Message != nil:
		message := update.Message
		margelet.ChatRepository.Add(message.Chat.ID)

		// If we have active session in this chat with this user, handle it first
		if command := margelet.SessionRepository.Command(message.Chat.ID, message.From.ID); len(command) > 0 {
			margelet.HandleSession(message, command)
		} else {
			if message.IsCommand() {
				handleCommand(margelet, message)
			} else {
				handleMessage(margelet, message)
			}
		}
	case update.InlineQuery != nil:
		handleInline(margelet, update.InlineQuery)
	case update.CallbackQuery != nil:
		handleCallback(margelet, update.CallbackQuery)
	}
}

func handleInline(margelet *Margelet, query *tgbotapi.InlineQuery) {
	handler := margelet.InlineHandler

	if handler != nil {
		handler.HandleInline(margelet, query)
	}
}

func handleCallback(margelet *Margelet, query *tgbotapi.CallbackQuery) {
	handler := margelet.CallbackHandler

	if handler != nil {
		handler.HandleCallback(margelet, query)
	}
}

func handleCommand(margelet *Margelet, message *tgbotapi.Message) {
	if authHandler, ok := margelet.CommandHandlers[strings.TrimSpace(message.Command())]; ok {
		if err := authHandler.Allow(message); err != nil {
			margelet.QuickSend(message.Chat.ID, "Authorization error: "+err.Error())
			return
		}
		err := authHandler.handler.HandleCommand(margelet, message)

		if err != nil {
			margelet.QuickSend(message.Chat.ID, "Error occured: "+err.Error())
		}
		return
	}

	if authHandler, ok := margelet.SessionHandlers[strings.TrimSpace(message.Command())]; ok {
		margelet.SessionRepository.Create(message.Chat.ID, message.From.ID, strings.TrimSpace(message.Command()))
		handleSession(margelet, message, authHandler)
		return
	}
}

func handleMessage(margelet *Margelet, message *tgbotapi.Message) {
	for _, handler := range margelet.MessageHandlers {
		err := handler.HandleMessage(margelet, message)

		if err != nil {
			margelet.QuickSend(message.Chat.ID, "Error occured: "+err.Error())
		}
	}
}

func handleSession(margelet *Margelet, message *tgbotapi.Message, authHandler authorizedSessionHandler) {
	if err := authHandler.Allow(message); err != nil {
		margelet.QuickSend(message.Chat.ID, "Authorization error: "+err.Error())
		return
	}
	if strings.TrimSpace(message.Command()) == "cancel" {
		authHandler.handler.CancelSession(margelet, message, margelet.SessionRepository.Dialog(message.Chat.ID, message.From.ID))
		margelet.SessionRepository.Remove(message.Chat.ID, message.From.ID)
		return
	}

	finish, err := authHandler.handler.HandleSession(margelet, message, margelet.SessionRepository.Dialog(message.Chat.ID, message.From.ID))

	if finish {
		margelet.SessionRepository.Remove(message.Chat.ID, message.From.ID)
		return
	}

	if err != nil {
		log.Printf("Margelet handling session error %s", err.Error())
	}
	margelet.SessionRepository.Add(message.Chat.ID, message.From.ID, message)

}
<|endoftext|>


<commit_before>package lxc

import (
	"bufio"
	"errors"
	"strconv"
	"strings"
)

var (
	ErrCannotParse = errors.New("cannot parse raw input")
)

type lxcInfo struct {
	Running bool
	Pid     int
}

func parseLxcInfo(raw string) (*lxcInfo, error) {
	if raw == "" {
		return nil, ErrCannotParse
	}
	var (
		err  error
		s    = bufio.NewScanner(strings.NewReader(raw))
		info = &lxcInfo{}
	)
	for s.Scan() {
		text := s.Text()

		if s.Err() != nil {
			return nil, s.Err()
		}

		parts := strings.Split(text, ":")
		if len(parts) < 2 {
			continue
		}
		switch strings.TrimSpace(parts[0]) {
		case "state":
			info.Running = strings.TrimSpace(parts[1]) == "RUNNING"
		case "pid":
			info.Pid, err = strconv.Atoi(strings.TrimSpace(parts[1]))
			if err != nil {
				return nil, err
			}
		}
	}
	return info, nil
}
<commit_msg>Update parseLxcInfo to comply with new lxc1.0 format<commit_after>package lxc

import (
	"bufio"
	"errors"
	"strconv"
	"strings"
)

var (
	ErrCannotParse = errors.New("cannot parse raw input")
)

type lxcInfo struct {
	Running bool
	Pid     int
}

func parseLxcInfo(raw string) (*lxcInfo, error) {
	if raw == "" {
		return nil, ErrCannotParse
	}
	var (
		err  error
		s    = bufio.NewScanner(strings.NewReader(raw))
		info = &lxcInfo{}
	)
	for s.Scan() {
		text := s.Text()

		if s.Err() != nil {
			return nil, s.Err()
		}

		parts := strings.Split(text, ":")
		if len(parts) < 2 {
			continue
		}
		switch strings.ToLower(strings.TrimSpace(parts[0])) {
		case "state":
			info.Running = strings.TrimSpace(parts[1]) == "RUNNING"
		case "pid":
			info.Pid, err = strconv.Atoi(strings.TrimSpace(parts[1]))
			if err != nil {
				return nil, err
			}
		}
	}
	return info, nil
}
<|endoftext|>


<commit_before>package kafka

import (
	"strings"
	"sync"
	"time"

	"github.com/funkygao/gafka/cmd/kateway/structs"
	"github.com/funkygao/gafka/cmd/kguard/monitor"
	"github.com/funkygao/gafka/telemetry"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/go-metrics"
	log "github.com/funkygao/log4go"
)

func init() {
	monitor.RegisterWatcher("kafka.consumer", func() monitor.Watcher {
		return &WatchConsumers{
			Tick: time.Minute,
		}
	})
}

// WatchConsumers monitors num of kafka online consumer groups over the time.
type WatchConsumers struct {
	Zkzone *zk.ZkZone
	Stop   <-chan struct{}
	Tick   time.Duration
	Wg     *sync.WaitGroup

	offsetMtimeMap map[structs.GroupTopicPartition]time.Time

	consumerQps map[string]metrics.Meter
	lastOffsets map[string]int64
}

func (this *WatchConsumers) Init(ctx monitor.Context) {
	this.Zkzone = ctx.ZkZone()
	this.Stop = ctx.StopChan()
	this.Wg = ctx.Inflight()
	this.offsetMtimeMap = make(map[structs.GroupTopicPartition]time.Time, 100)
}

func (this *WatchConsumers) Run() {
	defer this.Wg.Done()

	this.consumerQps = make(map[string]metrics.Meter, 10)
	this.lastOffsets = make(map[string]int64, 10)

	ticker := time.NewTicker(this.Tick)
	defer ticker.Stop()

	frequentCommitTick := time.NewTicker(time.Second * 30)
	defer frequentCommitTick.Stop()

	consumerGroupsOnline := metrics.NewRegisteredGauge("consumer.groups.online", nil)
	consumerGroupsOffline := metrics.NewRegisteredGauge("consumer.groups.offline", nil)
	tooFrequentOffsetCommit := metrics.NewRegisteredGauge("consumer.frequent.offset.commit", nil)
	for {
		select {
		case <-this.Stop:
			log.Info("kafka.consumer stopped")
			return

		case <-ticker.C:
			online, offline := this.report()
			consumerGroupsOffline.Update(offline)
			consumerGroupsOnline.Update(online)

			this.runSubQpsTimer()

		case <-frequentCommitTick.C:
			tooFrequentOffsetCommit.Update(this.frequentOffsetCommit())
		}
	}
}

func (this *WatchConsumers) report() (online, offline int64) {
	this.Zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		for _, cgInfo := range zkcluster.ConsumerGroups() {
			if len(cgInfo) > 0 {
				online++
			} else {
				offline++
			}
		}
	})

	return
}

func (this *WatchConsumers) frequentOffsetCommit() (n int64) {
	const frequentThreshold = time.Second * 10

	this.Zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		for group, consumers := range zkcluster.ConsumersByGroup("") {
			for _, c := range consumers {
				if !c.Online {
					continue
				}

				if c.ConsumerZnode == nil {
					log.Warn("cluster[%s] group[%s] topic[%s/%s] unrecognized consumer", zkcluster.Name(), group, c.Topic, c.PartitionId)

					continue
				}

				gtp := structs.GroupTopicPartition{Group: group, Topic: c.Topic, PartitionID: c.PartitionId}
				if t, present := this.offsetMtimeMap[gtp]; present {
					if interval := c.Mtime.Time().Sub(t); interval < frequentThreshold {
						log.Error("cluster[%s] group[%s] topic[%s/%s] too frequent offset commit: %s", zkcluster.Name(), group, c.Topic, c.PartitionId, interval)

						n++
					}
				}

				this.offsetMtimeMap[gtp] = c.Mtime.Time()
			}
		}
	})

	return
}

func (this *WatchConsumers) runSubQpsTimer() {
	this.Zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		consumerGroups := zkcluster.ConsumerGroups()
		for group, _ := range consumerGroups {
			offsetMap := zkcluster.ConsumerOffsetsOfGroup(group)
			for topic, m := range offsetMap {
				offsetOfGroupOnTopic := int64(0)
				for _, offset := range m {
					offsetOfGroupOnTopic += offset
				}

				// cluster, topic, group, offset
				tag := telemetry.Tag(zkcluster.Name(), strings.Replace(topic, ".", "_", -1), strings.Replace(group, ".", "_", -1))
				if _, present := this.consumerQps[tag]; !present {
					this.consumerQps[tag] = metrics.NewRegisteredMeter(tag+"consumer.qps", nil)
				}
				lastOffset := this.lastOffsets[tag]
				if lastOffset == 0 {
					// first run
					this.lastOffsets[tag] = offsetOfGroupOnTopic
				} else {
					delta := offsetOfGroupOnTopic - lastOffset
					if delta >= 0 {
						this.consumerQps[tag].Mark(delta)
						this.lastOffsets[tag] = offsetOfGroupOnTopic
					} else {
						log.Warn("cluster[%s] topic[%s] group[%s] offset rewinds: %d %d",
							zkcluster.Name(), topic, group, offsetOfGroupOnTopic, lastOffset)
					}

				}
			}
		}
	})
}
<commit_msg>tweak of the log level<commit_after>package kafka

import (
	"strings"
	"sync"
	"time"

	"github.com/funkygao/gafka/cmd/kateway/structs"
	"github.com/funkygao/gafka/cmd/kguard/monitor"
	"github.com/funkygao/gafka/telemetry"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/go-metrics"
	log "github.com/funkygao/log4go"
)

func init() {
	monitor.RegisterWatcher("kafka.consumer", func() monitor.Watcher {
		return &WatchConsumers{
			Tick: time.Minute,
		}
	})
}

// WatchConsumers monitors num of kafka online consumer groups over the time.
type WatchConsumers struct {
	Zkzone *zk.ZkZone
	Stop   <-chan struct{}
	Tick   time.Duration
	Wg     *sync.WaitGroup

	offsetMtimeMap map[structs.GroupTopicPartition]time.Time

	consumerQps map[string]metrics.Meter
	lastOffsets map[string]int64
}

func (this *WatchConsumers) Init(ctx monitor.Context) {
	this.Zkzone = ctx.ZkZone()
	this.Stop = ctx.StopChan()
	this.Wg = ctx.Inflight()
	this.offsetMtimeMap = make(map[structs.GroupTopicPartition]time.Time, 100)
}

func (this *WatchConsumers) Run() {
	defer this.Wg.Done()

	this.consumerQps = make(map[string]metrics.Meter, 10)
	this.lastOffsets = make(map[string]int64, 10)

	ticker := time.NewTicker(this.Tick)
	defer ticker.Stop()

	frequentCommitTick := time.NewTicker(time.Second * 30)
	defer frequentCommitTick.Stop()

	consumerGroupsOnline := metrics.NewRegisteredGauge("consumer.groups.online", nil)
	consumerGroupsOffline := metrics.NewRegisteredGauge("consumer.groups.offline", nil)
	tooFrequentOffsetCommit := metrics.NewRegisteredGauge("consumer.frequent.offset.commit", nil)
	for {
		select {
		case <-this.Stop:
			log.Info("kafka.consumer stopped")
			return

		case <-ticker.C:
			online, offline := this.report()
			consumerGroupsOffline.Update(offline)
			consumerGroupsOnline.Update(online)

			this.runSubQpsTimer()

		case <-frequentCommitTick.C:
			tooFrequentOffsetCommit.Update(this.frequentOffsetCommit())
		}
	}
}

func (this *WatchConsumers) report() (online, offline int64) {
	this.Zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		for _, cgInfo := range zkcluster.ConsumerGroups() {
			if len(cgInfo) > 0 {
				online++
			} else {
				offline++
			}
		}
	})

	return
}

func (this *WatchConsumers) frequentOffsetCommit() (n int64) {
	const frequentThreshold = time.Second * 10

	this.Zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		for group, consumers := range zkcluster.ConsumersByGroup("") {
			for _, c := range consumers {
				if !c.Online {
					continue
				}

				if c.ConsumerZnode == nil {
					log.Warn("cluster[%s] group[%s] topic[%s/%s] unrecognized consumer", zkcluster.Name(), group, c.Topic, c.PartitionId)

					continue
				}

				gtp := structs.GroupTopicPartition{Group: group, Topic: c.Topic, PartitionID: c.PartitionId}
				if t, present := this.offsetMtimeMap[gtp]; present {
					if interval := c.Mtime.Time().Sub(t); interval < frequentThreshold {
						log.Warn("cluster[%s] group[%s] topic[%s/%s] too frequent offset commit: %s", zkcluster.Name(), group, c.Topic, c.PartitionId, interval)

						n++
					}
				}

				this.offsetMtimeMap[gtp] = c.Mtime.Time()
			}
		}
	})

	return
}

func (this *WatchConsumers) runSubQpsTimer() {
	this.Zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		consumerGroups := zkcluster.ConsumerGroups()
		for group, _ := range consumerGroups {
			offsetMap := zkcluster.ConsumerOffsetsOfGroup(group)
			for topic, m := range offsetMap {
				offsetOfGroupOnTopic := int64(0)
				for _, offset := range m {
					offsetOfGroupOnTopic += offset
				}

				// cluster, topic, group, offset
				tag := telemetry.Tag(zkcluster.Name(), strings.Replace(topic, ".", "_", -1), strings.Replace(group, ".", "_", -1))
				if _, present := this.consumerQps[tag]; !present {
					this.consumerQps[tag] = metrics.NewRegisteredMeter(tag+"consumer.qps", nil)
				}
				lastOffset := this.lastOffsets[tag]
				if lastOffset == 0 {
					// first run
					this.lastOffsets[tag] = offsetOfGroupOnTopic
				} else {
					delta := offsetOfGroupOnTopic - lastOffset
					if delta >= 0 {
						this.consumerQps[tag].Mark(delta)
						this.lastOffsets[tag] = offsetOfGroupOnTopic
					} else {
						log.Warn("cluster[%s] topic[%s] group[%s] offset rewinds: %d %d",
							zkcluster.Name(), topic, group, offsetOfGroupOnTopic, lastOffset)
					}

				}
			}
		}
	})
}
<|endoftext|>


<commit_before>package uploader

import (
	"bytes"
	"fmt"
	"io"
	"time"

	"github.com/lomik/carbon-clickhouse/helper/RowBinary"
)

type Series struct {
	*cached
	isReverse bool
}

var _ Uploader = &Series{}
var _ UploaderWithReset = &Series{}

func NewSeries(base *Base, reverse bool) *Series {
	u := &Series{}
	u.cached = newCached(base)
	u.cached.parser = u.parseFile
	u.isReverse = reverse
	return u
}

func (u *Series) parseFile(filename string, out io.Writer) (uint64, map[string]bool, error) {
	var reader *RowBinary.Reader
	var err error
	var n uint64

	reader, err = RowBinary.NewReader(filename, u.isReverse)
	if err != nil {
		return n, nil, err
	}
	defer reader.Close()

	version := uint32(time.Now().Unix())
	newSeries := make(map[string]bool)
	wb := RowBinary.GetWriteBuffer()

	var level int

LineLoop:
	for {
		name, err := reader.ReadRecord()
		if err != nil { // io.EOF or corrupted file
			break
		}

		// skip tagged
		if bytes.IndexByte(name, '?') >= 0 {
			continue
		}

		key := fmt.Sprintf("%d:%s", reader.Days(), unsafeString(name))

		if u.existsCache.Exists(key) {
			continue LineLoop
		}

		if newSeries[key] {
			continue LineLoop
		}
		n++

		level = pathLevel(name)

		wb.Reset()

		newSeries[key] = true
		wb.WriteUint16(reader.Days())
		wb.WriteUint32(uint32(level))
		wb.WriteBytes(name)
		wb.WriteUint32(version)

		_, err = out.Write(wb.Bytes())
		if err != nil {
			return n, nil, err
		}
	}

	wb.Release()

	return n, newSeries, nil
}
<commit_msg>uploader: reduce allocations in Series.parseFile<commit_after>package uploader

import (
	"bytes"
	"io"
	"strconv"
	"time"

	"github.com/lomik/carbon-clickhouse/helper/RowBinary"
)

type Series struct {
	*cached
	isReverse bool
}

var _ Uploader = &Series{}
var _ UploaderWithReset = &Series{}

func NewSeries(base *Base, reverse bool) *Series {
	u := &Series{}
	u.cached = newCached(base)
	u.cached.parser = u.parseFile
	u.isReverse = reverse
	return u
}

func (u *Series) parseFile(filename string, out io.Writer) (uint64, map[string]bool, error) {
	var reader *RowBinary.Reader
	var err error
	var n uint64

	reader, err = RowBinary.NewReader(filename, u.isReverse)
	if err != nil {
		return n, nil, err
	}
	defer reader.Close()

	version := uint32(time.Now().Unix())
	newSeries := make(map[string]bool)
	wb := RowBinary.GetWriteBuffer()

	var level int

LineLoop:
	for {
		name, err := reader.ReadRecord()
		if err != nil { // io.EOF or corrupted file
			break
		}

		// skip tagged
		if bytes.IndexByte(name, '?') >= 0 {
			continue
		}

		key := strconv.Itoa(int(reader.Days())) + ":" + unsafeString(name)

		if u.existsCache.Exists(key) {
			continue LineLoop
		}

		if newSeries[key] {
			continue LineLoop
		}
		n++

		level = pathLevel(name)

		wb.Reset()

		newSeries[key] = true
		wb.WriteUint16(reader.Days())
		wb.WriteUint32(uint32(level))
		wb.WriteBytes(name)
		wb.WriteUint32(version)

		_, err = out.Write(wb.Bytes())
		if err != nil {
			return n, nil, err
		}
	}

	wb.Release()

	return n, newSeries, nil
}
<|endoftext|>


<commit_before>package uploader

import (
	"bytes"
	"fmt"
	"io"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/lomik/carbon-clickhouse/helper/RowBinary"
)

type Tagged struct {
	*cached
	ignoredMetrics map[string]bool
}

var _ Uploader = &Tagged{}
var _ UploaderWithReset = &Tagged{}

func NewTagged(base *Base) *Tagged {
	u := &Tagged{}
	u.cached = newCached(base)
	u.cached.parser = u.parseFile
	u.query = fmt.Sprintf("%s (Date, Tag1, Path, Tags, Version)", u.config.TableName)

	u.ignoredMetrics = make(map[string]bool, len(u.config.IgnoredTaggedMetrics))
	for _, metric := range u.config.IgnoredTaggedMetrics {
		u.ignoredMetrics[metric] = true
	}

	return u
}

func urlParse(rawurl string) (*url.URL, error) {
	p := strings.IndexByte(rawurl, '?')
	if p < 0 {
		return url.Parse(rawurl)
	}
	m, err := url.Parse(rawurl[p:])
	if m != nil {
		m.Path, err = url.PathUnescape(rawurl[:p])
		if err != nil {
			return nil, err
		}
	}
	return m, err
}

func (u *Tagged) parseFile(filename string, out io.Writer) (uint64, map[string]bool, error) {
	var reader *RowBinary.Reader
	var err error
	var n uint64

	reader, err = RowBinary.NewReader(filename, false)
	if err != nil {
		return n, nil, err
	}
	defer reader.Close()

	version := uint32(time.Now().Unix())

	newTagged := make(map[string]bool)

	wb := RowBinary.GetWriteBuffer()
	tagsBuf := RowBinary.GetWriteBuffer()
	defer wb.Release()
	defer tagsBuf.Release()

	tag1 := make([]string, 0)

LineLoop:
	for {
		name, err := reader.ReadRecord()
		if err != nil { // io.EOF or corrupted file
			break
		}

		// skip not tagged
		if bytes.IndexByte(name, '?') < 0 {
			continue
		}

		key := strconv.Itoa(int(reader.Days())) + ":" + unsafeString(name)

		if u.existsCache.Exists(key) {
			continue LineLoop
		}

		if newTagged[key] {
			continue LineLoop
		}
		n++

		m, err := urlParse(unsafeString(name))
		if err != nil {
			continue
		}

		newTagged[key] = true

		wb.Reset()
		tagsBuf.Reset()
		tag1 = tag1[:0]

		t := fmt.Sprintf("__name__=%s", m.Path)
		tag1 = append(tag1, t)
		tagsBuf.WriteString(t)

		// don't upload any other tag but __name__
		// if either main metric (m.Path) or each metric (*) is ignored
		ignoreAllButName := u.ignoredMetrics[m.Path] || u.ignoredMetrics["*"]
		tagsWritten := 1
		for k, v := range m.Query() {
			t := fmt.Sprintf("%s=%s", k, v[0])
			tagsBuf.WriteString(t)
			tagsWritten++

			if !ignoreAllButName {
				tag1 = append(tag1, t)
			}
		}

		for i := 0; i < len(tag1); i++ {
			wb.WriteUint16(reader.Days())
			wb.WriteString(tag1[i])
			wb.WriteBytes(name)
			wb.WriteUVarint(uint64(tagsWritten))
			wb.Write(tagsBuf.Bytes())
			wb.WriteUint32(version)
		}

		_, err = out.Write(wb.Bytes())
		if err != nil {
			return n, nil, err
		}
	}

	return n, newTagged, nil
}
<commit_msg>uploader: refactor tagged.parseFile<commit_after>package uploader

import (
	"bytes"
	"fmt"
	"io"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/lomik/carbon-clickhouse/helper/RowBinary"
	"go.uber.org/zap"
)

type Tagged struct {
	*cached
	ignoredMetrics map[string]bool
}

var _ Uploader = &Tagged{}
var _ UploaderWithReset = &Tagged{}

func NewTagged(base *Base) *Tagged {
	u := &Tagged{}
	u.cached = newCached(base)
	u.cached.parser = u.parseFile
	u.query = fmt.Sprintf("%s (Date, Tag1, Path, Tags, Version)", u.config.TableName)

	u.ignoredMetrics = make(map[string]bool, len(u.config.IgnoredTaggedMetrics))
	for _, metric := range u.config.IgnoredTaggedMetrics {
		u.ignoredMetrics[metric] = true
	}

	return u
}

func urlParse(rawurl string) (*url.URL, error) {
	p := strings.IndexByte(rawurl, '?')
	if p < 0 {
		return url.Parse(rawurl)
	}
	m, err := url.Parse(rawurl[p:])
	if m != nil {
		m.Path, err = url.PathUnescape(rawurl[:p])
		if err != nil {
			return nil, err
		}
	}
	return m, err
}

func (u *Tagged) parseName(name string, days uint16,
	// reusable buffers
	tag1 []string, wb *RowBinary.WriteBuffer, tagsBuf *RowBinary.WriteBuffer) error {

	m, err := urlParse(name)
	if err != nil {
		return err
	}

	version := uint32(time.Now().Unix())

	wb.Reset()
	tagsBuf.Reset()
	tag1 = tag1[:0]

	t := fmt.Sprintf("__name__=%s", m.Path)
	tag1 = append(tag1, t)
	tagsBuf.WriteString(t)

	// don't upload any other tag but __name__
	// if either main metric (m.Path) or each metric (*) is ignored
	ignoreAllButName := u.ignoredMetrics[m.Path] || u.ignoredMetrics["*"]
	tagsWritten := 1
	for k, v := range m.Query() {
		t := fmt.Sprintf("%s=%s", k, v[0])
		tagsBuf.WriteString(t)
		tagsWritten++

		if !ignoreAllButName {
			tag1 = append(tag1, t)
		}
	}

	for i := 0; i < len(tag1); i++ {
		wb.WriteUint16(days)
		wb.WriteString(tag1[i])
		wb.WriteString(name)
		wb.WriteUVarint(uint64(tagsWritten))
		wb.Write(tagsBuf.Bytes())
		wb.WriteUint32(version)
	}

	return nil
}

func (u *Tagged) parseFile(filename string, out io.Writer) (uint64, map[string]bool, error) {
	var reader *RowBinary.Reader
	var err error
	var n uint64

	reader, err = RowBinary.NewReader(filename, false)
	if err != nil {
		return n, nil, err
	}
	defer reader.Close()

	newTagged := make(map[string]bool)

	wb := RowBinary.GetWriteBuffer()
	tagsBuf := RowBinary.GetWriteBuffer()
	defer wb.Release()
	defer tagsBuf.Release()

	tag1 := make([]string, 0)

LineLoop:
	for {
		name, err := reader.ReadRecord()
		if err != nil { // io.EOF or corrupted file
			break
		}

		// skip not tagged
		if bytes.IndexByte(name, '?') < 0 {
			continue
		}

		nameStr := unsafeString(name)

		days := reader.Days()
		key := strconv.Itoa(int(days)) + ":" + nameStr
		if u.existsCache.Exists(key) {
			continue LineLoop
		}

		if newTagged[key] {
			// already processed
			continue LineLoop
		}

		n++

		if err = u.parseName(nameStr, days, tag1, wb, tagsBuf); err != nil {
			u.logger.Warn("parse",
				zap.String("metric", string(name)), zap.String("type", "tagged"), zap.String("name", filename), zap.Error(err),
			)
			continue LineLoop
		} else if _, err = out.Write(wb.Bytes()); err != nil {
			return n, nil, err
		}
		newTagged[key] = true
	}

	return n, newTagged, nil
}
<|endoftext|>


<commit_before>package command

import (
	"os"
	"path/filepath"
	"time"

	"github.com/mackerelio/mackerel-agent/config"
	"github.com/mackerelio/mackerel-agent/metadata"
)

func metadataGenerators(conf *config.Config) []*metadata.Generator {
	generators := make([]*metadata.Generator, 0, len(conf.MetadataPlugins))

	workdir := os.Getenv("MACKEREL_PLUGIN_WORKDIR")
	if workdir == "" {
		workdir = os.TempDir()
	}

	for name, pluginConfig := range conf.MetadataPlugins {
		generator := &metadata.Generator{
			Name:     name,
			Config:   pluginConfig,
			Tempfile: filepath.Join(workdir, "mackerel-metadata", name),
		}
		logger.Debugf("Metadata plugin generator created: %#v %#v", generator, generator.Config)
		generators = append(generators, generator)
	}

	return generators
}

type metadataResult struct {
	namespace string
	metadata  interface{}
}

func runMetadataLoop(c *Context, termMetadataCh <-chan struct{}, quit <-chan struct{}) {
	resultCh := make(chan *metadataResult)
	for _, g := range c.Agent.MetadataGenerators {
		go runEachMetadataLoop(g, resultCh, quit)
	}

	exit := false
	for !exit {
		select {
		case <-time.After(1 * time.Minute):
		case <-termMetadataCh:
			logger.Debugf("received 'term' chan for metadata loop")
			exit = true
		}

		results := []*metadataResult{}
	ConsumeResults:
		for {
			select {
			case result := <-resultCh:
				results = append(results, result)
			default:
				break ConsumeResults
			}
		}

		for _, result := range results {
			resp, err := c.API.PutMetadata(c.Host.ID, result.namespace, result.metadata)
			// retry on 5XX errors
			if resp != nil && resp.StatusCode >= 500 {
				logger.Errorf("put metadata %q failed: status %s", result.namespace, resp.Status)
				resultCh <- &metadataResult{
					namespace: result.namespace,
					metadata:  result.metadata,
				}
				continue
			}
			if err != nil {
				logger.Errorf("put metadata %q failed: %v", result.namespace, err)
				clearMetadataCache(c.Agent.MetadataGenerators, result.namespace)
				continue
			}
		}
	}
}

func clearMetadataCache(generators []*metadata.Generator, namespace string) {
	for _, g := range generators {
		if g.Name == namespace {
			g.Clear()
			return
		}
	}
}

func runEachMetadataLoop(g *metadata.Generator, resultCh chan<- *metadataResult, quit <-chan struct{}) {
	interval := g.Interval()
	nextInterval := 10 * time.Second
	nextTime := time.Now()

	for {
		select {
		case <-time.After(nextInterval):
			metadata, err := g.Fetch()

			// case for laptop sleep mode (now >> nextTime + interval)
			now := time.Now()
			nextInterval = interval - (now.Sub(nextTime) % interval)
			nextTime = now.Add(nextInterval)

			if err != nil {
				logger.Warningf("metadata plugin %q: %s", g.Name, err.Error())
				continue
			}

			if !g.IsChanged(metadata) {
				logger.Debugf("skipping metadata %q, metadata does not change", g.Name)
				continue
			}

			if err := g.Save(metadata); err != nil {
				logger.Warningf("metadata plugin %q: %s", g.Name, err.Error())
				continue
			}

			logger.Debugf("generated metadata %q (saved cache to file: %s)", g.Name, g.Tempfile)
			resultCh <- &metadataResult{
				namespace: g.Name,
				metadata:  metadata,
			}

		case <-quit:
			return
		}
	}
}
<commit_msg>change results from slice to map to prefer the new result and avoid infinite retries<commit_after>package command

import (
	"os"
	"path/filepath"
	"time"

	"github.com/mackerelio/mackerel-agent/config"
	"github.com/mackerelio/mackerel-agent/metadata"
)

func metadataGenerators(conf *config.Config) []*metadata.Generator {
	generators := make([]*metadata.Generator, 0, len(conf.MetadataPlugins))

	workdir := os.Getenv("MACKEREL_PLUGIN_WORKDIR")
	if workdir == "" {
		workdir = os.TempDir()
	}

	for name, pluginConfig := range conf.MetadataPlugins {
		generator := &metadata.Generator{
			Name:     name,
			Config:   pluginConfig,
			Tempfile: filepath.Join(workdir, "mackerel-metadata", name),
		}
		logger.Debugf("Metadata plugin generator created: %#v %#v", generator, generator.Config)
		generators = append(generators, generator)
	}

	return generators
}

type metadataResult struct {
	namespace string
	metadata  interface{}
}

func runMetadataLoop(c *Context, termMetadataCh <-chan struct{}, quit <-chan struct{}) {
	resultCh := make(chan *metadataResult)
	for _, g := range c.Agent.MetadataGenerators {
		go runEachMetadataLoop(g, resultCh, quit)
	}

	exit := false
	for !exit {
		select {
		case <-time.After(1 * time.Minute):
		case <-termMetadataCh:
			logger.Debugf("received 'term' chan for metadata loop")
			exit = true
		}

		results := make(map[string]*metadataResult)
	ConsumeResults:
		for {
			select {
			case result := <-resultCh:
				// prefer new result to old metadata which remains to be posted on retry
				// and also avoid infinite number of retries
				results[result.namespace] = result
			default:
				break ConsumeResults
			}
		}

		for _, result := range results {
			resp, err := c.API.PutMetadata(c.Host.ID, result.namespace, result.metadata)
			// retry on 5XX errors
			if resp != nil && resp.StatusCode >= 500 {
				logger.Errorf("put metadata %q failed: status %s", result.namespace, resp.Status)
				resultCh <- &metadataResult{
					namespace: result.namespace,
					metadata:  result.metadata,
				}
				continue
			}
			if err != nil {
				logger.Errorf("put metadata %q failed: %v", result.namespace, err)
				clearMetadataCache(c.Agent.MetadataGenerators, result.namespace)
				continue
			}
		}
		results = nil
	}
}

func clearMetadataCache(generators []*metadata.Generator, namespace string) {
	for _, g := range generators {
		if g.Name == namespace {
			g.Clear()
			return
		}
	}
}

func runEachMetadataLoop(g *metadata.Generator, resultCh chan<- *metadataResult, quit <-chan struct{}) {
	interval := g.Interval()
	nextInterval := 10 * time.Second
	nextTime := time.Now()

	for {
		select {
		case <-time.After(nextInterval):
			metadata, err := g.Fetch()

			// case for laptop sleep mode (now >> nextTime + interval)
			now := time.Now()
			nextInterval = interval - (now.Sub(nextTime) % interval)
			nextTime = now.Add(nextInterval)

			if err != nil {
				logger.Warningf("metadata plugin %q: %s", g.Name, err.Error())
				continue
			}

			if !g.IsChanged(metadata) {
				logger.Debugf("skipping metadata %q, metadata does not change", g.Name)
				continue
			}

			if err := g.Save(metadata); err != nil {
				logger.Warningf("metadata plugin %q: %s", g.Name, err.Error())
				continue
			}

			logger.Debugf("generated metadata %q (saved cache to file: %s)", g.Name, g.Tempfile)
			resultCh <- &metadataResult{
				namespace: g.Name,
				metadata:  metadata,
			}

		case <-quit:
			return
		}
	}
}
<|endoftext|>


<commit_before>package urltest

import (
	log "github.com/Sirupsen/logrus"
	"net/http"
)

// TestUrlIsLockedDown checks a URL to see if it returns a 401 or has
// X-Auto-Login headers
func TestUrlIsLockedDown(url string) (bool, error) {

	lockeddown := false
	resp, err := http.Get(url)
	if err != nil {
		log.WithFields(log.Fields{"err": err}).Fatal("Failed to get URL")
	}

	// HTTP 401, or User Service login
	if (resp.StatusCode == 401) || (len(resp.Header["X-Auto-Login"]) > 0) {
		lockeddown = true
	}

	log.Debug("url: ", url, ", lockeddown: ", lockeddown)

	return lockeddown, err
}
<commit_msg>Made urltest more flexible to allow for connections reset by peer<commit_after>package urltest

import (
	log "github.com/Sirupsen/logrus"
	"net/http"
)

// TestUrlIsLockedDown checks a URL to see if it returns a 401 or has
// X-Auto-Login headers
func TestUrlIsLockedDown(url string) (bool, error) {

	lockeddown := false
	resp, err := http.Get(url)
	if err != nil {
		log.Warnf("Failed to get URL [%s]:\n%s", resp.Status, err)
		return lockeddown, err
	}

	// HTTP 401, or User Service login
	if (resp.StatusCode == 401) || (len(resp.Header["X-Auto-Login"]) > 0) {
		lockeddown = true
	}

	log.Debug("url: ", url, ", lockeddown: ", lockeddown)

	return lockeddown, err
}
<|endoftext|>


<commit_before>package v3

import (
	"code.cloudfoundry.org/cli/actors/v3actions"
	"code.cloudfoundry.org/cli/commands"
	"code.cloudfoundry.org/cli/commands/flags"
	"code.cloudfoundry.org/cli/commands/v3/common"
)

//go:generate counterfeiter . TerminateTaskActor

type TerminateTaskActor interface {
	GetApplicationByNameAndSpace(appName string, spaceGUID string) (v3actions.Application, v3actions.Warnings, error)
	GetTaskBySequenceIDAndApplication(sequenceID int, appGUID string) (v3actions.Task, v3actions.Warnings, error)
	TerminateTask(taskGUID string) (v3actions.Task, v3actions.Warnings, error)
}

type TerminateTaskCommand struct {
	RequiredArgs    flags.TerminateTaskArgs `positional-args:"yes"`
	usage           interface{}             `usage:"CF_NAME terminate-task APP_NAME TASK_ID\n\nEXAMPLES:\n   CF_NAME terminate-task my-app 3"`
	relatedCommands interface{}             `related_commands:"tasks"`

	UI     commands.UI
	Actor  TerminateTaskActor
	Config commands.Config
}

func (cmd *TerminateTaskCommand) Setup(config commands.Config, ui commands.UI) error {
	cmd.UI = ui
	cmd.Config = config

	client, err := common.NewClients(config, ui)
	if err != nil {
		return err
	}
	cmd.Actor = v3actions.NewActor(client)

	return nil
}

func (cmd TerminateTaskCommand) Execute(args []string) error {
	err := common.CheckTarget(cmd.Config, true, true)
	if err != nil {
		return err
	}

	space := cmd.Config.TargetedSpace()

	user, err := cmd.Config.CurrentUser()
	if err != nil {
		return err
	}

	application, warnings, err := cmd.Actor.GetApplicationByNameAndSpace(cmd.RequiredArgs.AppName, space.GUID)
	cmd.UI.DisplayWarnings(warnings)
	if err != nil {
		return common.HandleError(err)
	}

	task, warnings, err := cmd.Actor.GetTaskBySequenceIDAndApplication(cmd.RequiredArgs.SequenceID, application.GUID)
	cmd.UI.DisplayWarnings(warnings)
	if err != nil {
		return common.HandleError(err)
	}

	cmd.UI.DisplayHeaderFlavorText("Terminating task {{.TaskSequenceID}} of app {{.AppName}} in org {{.OrgName}} / space {{.SpaceName}} as {{.CurrentUser}}...",
		map[string]interface{}{
			"TaskSequenceID": cmd.RequiredArgs.SequenceID,
			"AppName":        cmd.RequiredArgs.AppName,
			"OrgName":        cmd.Config.TargetedOrganization().Name,
			"SpaceName":      space.Name,
			"CurrentUser":    user.Name,
		})

	_, warnings, err = cmd.Actor.TerminateTask(task.GUID)
	cmd.UI.DisplayWarnings(warnings)
	if err != nil {
		return common.HandleError(err)
	}

	cmd.UI.DisplayOK()

	return nil
}
<commit_msg>fix incorrect method call<commit_after>package v3

import (
	"code.cloudfoundry.org/cli/actors/v3actions"
	"code.cloudfoundry.org/cli/commands"
	"code.cloudfoundry.org/cli/commands/flags"
	"code.cloudfoundry.org/cli/commands/v3/common"
)

//go:generate counterfeiter . TerminateTaskActor

type TerminateTaskActor interface {
	GetApplicationByNameAndSpace(appName string, spaceGUID string) (v3actions.Application, v3actions.Warnings, error)
	GetTaskBySequenceIDAndApplication(sequenceID int, appGUID string) (v3actions.Task, v3actions.Warnings, error)
	TerminateTask(taskGUID string) (v3actions.Task, v3actions.Warnings, error)
}

type TerminateTaskCommand struct {
	RequiredArgs    flags.TerminateTaskArgs `positional-args:"yes"`
	usage           interface{}             `usage:"CF_NAME terminate-task APP_NAME TASK_ID\n\nEXAMPLES:\n   CF_NAME terminate-task my-app 3"`
	relatedCommands interface{}             `related_commands:"tasks"`

	UI     commands.UI
	Actor  TerminateTaskActor
	Config commands.Config
}

func (cmd *TerminateTaskCommand) Setup(config commands.Config, ui commands.UI) error {
	cmd.UI = ui
	cmd.Config = config

	client, err := common.NewClients(config, ui)
	if err != nil {
		return err
	}
	cmd.Actor = v3actions.NewActor(client)

	return nil
}

func (cmd TerminateTaskCommand) Execute(args []string) error {
	err := common.CheckTarget(cmd.Config, true, true)
	if err != nil {
		return err
	}

	space := cmd.Config.TargetedSpace()

	user, err := cmd.Config.CurrentUser()
	if err != nil {
		return err
	}

	application, warnings, err := cmd.Actor.GetApplicationByNameAndSpace(cmd.RequiredArgs.AppName, space.GUID)
	cmd.UI.DisplayWarnings(warnings)
	if err != nil {
		return common.HandleError(err)
	}

	task, warnings, err := cmd.Actor.GetTaskBySequenceIDAndApplication(cmd.RequiredArgs.SequenceID, application.GUID)
	cmd.UI.DisplayWarnings(warnings)
	if err != nil {
		return common.HandleError(err)
	}

	cmd.UI.DisplayTextWithFlavor("Terminating task {{.TaskSequenceID}} of app {{.AppName}} in org {{.OrgName}} / space {{.SpaceName}} as {{.CurrentUser}}...",
		map[string]interface{}{
			"TaskSequenceID": cmd.RequiredArgs.SequenceID,
			"AppName":        cmd.RequiredArgs.AppName,
			"OrgName":        cmd.Config.TargetedOrganization().Name,
			"SpaceName":      space.Name,
			"CurrentUser":    user.Name,
		})

	_, warnings, err = cmd.Actor.TerminateTask(task.GUID)
	cmd.UI.DisplayWarnings(warnings)
	if err != nil {
		return common.HandleError(err)
	}

	cmd.UI.DisplayOK()

	return nil
}
<|endoftext|>


<commit_before>package cmd

import (
	"bytes"
	"context"

	"github.com/mitchellh/cli"
	"github.com/nerdalize/nerd/pkg/kubevisor"
	"github.com/nerdalize/nerd/svc"
	"github.com/pkg/errors"
)

//JobLogs command
type JobLogs struct {
	KubeOpts
	Tail int64 `long:"tail" short:"t" description:"only return the oldest N lines of the process logs"`

	*command
}

//JobLogsFactory creates the command
func JobLogsFactory(ui cli.Ui) cli.CommandFactory {
	cmd := &JobLogs{}
	cmd.command = createCommand(ui, cmd.Execute, cmd.Description, cmd.Usage, cmd)
	return func() (cli.Command, error) {
		return cmd, nil
	}
}

//Execute runs the command
func (cmd *JobLogs) Execute(args []string) (err error) {
	if len(args) < 1 {
		return errors.New(MessageNotEnoughArguments)
	}

	kopts := cmd.KubeOpts
	deps, err := NewDeps(cmd.Logger(), kopts)
	if err != nil {
		return errors.Wrap(err, "failed to configure")
	}

	ctx := context.Background()
	ctx, cancel := context.WithTimeout(ctx, cmd.Timeout)
	defer cancel()

	in := &svc.FetchJobLogsInput{
		Name: args[0],
		Tail: cmd.Tail,
	}

	kube := svc.NewKube(deps, kopts.Namespace)
	out, err := kube.FetchJobLogs(ctx, in)
	if err != nil {
		return renderServiceError(err, "failed to fetch job logs")
	}

	lines := string(bytes.TrimSpace(out.Data))
	if len(lines) < 1 {
		cmd.out.Info("-- no visible logs returned --")
		return nil
	}

	cmd.out.Output(string(out.Data))
	if int64(len(out.Data)) == kubevisor.MaxLogBytes {
		cmd.out.Info("-- logs are trimmed after this point --")
	}

	return nil
}

// Description returns long-form help text
func (cmd *JobLogs) Description() string { return cmd.Synopsis() }

// Synopsis returns a one-line
func (cmd *JobLogs) Synopsis() string { return "Return logs for a running job" }

// Usage shows usage
func (cmd *JobLogs) Usage() string { return "nerd job logs [NAME]" }
<commit_msg>fix compile error<commit_after>package cmd

import (
	"bytes"
	"context"

	"github.com/mitchellh/cli"
	"github.com/nerdalize/nerd/pkg/kubevisor"
	"github.com/nerdalize/nerd/svc"
	"github.com/pkg/errors"
)

//JobLogs command
type JobLogs struct {
	KubeOpts
	Tail int64 `long:"tail" short:"t" description:"only return the oldest N lines of the process logs"`

	*command
}

//JobLogsFactory creates the command
func JobLogsFactory(ui cli.Ui) cli.CommandFactory {
	cmd := &JobLogs{}
	cmd.command = createCommand(ui, cmd.Execute, cmd.Description, cmd.Usage, cmd)
	return func() (cli.Command, error) {
		return cmd, nil
	}
}

//Execute runs the command
func (cmd *JobLogs) Execute(args []string) (err error) {
	if len(args) < 1 {
		return errors.New(MessageNotEnoughArguments)
	}

	kopts := cmd.KubeOpts
	deps, err := NewDeps(cmd.Logger(), kopts)
	if err != nil {
		return errors.Wrap(err, "failed to configure")
	}

	ctx := context.Background()
	ctx, cancel := context.WithTimeout(ctx, cmd.Timeout)
	defer cancel()

	in := &svc.FetchJobLogsInput{
		Name: args[0],
		Tail: cmd.Tail,
	}

	kube := svc.NewKube(deps)
	out, err := kube.FetchJobLogs(ctx, in)
	if err != nil {
		return renderServiceError(err, "failed to fetch job logs")
	}

	lines := string(bytes.TrimSpace(out.Data))
	if len(lines) < 1 {
		cmd.out.Info("-- no visible logs returned --")
		return nil
	}

	cmd.out.Output(string(out.Data))
	if int64(len(out.Data)) == kubevisor.MaxLogBytes {
		cmd.out.Info("-- logs are trimmed after this point --")
	}

	return nil
}

// Description returns long-form help text
func (cmd *JobLogs) Description() string { return cmd.Synopsis() }

// Synopsis returns a one-line
func (cmd *JobLogs) Synopsis() string { return "Return logs for a running job" }

// Usage shows usage
func (cmd *JobLogs) Usage() string { return "nerd job logs [NAME]" }
<|endoftext|>


<commit_before>// Package cmds provides commands to a bot
package cmds

import (
    "fmt"
    "os"
    "time"

    "encoding/json"
    "net/http"

    "github.com/weirdtales/senor-rosado/slack"
)

// partial struct for a google map api hit - so gross
type weatherLocation struct {
    Results []struct {
        FormattedAddress string `json:"formatted_address"`
        Geometry struct {
            Location struct {
                Lat float64 `json:"lat"`
                Lng float64 `json:"lng"`
            } `json:"location"`
        } `json:"geometry"`
        Status string `json:"status"`
    } `json:"results"`
}

// partial struct for a darksky api hit - so pretty
type weatherInfo struct {
    Latitude float64 `json:"latitude"`
    Longitude float64 `json:"longitude"`
    Timezone string `json:"timezone"`
    Currently struct {
        Time int64 `json:"time"`
        Summary string `json:"summary"`
        Icon string `json:"icon"`
        Temperature float64 `json:"temperature"`
        ApparentTemperature float64 `json:"apparentTemperature"`
        DewPoint float64 `json:"dewPoint"`
        Humidity float64 `json:"humidity"`
        WindSpeed float64 `json:"windSpeed"`
        Visibility float64 `json:"visibility"`
        CloudCover float64 `json:"cloudCover"`
        Pressure float64 `json:"pressure"`
        Ozone float64 `json:"ozone"`
    } `json:"currently"`
    Daily struct {
        Summary string `json:"summary"`
    } `json:"daily"`
    Hourly struct {
        Summary string `json:"summary"`
    } `json:"hourly"`

    GoogleName string // use the google maps FormattedAddress value - it's good
}

var httpClient = &http.Client{Timeout: 10 * time.Second}

// Weather reports the current forecast for a location
// The location is queried against the Google maps API
// The location's long and lat are sent to Darksky to get the weather
func Weather(m slack.Message, r *slack.Reply) error {
    token := os.Getenv("DARKSKY_API")
    if token == "" {
        r.Text = "I got no DARKSKY_API token in my env. Sort it."
        return nil
    }

    if m.Subcommand == "" {
        r.Text = "Specify a location, fool, then I can help."
        return nil
    }

    // location query. not sure what to do about m.Tail...
    q := m.Subcommand
    loc, err := getLocation(q)
    if err != nil {
        return err
    }

    // weather query
    w, err := getWeather(token, loc)
    if err != nil {
        return err
    }

    if w.Timezone == "" {
        r.Text = "hmm, something went wrong... (no data)"
        return nil
    }

    report := "*%s* :point_right: %s %s, %.1f°C. %s %s"
    t := time.Unix(w.Currently.Time, 0).Format("15:04")
    r.Text = fmt.Sprintf(report, w.GoogleName, t, w.Currently.Summary, w.Currently.Temperature, w.Hourly.Summary, w.Daily.Summary)

    return nil

}

// returns a weatherLocation struct from google maps api
func getLocation(q string) (loc weatherLocation, err error) {
    url := "http://maps.googleapis.com/maps/api/geocode/json?address=%s&sensor=false"
    url = fmt.Sprintf(url, q) // TODO does `q` need to be sanitized?
    err = getJson(url, &loc)
    return
}

// returns a weatherInfo struct from darksky
func getWeather(token string, loc weatherLocation) (w weatherInfo, err error) {
    w.GoogleName = loc.Results[0].FormattedAddress
    url := "https://api.darksky.net/forecast/%s/%f,%f?units=si"
    url = fmt.Sprintf(url, token, loc.Results[0].Geometry.Location.Lat, loc.Results[0].Geometry.Location.Lng)
    err = getJson(url, &w)
    return
}

// pointless scaffolding
func getJson(url string, target interface{}) error {
    r, err := httpClient.Get(url)
    if err != nil {
        return err
    }
    defer r.Body.Close()
    if r.StatusCode != 200 {
        return err
    }

    return json.NewDecoder(r.Body).Decode(target)
}
<commit_msg>clean up<commit_after>// Package cmds provides commands to a bot
package cmds

import (
    "fmt"
    "os"
    "time"

    "encoding/json"
    "net/http"

    "github.com/weirdtales/senor-rosado/slack"
)

// partial struct for a google map api hit - so gross
type weatherLocation struct {
    Results []struct {
        FormattedAddress string `json:"formatted_address"`
        Geometry struct {
            Location struct {
                Lat float64 `json:"lat"`
                Lng float64 `json:"lng"`
            } `json:"location"`
        } `json:"geometry"`
        Status string `json:"status"`
    } `json:"results"`
}

// partial struct for a darksky api hit - so pretty
type weatherInfo struct {
    Latitude float64 `json:"latitude"`
    Longitude float64 `json:"longitude"`
    Timezone string `json:"timezone"`
    Currently struct {
        Time int64 `json:"time"`
        Summary string `json:"summary"`
        Icon string `json:"icon"`
        Temperature float64 `json:"temperature"`
        ApparentTemperature float64 `json:"apparentTemperature"`
        DewPoint float64 `json:"dewPoint"`
        Humidity float64 `json:"humidity"`
        WindSpeed float64 `json:"windSpeed"`
        Visibility float64 `json:"visibility"`
        CloudCover float64 `json:"cloudCover"`
        Pressure float64 `json:"pressure"`
        Ozone float64 `json:"ozone"`
    } `json:"currently"`
    Daily struct {
        Summary string `json:"summary"`
    } `json:"daily"`
    Hourly struct {
        Summary string `json:"summary"`
    } `json:"hourly"`

    GoogleName string // use the google maps FormattedAddress value - it's good
}

var httpClient = &http.Client{Timeout: 10 * time.Second}

// Weather reports the current forecast for a location
// The location is queried against the Google maps API
// The location's long and lat are sent to Darksky to get the weather
func Weather(m slack.Message, r *slack.Reply) error {
    token := os.Getenv("DARKSKY_API")
    if token == "" {
        r.Text = "I got no DARKSKY_API token in my env. Sort it."
        return nil
    }

    if m.Subcommand == "" {
        r.Text = "Specify a location, fool, then I can help."
        return nil
    }

    loc, err := getLocation(m.Subcommand)
    if err != nil {
        return err
    }

    w, err := getWeather(token, loc)
    if err != nil {
        return err
    }

    if w.Timezone == "" {
        r.Text = "hmm, something went wrong... (no data)"
        return nil
    }

    report := "*%s* :point_right: %s %s, %.1f°C. %s %s"
    t := time.Unix(w.Currently.Time, 0).Format("15:04")
    r.Text = fmt.Sprintf(report, w.GoogleName, t, w.Currently.Summary, w.Currently.Temperature, w.Hourly.Summary, w.Daily.Summary)

    return nil

}

// returns a weatherLocation struct from google maps api
func getLocation(q string) (loc weatherLocation, err error) {
    url := "http://maps.googleapis.com/maps/api/geocode/json?address=%s&sensor=false"
    url = fmt.Sprintf(url, q) // TODO does `q` need to be sanitized?
    err = getJson(url, &loc)
    return
}

// returns a weatherInfo struct from darksky
func getWeather(token string, loc weatherLocation) (w weatherInfo, err error) {
    w.GoogleName = loc.Results[0].FormattedAddress
    url := "https://api.darksky.net/forecast/%s/%f,%f?units=si"
    url = fmt.Sprintf(url, token, loc.Results[0].Geometry.Location.Lat, loc.Results[0].Geometry.Location.Lng)
    err = getJson(url, &w)
    return
}

// pointless scaffolding
func getJson(url string, target interface{}) error {
    r, err := httpClient.Get(url)
    if err != nil {
        return err
    }
    defer r.Body.Close()
    if r.StatusCode != 200 {
        return err
    }

    return json.NewDecoder(r.Body).Decode(target)
}
<|endoftext|>


<commit_before>package command

import (
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/djimenez/iconv-go"
)

// Menu returns food in available restaurants
func Menu() Command {
	return NewCommand("menu", func(args ...string) ([]byte, error) {
		resturant := []string{"angel", "hisa", "menza"}

		// check amount of arguments, if restaurant is missing return list
		if len(args) < 2 {
			return []byte(strings.Join(resturant, " ")), nil
		}

		out := []string{}
		switch args[1] {
		case "angel":
			out = angel()
		case "hisa":
			out = dobrahisa()
		case "menza":
			out = menza()
		default:
			out = append(out, "Don't know "+args[1]+"!")
		}
		return []byte(strings.Join(out, "\n")), nil
	})
}

func angel() []string {
	r := []string{}

	// url for Dobra Hisa
	url := "http://www.kaval-group.si/ANGEL,,ponudba/kosila"

	// load the URL
	res, err := http.Get(url)
	if err != nil {
		r = append(r, err.Error())
	}
	defer res.Body.Close()

	// convert windows-1250 HTML to utf-8 encoded HTML
	utfBody, err := iconv.NewReader(res.Body, "windows-1250", "utf-8")
	if err != nil {
		r = append(r, err.Error())
	}

	// get document
	doc, err := goquery.NewDocumentFromReader(utfBody)
	if err != nil {
		r = append(r, err.Error())
		return r
	}

	// add title
	r = append(r, "Ponudba v Piceriji Angel:")

	// get right class selector from day of week
	dow := int(time.Now().Weekday()) - 1
	cls := ".show-" + strconv.Itoa(dow)

	// get div for first day
	doc.Find(cls).Each(func(i int, d *goquery.Selection) {
		d.Find("p").Each(func(i int, p *goquery.Selection) {
			html, err := p.Html()
			if err != nil {
				r = append(r, err.Error())
			}

			// manipulate html for nicer output
			html = strings.Replace(html, "", "*", -1)
			html = strings.Replace(html, "", "*", -1)
			html = strings.Replace(html, "* *", "", -1)
			html = strings.Replace(html, "* *", "", -1)
			html = strings.Replace(html, "
", "", -1)
			html = strings.TrimSpace(html)

			// append menu to output
			if len(html) > 0 {
				r = append(r, html)
			}
		})
	})

	return r
}

func dobrahisa() []string {
	r := []string{}

	// url for Dobra Hisa
	url := "https://api.malcajt.com/getApiData.php?action=embed&id=2030&show=1001"

	// get document
	doc, err := goquery.NewDocument(url)
	if err != nil {
		r = append(r, err.Error())
		return r
	}

	// find first day
	doc.Find("a").Each(func(i int, a *goquery.Selection) {
		h, ok := a.Attr("href")
		if ok && h == "#day0" {
			r = append(r, "Ponudba v Dobri Hisi: *"+a.Text()+"*")
		}
	})

	// get div for day0
	doc.Find("#day0").Each(func(i int, d *goquery.Selection) {
		html, err := d.Html()
		if err != nil {
			r = append(r, err.Error())
		}

		// manipulate html for nicer output
		html = strings.Replace(html, "—", "", -1)
		html = strings.Replace(html, "
", "
", -1)
		html = strings.Replace(html, "", "*", -1)
		html = strings.Replace(html, "", "*", -1)

		// split by lines
		s := strings.Split(html, "
")

		// append menu to output
		r = append(r, s...)
	})

	return r
}

func menza() []string {
	r := []string{}

	// url for Menza
	url := "http://www.menza-ljubljana.si/jedilnik/"

	// get document
	doc, err := goquery.NewDocument(url)
	if err != nil {
		r = append(r, err.Error())
		return r
	}

	// add title
	r = append(r, "Ponudba v Menzi:")

	// get div for day0
	doc.Find(".jedilnik").Each(func(i int, div *goquery.Selection) {
		div.Find("td").Each(func(i int, td *goquery.Selection) {
			s := td.Find("h3").Text() + " " + td.Find("p").Text()

			// append menu to output
			r = append(r, s)
		})
	})

	return r
}
<commit_msg>Fixed menza<commit_after>package command

import (
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/djimenez/iconv-go"
)

// Menu returns food in available restaurants
func Menu() Command {
	return NewCommand("menu", func(args ...string) ([]byte, error) {
		resturant := []string{"angel", "hisa", "menza"}

		// check amount of arguments, if restaurant is missing return list
		if len(args) < 2 {
			return []byte(strings.Join(resturant, " ")), nil
		}

		out := []string{}
		switch args[1] {
		case "angel":
			out = angel()
		case "hisa":
			out = dobrahisa()
		case "menza":
			out = menza()
		default:
			out = append(out, "Don't know "+args[1]+"!")
		}
		return []byte(strings.Join(out, "\n")), nil
	})
}

func angel() []string {
	r := []string{}

	// url for Dobra Hisa
	url := "http://www.kaval-group.si/ANGEL,,ponudba/kosila"

	// load the URL
	res, err := http.Get(url)
	if err != nil {
		r = append(r, err.Error())
	}
	defer res.Body.Close()

	// convert windows-1250 HTML to utf-8 encoded HTML
	utfBody, err := iconv.NewReader(res.Body, "windows-1250", "utf-8")
	if err != nil {
		r = append(r, err.Error())
	}

	// get document
	doc, err := goquery.NewDocumentFromReader(utfBody)
	if err != nil {
		r = append(r, err.Error())
		return r
	}

	// add title
	r = append(r, "Ponudba v Piceriji Angel:")

	// get right class selector from day of week
	dow := int(time.Now().Weekday()) - 1
	cls := ".show-" + strconv.Itoa(dow)

	// get div for first day
	doc.Find(cls).Each(func(i int, d *goquery.Selection) {
		d.Find("p").Each(func(i int, p *goquery.Selection) {
			html, err := p.Html()
			if err != nil {
				r = append(r, err.Error())
			}

			// manipulate html for nicer output
			html = strings.Replace(html, "", "*", -1)
			html = strings.Replace(html, "", "*", -1)
			html = strings.Replace(html, "* *", "", -1)
			html = strings.Replace(html, "* *", "", -1)
			html = strings.Replace(html, "
", "", -1)
			html = strings.TrimSpace(html)

			// append menu to output
			if len(html) > 0 {
				r = append(r, html)
			}
		})
	})

	return r
}

func dobrahisa() []string {
	r := []string{}

	// url for Dobra Hisa
	url := "https://api.malcajt.com/getApiData.php?action=embed&id=2030&show=1001"

	// get document
	doc, err := goquery.NewDocument(url)
	if err != nil {
		r = append(r, err.Error())
		return r
	}

	// find first day
	doc.Find("a").Each(func(i int, a *goquery.Selection) {
		h, ok := a.Attr("href")
		if ok && h == "#day0" {
			r = append(r, "Ponudba v Dobri Hisi: *"+a.Text()+"*")
		}
	})

	// get div for day0
	doc.Find("#day0").Each(func(i int, d *goquery.Selection) {
		html, err := d.Html()
		if err != nil {
			r = append(r, err.Error())
		}

		// manipulate html for nicer output
		html = strings.Replace(html, "—", "", -1)
		html = strings.Replace(html, "
", "
", -1)
		html = strings.Replace(html, "", "*", -1)
		html = strings.Replace(html, "", "*", -1)

		// split by lines
		s := strings.Split(html, "
")

		// append menu to output
		r = append(r, s...)
	})

	return r
}

func menza() []string {
	r := []string{}

	// url for Menza
	url := "https://www.studentska-prehrana.si/sl/restaurant/Details/2710"

	// get document
	doc, err := goquery.NewDocument(url)
	if err != nil {
		r = append(r, err.Error())
		return r
	}

	// add title
	r = append(r, "Ponudba v Menzi:")

	doc.Find("#menu-list").Find(".shadow-wrapper").Each(func(i int, div *goquery.Selection) {
		s := div.Find("h5").Find("strong").Text()

		s = strings.ToLower(s)
		split := strings.Split(s, " ")
		// capitalize first word. for some reason first split element is empty string
		split[1] = strings.Title(split[1])
		s = strings.Join(split, " ")
		r = append(r, s)
	})

	r = append(r, "© 2016 Študentska organizacija Slovenije – Vse pravice pridržane")

	return r
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os/exec"
	"strconv"
	"strings"
	"testing"
	"time"
)

func TestCLIGetEventsUntag(t *testing.T) {
	out, _, _ := cmd(t, "images", "-q")
	image := strings.Split(out, "\n")[0]
	cmd(t, "tag", image, "utest:tag1")
	cmd(t, "tag", image, "utest:tag2")
	cmd(t, "rmi", "utest:tag1")
	cmd(t, "rmi", "utest:tag2")
	eventsCmd := exec.Command("timeout", "0.2", dockerBinary, "events", "--since=1")
	out, _, _ = runCommandWithOutput(eventsCmd)
	events := strings.Split(out, "\n")
	n_events := len(events)
	// The last element after the split above will be an empty string, so we
	// get the two elements before the last, which are the untags we're
	// looking for.
	for _, v := range events[n_events-3 : n_events-1] {
		if !strings.Contains(v, "untag") {
			t.Fatalf("event should be untag, not %#v", v)
		}
	}
	logDone("events - untags are logged")
}

func TestCLIGetEventsPause(t *testing.T) {
	out, _, _ := cmd(t, "images", "-q")
	image := strings.Split(out, "\n")[0]
	cmd(t, "run", "-d", "--name", "testeventpause", image, "sleep", "2")
	cmd(t, "pause", "testeventpause")
	cmd(t, "unpause", "testeventpause")
	eventsCmd := exec.Command(dockerBinary, "events", "--since=0", fmt.Sprintf("--until=%d", time.Now().Unix()))
	out, _, _ = runCommandWithOutput(eventsCmd)
	events := strings.Split(out, "\n")
	if len(events) <= 1 {
		t.Fatalf("Missing expected event")
	}

	pauseEvent := strings.Fields(events[len(events)-3])
	unpauseEvent := strings.Fields(events[len(events)-2])

	if pauseEvent[len(pauseEvent)-1] != "pause" {
		t.Fatalf("event should be pause, not %#v", pauseEvent)
	}
	if unpauseEvent[len(unpauseEvent)-1] != "unpause" {
		t.Fatalf("event should be pause, not %#v", unpauseEvent)
	}

	logDone("events - pause/unpause is logged")
}

func TestCLILimitEvents(t *testing.T) {
	for i := 0; i < 30; i++ {
		cmd(t, "run", "busybox", "echo", strconv.Itoa(i))
	}
	eventsCmd := exec.Command(dockerBinary, "events", "--since=0", fmt.Sprintf("--until=%d", time.Now().Unix()))
	out, _, _ := runCommandWithOutput(eventsCmd)
	events := strings.Split(out, "\n")
	n_events := len(events) - 1
	if n_events != 64 {
		t.Fatalf("events should be limited to 64, but received %d", n_events)
	}
	logDone("events - limited to 64 entries")
}

func TestCLIGetEventsContainerEvents(t *testing.T) {
	cmd(t, "run", "--rm", "busybox", "true")
	eventsCmd := exec.Command(dockerBinary, "events", "--since=0", fmt.Sprintf("--until=%d", time.Now().Unix()))
	out, exitCode, err := runCommandWithOutput(eventsCmd)
	if exitCode != 0 || err != nil {
		t.Fatal("Failed to get events with exit code %d: %s", exitCode, err)
	}
	events := strings.Split(out, "\n")
	events = events[:len(events)-1]
	if len(events) < 4 {
		t.Fatalf("Missing expected event")
	}
	createEvent := strings.Fields(events[len(events)-4])
	startEvent := strings.Fields(events[len(events)-3])
	dieEvent := strings.Fields(events[len(events)-2])
	destroyEvent := strings.Fields(events[len(events)-1])
	if createEvent[len(createEvent)-1] != "create" {
		t.Fatalf("event should be create, not %#v", createEvent)
	}
	if startEvent[len(startEvent)-1] != "start" {
		t.Fatalf("event should be pause, not %#v", startEvent)
	}
	if dieEvent[len(dieEvent)-1] != "die" {
		t.Fatalf("event should be pause, not %#v", dieEvent)
	}
	if destroyEvent[len(destroyEvent)-1] != "destroy" {
		t.Fatalf("event should be pause, not %#v", destroyEvent)
	}

	logDone("events - container create, start, die, destroy is logged")
}
<commit_msg>Test on untag and delete events<commit_after>package main

import (
	"fmt"
	"os/exec"
	"strconv"
	"strings"
	"testing"
	"time"
)

func TestCLIGetEventsUntag(t *testing.T) {
	out, _, _ := cmd(t, "images", "-q")
	image := strings.Split(out, "\n")[0]
	cmd(t, "tag", image, "utest:tag1")
	cmd(t, "tag", image, "utest:tag2")
	cmd(t, "rmi", "utest:tag1")
	cmd(t, "rmi", "utest:tag2")
	eventsCmd := exec.Command("timeout", "0.2", dockerBinary, "events", "--since=1")
	out, _, _ = runCommandWithOutput(eventsCmd)
	events := strings.Split(out, "\n")
	n_events := len(events)
	// The last element after the split above will be an empty string, so we
	// get the two elements before the last, which are the untags we're
	// looking for.
	for _, v := range events[n_events-3 : n_events-1] {
		if !strings.Contains(v, "untag") {
			t.Fatalf("event should be untag, not %#v", v)
		}
	}
	logDone("events - untags are logged")
}

func TestCLIGetEventsPause(t *testing.T) {
	out, _, _ := cmd(t, "images", "-q")
	image := strings.Split(out, "\n")[0]
	cmd(t, "run", "-d", "--name", "testeventpause", image, "sleep", "2")
	cmd(t, "pause", "testeventpause")
	cmd(t, "unpause", "testeventpause")
	eventsCmd := exec.Command(dockerBinary, "events", "--since=0", fmt.Sprintf("--until=%d", time.Now().Unix()))
	out, _, _ = runCommandWithOutput(eventsCmd)
	events := strings.Split(out, "\n")
	if len(events) <= 1 {
		t.Fatalf("Missing expected event")
	}

	pauseEvent := strings.Fields(events[len(events)-3])
	unpauseEvent := strings.Fields(events[len(events)-2])

	if pauseEvent[len(pauseEvent)-1] != "pause" {
		t.Fatalf("event should be pause, not %#v", pauseEvent)
	}
	if unpauseEvent[len(unpauseEvent)-1] != "unpause" {
		t.Fatalf("event should be pause, not %#v", unpauseEvent)
	}

	logDone("events - pause/unpause is logged")
}

func TestCLILimitEvents(t *testing.T) {
	for i := 0; i < 30; i++ {
		cmd(t, "run", "busybox", "echo", strconv.Itoa(i))
	}
	eventsCmd := exec.Command(dockerBinary, "events", "--since=0", fmt.Sprintf("--until=%d", time.Now().Unix()))
	out, _, _ := runCommandWithOutput(eventsCmd)
	events := strings.Split(out, "\n")
	n_events := len(events) - 1
	if n_events != 64 {
		t.Fatalf("events should be limited to 64, but received %d", n_events)
	}
	logDone("events - limited to 64 entries")
}

func TestCLIGetEventsContainerEvents(t *testing.T) {
	cmd(t, "run", "--rm", "busybox", "true")
	eventsCmd := exec.Command(dockerBinary, "events", "--since=0", fmt.Sprintf("--until=%d", time.Now().Unix()))
	out, exitCode, err := runCommandWithOutput(eventsCmd)
	if exitCode != 0 || err != nil {
		t.Fatal("Failed to get events with exit code %d: %s", exitCode, err)
	}
	events := strings.Split(out, "\n")
	events = events[:len(events)-1]
	if len(events) < 4 {
		t.Fatalf("Missing expected event")
	}
	createEvent := strings.Fields(events[len(events)-4])
	startEvent := strings.Fields(events[len(events)-3])
	dieEvent := strings.Fields(events[len(events)-2])
	destroyEvent := strings.Fields(events[len(events)-1])
	if createEvent[len(createEvent)-1] != "create" {
		t.Fatalf("event should be create, not %#v", createEvent)
	}
	if startEvent[len(startEvent)-1] != "start" {
		t.Fatalf("event should be pause, not %#v", startEvent)
	}
	if dieEvent[len(dieEvent)-1] != "die" {
		t.Fatalf("event should be pause, not %#v", dieEvent)
	}
	if destroyEvent[len(destroyEvent)-1] != "destroy" {
		t.Fatalf("event should be pause, not %#v", destroyEvent)
	}

	logDone("events - container create, start, die, destroy is logged")
}

func TestCLIGetEventsImageUntagDelete(t *testing.T) {
	name := "testimageevents"
	defer deleteImages(name)
	_, err := buildImage(name,
		`FROM scratch
		MAINTAINER "docker"`,
		true)
	if err != nil {
		t.Fatal(err)
	}
	if err := deleteImages(name); err != nil {
		t.Fatal(err)
	}
	eventsCmd := exec.Command(dockerBinary, "events", "--since=0", fmt.Sprintf("--until=%d", time.Now().Unix()))
	out, exitCode, err := runCommandWithOutput(eventsCmd)
	if exitCode != 0 || err != nil {
		t.Fatal("Failed to get events with exit code %d: %s", exitCode, err)
	}
	events := strings.Split(out, "\n")
	t.Log(events)
	events = events[:len(events)-1]
	if len(events) < 2 {
		t.Fatalf("Missing expected event")
	}
	untagEvent := strings.Fields(events[len(events)-2])
	deleteEvent := strings.Fields(events[len(events)-1])
	if untagEvent[len(untagEvent)-1] != "untag" {
		t.Fatalf("untag should be untag, not %#v", untagEvent)
	}
	if deleteEvent[len(deleteEvent)-1] != "delete" {
		t.Fatalf("delete should be delete, not %#v", deleteEvent)
	}
	logDone("events - image untag, delete is logged")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os/exec"
	"strings"
	"testing"
)

// search for repos named  "registry" on the central registry
func TestSearchOnCentralRegistry(t *testing.T) {
	searchCmd := exec.Command(dockerBinary)
	out, exitCode, err := runCommandWithOutput(searchCmd)
	errorOut(err, t, fmt.Sprintf("encountered error while searching: %v", err))

	if err != nil || exitCode != 0 {
		t.Fatal("failed to search on the central registry")
	}

	if !strings.Contains(out, "registry") {
		t.Fatal("couldn't find any repository named (or containing) 'registry'")
	}

	logDone("search - search for repositories named (or containing) 'registry'")
}
<commit_msg>Fix search integration-cli test<commit_after>package main

import (
	"fmt"
	"os/exec"
	"strings"
	"testing"
)

// search for repos named  "registry" on the central registry
func TestSearchOnCentralRegistry(t *testing.T) {
	searchCmd := exec.Command(dockerBinary, "search", "stackbrew/busybox")
	out, exitCode, err := runCommandWithOutput(searchCmd)
	errorOut(err, t, fmt.Sprintf("encountered error while searching: %v", err))

	if err != nil || exitCode != 0 {
		t.Fatal("failed to search on the central registry")
	}

	if !strings.Contains(out, "Busybox base image.") {
		t.Fatal("couldn't find any repository named (or containing) 'Busybox base image.'")
	}

	logDone("search - search for repositories named (or containing) 'Busybox base image.'")
}
<|endoftext|>


<commit_before>package utility

import (
    "fmt"
    "github.com/mitchellh/go-homedir"
    "gopkg.in/ini.v1"
    "log"
    "os"
    "strings"
)

type Util struct {
    ConfigPath     string
    CustomAppNames string
    Apps           map[string]*[]string
}

func NewUtil() *Util {
    home, err := homedir.Dir()
    if err != nil {
        log.Fatalln(err)
    }
    util := &Util{
        ConfigPath: fmt.Sprintf(
            "%s%s.twhelp.ini",
            home,
            string(os.PathSeparator),
        ),
        Apps: make(map[string]*[]string, 8),
    }
    util.Apps["android"] = &[]string{
        "3nVuSoBZnx6U4vzUxf5w",
        "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys",
    }
    util.Apps["win"] = &[]string{
        "TgHNMa7WZE7Cxi1JbkAMQ",
        "SHy9mBMBPNj3Y17et9BF4g5XeqS4y3vkeW24PttDcY",
    }
    util.Apps["wp"] = &[]string{
        "yN3DUNVO0Me63IAQdhTfCA",
        "c768oTKdzAjIYCmpSNIdZbGaG0t6rOhSFQP0S5uC79g",
    }
    util.Apps["google"] = &[]string{
        "iAtYJ4HpUVfIUoNnif1DA",
        "172fOpzuZoYzNYaU3mMYvE8m8MEyLbztOdbrUolU",
    }
    util.Apps["iphone"] = &[]string{
        "IQKbtAYlXLripLGPWd0HUA",
        "GgDYlkSvaPxGxC4X8liwpUoqKwwr3lCADbz8A7ADU",
    }
    util.Apps["ipad"] = &[]string{
        "CjulERsDeqhhjSme66ECg",
        "IQWdVyqFxghAtURHGeGiWAsmCAGmdW3WmbEx6Hck",
    }
    util.Apps["mac"] = &[]string{
        "3rJOl1ODzm9yZy63FACdg",
        "5jPoQ5kQvMJFDYRNE8bQ4rHuds4xJqhvgNJM4awaE8",
    }
    util.Apps["deck"] = &[]string{
        "yT577ApRtZw51q4NPMPPOQ",
        "3neq3XqN5fO3obqwZoajavGFCUrC42ZfbrLXy5sCv8",
    }
    if _, err := os.Stat(util.ConfigPath); err == nil {
        cfg, err := ini.Load(util.ConfigPath)
        if err != nil {
            log.Fatalln(err)
        }
        customAppNames := make([]string, 0)
        for _, name := range cfg.SectionStrings() {
            if name == "DEFAULT" {
                continue
            }
            section := cfg.Section(name)
            ck, ckerr := section.GetKey("consumer_key")
            if ckerr != nil {
                log.Fatalln(fmt.Sprintf(`"consumer_key" for %s does not exist`, name))
            }
            cs, cserr := section.GetKey("consumer_secret")
            if cserr != nil {
                log.Fatalln(fmt.Sprintf(`"consumer_secret" for %s does not exist`, name))
            }
            util.Apps[name] = &[]string{ck.String(), cs.String()}
            customAppNames = append(customAppNames, name)
        }
        if len(customAppNames) > 0 {
            util.CustomAppNames = fmt.Sprintf(
                "Config File: %s\nCustom Apps: %s\n",
                util.ConfigPath,
                strings.Join(customAppNames, ", "),
            )
        } else {
            util.CustomAppNames = fmt.Sprintf(
                `
Your own applications also can be defined in %s
Example:

[my_app_01]
consumer_key    = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
consumer_secret = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
`, util.ConfigPath)
        }
    }
    return util
}

func (util *Util) Usage() {
    os.Stderr.WriteString(fmt.Sprintf(`Usage: %s [options]
Options:
  -h, --help          Show help.

[ Output Format ]

  Default             Output line by line.
  -i, --ini           Output as INI.
  -y, --yaml          Output as YAML.
  -a, --array         Output as array that compatible with most languages.
  -A, --assoc         Output as PHP-style associative array.
  -j, --json          Output as JSON.

[ OAuth Process ]

  Default             xAuth manipulation with OAuth scraping.
  -x, --xauth         Pure xAuth. Only available with official keys.
  -o, --oauth         Pure OAuth. You have to authorize via web browser.

[ OAuth Credentials ]

  Insufficient components are required to input via STDIN.
  Password is masked.

  --ck  <value>       Specify consumer_key in advance.
  --cs  <value>       Specify consumer_secret in advance.
  --sn  <value>       Specify screen_name or email in advance.
  --pw  <value>       Specify password in advance. (Not masked, DEPRECATED)
  --app <value>       Speficy consumer_key and consumer_secret with app name.

                      app name | full name
                      ------------------------------------
                      android  | Twitter for Andriod
                      win      | Twitter for Andriod
                      wp       | Twitter for Windows Phone
                      google   | Twitter for Google TV
                      iphone   | Twitter for iPhone
                      ipad     | Twitter for iPad
                      mac      | Twitter for Mac
                      deck     | TweetDeck

%s
`, os.Args[0], util.CustomAppNames))
}
<commit_msg>Always show ~/.twhelp.ini info<commit_after>package utility

import (
    "fmt"
    "github.com/mitchellh/go-homedir"
    "gopkg.in/ini.v1"
    "log"
    "os"
    "strings"
)

type Util struct {
    ConfigPath     string
    CustomAppNames string
    Apps           map[string]*[]string
}

func NewUtil() *Util {
    home, err := homedir.Dir()
    if err != nil {
        log.Fatalln(err)
    }
    util := &Util{
        ConfigPath: fmt.Sprintf(
            "%s%s.twhelp.ini",
            home,
            string(os.PathSeparator),
        ),
        Apps: make(map[string]*[]string, 8),
    }
    util.Apps["android"] = &[]string{
        "3nVuSoBZnx6U4vzUxf5w",
        "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys",
    }
    util.Apps["win"] = &[]string{
        "TgHNMa7WZE7Cxi1JbkAMQ",
        "SHy9mBMBPNj3Y17et9BF4g5XeqS4y3vkeW24PttDcY",
    }
    util.Apps["wp"] = &[]string{
        "yN3DUNVO0Me63IAQdhTfCA",
        "c768oTKdzAjIYCmpSNIdZbGaG0t6rOhSFQP0S5uC79g",
    }
    util.Apps["google"] = &[]string{
        "iAtYJ4HpUVfIUoNnif1DA",
        "172fOpzuZoYzNYaU3mMYvE8m8MEyLbztOdbrUolU",
    }
    util.Apps["iphone"] = &[]string{
        "IQKbtAYlXLripLGPWd0HUA",
        "GgDYlkSvaPxGxC4X8liwpUoqKwwr3lCADbz8A7ADU",
    }
    util.Apps["ipad"] = &[]string{
        "CjulERsDeqhhjSme66ECg",
        "IQWdVyqFxghAtURHGeGiWAsmCAGmdW3WmbEx6Hck",
    }
    util.Apps["mac"] = &[]string{
        "3rJOl1ODzm9yZy63FACdg",
        "5jPoQ5kQvMJFDYRNE8bQ4rHuds4xJqhvgNJM4awaE8",
    }
    util.Apps["deck"] = &[]string{
        "yT577ApRtZw51q4NPMPPOQ",
        "3neq3XqN5fO3obqwZoajavGFCUrC42ZfbrLXy5sCv8",
    }
    customAppNames := make([]string, 0)
    if _, err := os.Stat(util.ConfigPath); err == nil {
        cfg, err := ini.Load(util.ConfigPath)
        if err != nil {
            log.Fatalln(err)
        }
        for _, name := range cfg.SectionStrings() {
            if name == "DEFAULT" {
                continue
            }
            section := cfg.Section(name)
            ck, ckerr := section.GetKey("consumer_key")
            if ckerr != nil {
                log.Fatalln(fmt.Sprintf(`"consumer_key" for %s does not exist`, name))
            }
            cs, cserr := section.GetKey("consumer_secret")
            if cserr != nil {
                log.Fatalln(fmt.Sprintf(`"consumer_secret" for %s does not exist`, name))
            }
            util.Apps[name] = &[]string{ck.String(), cs.String()}
            customAppNames = append(customAppNames, name)
        }
    }
    if len(customAppNames) > 0 {
        util.CustomAppNames = fmt.Sprintf(
            "Config File: %s\nCustom Apps: %s\n",
            util.ConfigPath,
            strings.Join(customAppNames, ", "),
        )
    } else {
        util.CustomAppNames = fmt.Sprintf(
            `Your own applications also can be defined in %s
Example:

[my_app_01]
consumer_key    = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
consumer_secret = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
`, util.ConfigPath)
    }
    return util
}

func (util *Util) Usage() {
    os.Stderr.WriteString(fmt.Sprintf(`Usage: %s [options]
Options:
  -h, --help          Show help.

[ Output Format ]

  Default             Output line by line.
  -i, --ini           Output as INI.
  -y, --yaml          Output as YAML.
  -a, --array         Output as array that compatible with most languages.
  -A, --assoc         Output as PHP-style associative array.
  -j, --json          Output as JSON.

[ OAuth Process ]

  Default             xAuth manipulation with OAuth scraping.
  -x, --xauth         Pure xAuth. Only available with official keys.
  -o, --oauth         Pure OAuth. You have to authorize via web browser.

[ OAuth Credentials ]

  Insufficient components are required to input via STDIN.
  Password is masked.

  --ck  <value>       Specify consumer_key in advance.
  --cs  <value>       Specify consumer_secret in advance.
  --sn  <value>       Specify screen_name or email in advance.
  --pw  <value>       Specify password in advance. (Not masked, DEPRECATED)
  --app <value>       Speficy consumer_key and consumer_secret with app name.

                      app name | full name
                      ------------------------------------
                      android  | Twitter for Andriod
                      win      | Twitter for Andriod
                      wp       | Twitter for Windows Phone
                      google   | Twitter for Google TV
                      iphone   | Twitter for iPhone
                      ipad     | Twitter for iPad
                      mac      | Twitter for Mac
                      deck     | TweetDeck

%s
`, os.Args[0], util.CustomAppNames))
}
<|endoftext|>