<commit_before>package pools

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"

	"github.com/kumoru/kumoru-sdk-go/kumoru"
)

type Location struct {
	CreatedAt  string `json:"created_at"`
	Identifier string `json:"location"`
	Provider   string `json:"provider"`
	PoolId     string `json:"stack_id"`
	Status     string `json:"status"`
	UpdatedAt  string `json:"updated_at"`
	Url        string `json:"url"`
	Uuid       string `json:"uuid"`
	Version    string `json:"version"`
}

// Create is a method on a Location that will create Kumoru resources in the provider region
func (l *Location) Create() (*Location, *http.Response, []error) {
	k := kumoru.New()

	k.Post(fmt.Sprintf("%v/v1/pools/", k.EndPoint.Pool))
	k.Send(fmt.Sprintf("location=%s", url.QueryEscape(l.Identifier)))
	k.SignRequest(true)

	resp, body, errs := k.End()

	if resp.StatusCode >= 400 {
		errs = append(errs, fmt.Errorf("%s", resp.Status))
		return l, resp, errs
	}

	err := json.Unmarshal([]byte(body), &l)

	if err != nil {
		errs = append(errs, err)
		return l, resp, errs
	}

	return l, resp, nil
}

//Delete is a method on a Location that will remove Kumoru resources from the provider region
func (l *Location) Delete(uuid string) (*Location, *http.Response, []error) {
	k := kumoru.New()

	k.Delete(fmt.Sprintf("%v/v1/pools/%s", k.EndPoint.Pool, uuid))
	k.SignRequest(true)

	resp, _, errs := k.End()

	if errs != nil {
		return l, resp, errs
	}

	return l, resp, nil
}

//List is a method on a Location that will list all Locations an user has access to
func (l *Location) List() (*[]Location, *http.Response, []error) {
	locations := []Location{}
	k := kumoru.New()

	k.Get(fmt.Sprintf("%v/v1/pools/", k.EndPoint.Pool))
	k.SignRequest(true)

	resp, body, errs := k.End()

	if resp.StatusCode >= 400 {
		errs = append(errs, fmt.Errorf("%s", resp.Status))
		return &locations, resp, errs
	}

	err := json.Unmarshal([]byte(body), &locations)

	if err != nil {
		errs = append(errs, err)
		return &locations, resp, errs
	}

	return &locations, resp, nil
}

//Show is a method on a Location that will show all the details of a particular Location
func (l *Location) Show() (*Location, *http.Response, []error) {
	k := kumoru.New()

	k.Get(fmt.Sprintf("%v/v1/pools/%s", k.EndPoint.Pool, l.Uuid))
	k.SignRequest(true)

	resp, body, errs := k.End()

	if resp.StatusCode >= 400 {
		errs = append(errs, fmt.Errorf("%s", resp.Status))
		return l, resp, errs
	}

	err := json.Unmarshal([]byte(body), l)

	if err != nil {
		errs = append(errs, err)
		return l, resp, errs
	}

	return l, resp, errs

}
<commit_msg>Add AggregateResources key to location struct<commit_after>package pools

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"

	"github.com/kumoru/kumoru-sdk-go/kumoru"
)

type Location struct {
	AggregateResources map[string]float32 `json:"aggregate_resources"`
	CreatedAt          string             `json:"created_at"`
	Identifier         string             `json:"location"`
	Provider           string             `json:"provider"`
	PoolId             string             `json:"stack_id"`
	Status             string             `json:"status"`
	UpdatedAt          string             `json:"updated_at"`
	Url                string             `json:"url"`
	Uuid               string             `json:"uuid"`
	ApiVersion         string             `json:"api_version"`
}

// Create is a method on a Location that will create Kumoru resources in the provider region
func (l *Location) Create() (*Location, *http.Response, []error) {
	k := kumoru.New()

	k.Post(fmt.Sprintf("%v/v1/pools/", k.EndPoint.Pool))
	k.Send(fmt.Sprintf("location=%s", url.QueryEscape(l.Identifier)))
	k.SignRequest(true)

	resp, body, errs := k.End()

	if resp.StatusCode >= 400 {
		errs = append(errs, fmt.Errorf("%s", resp.Status))
		return l, resp, errs
	}

	err := json.Unmarshal([]byte(body), &l)

	if err != nil {
		errs = append(errs, err)
		return l, resp, errs
	}

	return l, resp, nil
}

//Delete is a method on a Location that will remove Kumoru resources from the provider region
func (l *Location) Delete(uuid string) (*Location, *http.Response, []error) {
	k := kumoru.New()

	k.Delete(fmt.Sprintf("%v/v1/pools/%s", k.EndPoint.Pool, uuid))
	k.SignRequest(true)

	resp, _, errs := k.End()

	if errs != nil {
		return l, resp, errs
	}

	return l, resp, nil
}

//List is a method on a Location that will list all Locations an user has access to
func (l *Location) List() (*[]Location, *http.Response, []error) {
	locations := []Location{}
	k := kumoru.New()

	k.Get(fmt.Sprintf("%v/v1/pools/", k.EndPoint.Pool))
	k.SignRequest(true)

	resp, body, errs := k.End()

	if resp.StatusCode >= 400 {
		errs = append(errs, fmt.Errorf("%s", resp.Status))
		return &locations, resp, errs
	}

	err := json.Unmarshal([]byte(body), &locations)

	if err != nil {
		errs = append(errs, err)
		return &locations, resp, errs
	}

	return &locations, resp, nil
}

//Show is a method on a Location that will show all the details of a particular Location
func (l *Location) Show() (*Location, *http.Response, []error) {
	k := kumoru.New()

	k.Get(fmt.Sprintf("%v/v1/pools/%s", k.EndPoint.Pool, l.Uuid))
	k.SignRequest(true)

	resp, body, errs := k.End()

	if resp.StatusCode >= 400 {
		errs = append(errs, fmt.Errorf("%s", resp.Status))
		return l, resp, errs
	}

	err := json.Unmarshal([]byte(body), l)

	if err != nil {
		errs = append(errs, err)
		return l, resp, errs
	}

	return l, resp, errs

}
<|endoftext|>


<commit_before>package cleaner

import (
	"path/filepath"
	"time"

	"github.com/bmatcuk/doublestar"

	"github.com/qiniu/log"

	"github.com/qiniu/logkit/conf"
	"github.com/qiniu/logkit/reader"
	"github.com/qiniu/logkit/reader/config"
	. "github.com/qiniu/logkit/utils/models"
)

type Cleaner struct {
	cleanTicker   <-chan time.Time
	reserveNumber int64 //个
	reserveSize   int64 //byte
	meta          *reader.Meta
	exitChan      chan struct{}
	cleanChan     chan<- CleanSignal
	name          string
	logdir        string
}

type CleanSignal struct {
	Logdir   string
	Filename string
	Cleaner  string
	ReadMode string
}

const (
	KeyCleanEnable       = "delete_enable"
	KeyCleanInterval     = "delete_interval"
	KeyReserveFileNumber = "reserve_file_number"
	KeyReserveFileSize   = "reserve_file_size"
	cleanerName          = "cleaner_name"

	defaultDeleteInterval    = 300  //5分钟
	defaultReserveFileNumber = 10   //默认保存是个文件
	defaultReserveFileSize   = 2048 //单位MB，默认删除保存2G
	// 如果两项任意一项达到要求，就执行删除；如果两项容易一项有值设置，但是另一项为0，就认为另一项不做限制
)

// 删除文件时遍历全部
// 删除时生成filedeleted文件
func NewCleaner(conf conf.MapConf, meta *reader.Meta, cleanChan chan<- CleanSignal, logdir string) (*Cleaner, error) {
	enable, _ := conf.GetBoolOr(KeyCleanEnable, false)
	if !enable {
		return nil, nil
	}
	mode := meta.GetMode()
	if mode != config.ModeDir &&
		mode != config.ModeFile &&
		mode != config.ModeCloudTrail &&
		mode != config.ModeCloudTrailV2 &&
		mode != config.ModeTailx &&
		mode != config.ModeDirx {
		log.Errorf("Cleaner only supports reader mode dir|file|cloudtrail|tailx|dirx, current mode is %v, cleaner disabled", meta.GetMode())
		return nil, nil
	}
	interval, _ := conf.GetIntOr(KeyCleanInterval, 0) //单位，秒
	if interval <= 0 {
		interval = defaultDeleteInterval
	}
	name, _ := conf.GetStringOr(cleanerName, "unknown")
	reserveNumber, _ := conf.GetInt64Or(KeyReserveFileNumber, 0)
	reserveSize, _ := conf.GetInt64Or(KeyReserveFileSize, 0)
	if reserveNumber <= 0 && reserveSize <= 0 {
		reserveNumber = defaultReserveFileNumber
		reserveSize = defaultReserveFileSize
	}
	reserveSize = reserveSize * MB
	if mode != config.ModeTailx && mode != config.ModeDirx {
		var err error
		logdir, _, err = GetRealPath(logdir)
		if err != nil {
			log.Errorf("Failed to get real path of %q: %v", logdir, err)
			return nil, err
		}
	}
	return &Cleaner{
		cleanTicker:   time.NewTicker(time.Duration(interval) * time.Second).C,
		reserveNumber: reserveNumber,
		reserveSize:   reserveSize,
		meta:          meta,
		exitChan:      make(chan struct{}),
		cleanChan:     cleanChan,
		name:          name,
		logdir:        logdir,
	}, nil
}

func (c *Cleaner) Run() {
	for {
		select {
		case <-c.exitChan:
			log.Warnf("%v receive exit signal, cleaner exiting...", c.name)
			return
		case <-c.cleanTicker:
		}
		err := c.Clean()
		if err != nil {
			log.Error(err)
		}
	}
}

func (c *Cleaner) Close() {
	c.exitChan <- struct{}{}
}

func (c *Cleaner) Name() string {
	return c.name
}

func (c *Cleaner) shouldClean(size, count int64) bool {
	if c.reserveNumber > 0 && count > c.reserveNumber {
		return true
	}
	if c.reserveSize > 0 && size > c.reserveSize {
		return true
	}
	return false
}

func (c *Cleaner) checkBelong(path string) bool {
	dir := filepath.Dir(path)
	dir, _, err := GetRealPath(dir)
	if err != nil {
		log.Errorf("GetRealPath for %v error %v", path, err)
		return false
	}

	switch c.meta.GetMode() {
	case config.ModeTailx:
		matched, err := filepath.Match(filepath.Dir(c.logdir), filepath.Dir(path))
		if err != nil {
			log.Errorf("Failed to check if %q belongs to %q: %v", path, c.logdir, err)
			return false
		}
		return matched

	case config.ModeDirx:
		matched, err := doublestar.Match(c.logdir, filepath.Dir(path))
		if err != nil {
			log.Errorf("Failed to check if %q belongs to %q: %v", path, c.logdir, err)
			return false
		}
		return matched
	}

	if dir != c.logdir {
		return false
	}
	return true
}

func (c *Cleaner) Clean() (err error) {
	var size int64 = 0
	var count int64 = 0
	beginClean := false
	doneFiles, err := c.meta.GetDoneFiles()
	if err != nil {
		return err
	}
	checked := make(map[string]struct{})
	for _, f := range doneFiles {
		logFiles := GetLogFiles(f.Path)
		allremoved := true
		for _, logf := range logFiles {
			if !c.checkBelong(logf.Path) {
				continue
			}
			if _, ok := checked[logf.Path]; ok {
				continue
			}
			checked[logf.Path] = struct{}{}
			size += logf.Info.Size()
			count++
			// 一旦符合条件，更老的文件必然都要删除
			if beginClean || c.shouldClean(size, count) {
				beginClean = true
				sig := CleanSignal{
					Logdir:   filepath.Dir(logf.Path),
					Filename: logf.Info.Name(),
					Cleaner:  c.name,
					ReadMode: c.meta.GetMode(),
				}
				log.Infof("send clean signal %v", sig)
				c.cleanChan <- sig
				if err = c.meta.AppendDeleteFile(logf.Path); err != nil {
					log.Error(err)
				}
			} else {
				allremoved = false
			}
		}
		if allremoved {
			if err = c.meta.DeleteDoneFile(f.Path); err != nil {
				log.Error(err)
			}
		}
	}
	return nil
}

func (c *Cleaner) LogDir() string {
	return c.logdir
}
<commit_msg>cleaner增加状态同步<commit_after>package cleaner

import (
	"path/filepath"
	"runtime/debug"
	"sync/atomic"
	"time"

	"github.com/bmatcuk/doublestar"

	"github.com/qiniu/log"

	"github.com/qiniu/logkit/conf"
	"github.com/qiniu/logkit/reader"
	"github.com/qiniu/logkit/reader/config"
	. "github.com/qiniu/logkit/utils/models"
)

type Cleaner struct {
	cleanTicker   <-chan time.Time
	reserveNumber int64 //个
	reserveSize   int64 //byte
	meta          *reader.Meta
	exitChan      chan struct{}
	cleanChan     chan<- CleanSignal
	name          string
	logdir        string
	status        int32
}

type CleanSignal struct {
	Logdir   string
	Filename string
	Cleaner  string
	ReadMode string
}

const (
	KeyCleanEnable       = "delete_enable"
	KeyCleanInterval     = "delete_interval"
	KeyReserveFileNumber = "reserve_file_number"
	KeyReserveFileSize   = "reserve_file_size"
	cleanerName          = "cleaner_name"

	defaultDeleteInterval    = 300  //5分钟
	defaultReserveFileNumber = 10   //默认保存是个文件
	defaultReserveFileSize   = 2048 //单位MB，默认删除保存2G
	// 如果两项任意一项达到要求，就执行删除；如果两项容易一项有值设置，但是另一项为0，就认为另一项不做限制
)

// 删除文件时遍历全部
// 删除时生成filedeleted文件
func NewCleaner(conf conf.MapConf, meta *reader.Meta, cleanChan chan<- CleanSignal, logdir string) (*Cleaner, error) {
	enable, _ := conf.GetBoolOr(KeyCleanEnable, false)
	if !enable {
		return nil, nil
	}
	mode := meta.GetMode()
	if mode != config.ModeDir &&
		mode != config.ModeFile &&
		mode != config.ModeCloudTrail &&
		mode != config.ModeCloudTrailV2 &&
		mode != config.ModeTailx &&
		mode != config.ModeDirx {
		log.Errorf("Cleaner only supports reader mode dir|file|cloudtrail|tailx|dirx, current mode is %v, cleaner disabled", meta.GetMode())
		return nil, nil
	}
	interval, _ := conf.GetIntOr(KeyCleanInterval, 0) //单位，秒
	if interval <= 0 {
		interval = defaultDeleteInterval
	}
	name, _ := conf.GetStringOr(cleanerName, "unknown")
	reserveNumber, _ := conf.GetInt64Or(KeyReserveFileNumber, 0)
	reserveSize, _ := conf.GetInt64Or(KeyReserveFileSize, 0)
	if reserveNumber <= 0 && reserveSize <= 0 {
		reserveNumber = defaultReserveFileNumber
		reserveSize = defaultReserveFileSize
	}
	reserveSize = reserveSize * MB
	if mode != config.ModeTailx && mode != config.ModeDirx {
		var err error
		logdir, _, err = GetRealPath(logdir)
		if err != nil {
			log.Errorf("Failed to get real path of %q: %v", logdir, err)
			return nil, err
		}
	}
	return &Cleaner{
		cleanTicker:   time.NewTicker(time.Duration(interval) * time.Second).C,
		reserveNumber: reserveNumber,
		reserveSize:   reserveSize,
		meta:          meta,
		exitChan:      make(chan struct{}),
		cleanChan:     cleanChan,
		name:          name,
		logdir:        logdir,
		status:        config.StatusInit,
	}, nil
}

func (c *Cleaner) Run() {
	if !atomic.CompareAndSwapInt32(&c.status, config.StatusInit, config.StatusRunning) {
		if c.hasStopped() {
			log.Warnf("cleaner[%v] has stopped, run operation ignored", c.name)
		} else {
			log.Warnf("cleaner[%v] has already running, run operation ignored", c.name)
		}
		return
	}
	for {
		select {
		case <-c.exitChan:
			log.Warnf("%v receive exit signal, cleaner exiting...", c.name)
			return
		case <-c.cleanTicker:
		}
		err := c.Clean()
		if err != nil {
			log.Error(err)
		}
	}
}

func (c *Cleaner) Close() {
	if !atomic.CompareAndSwapInt32(&c.status, config.StatusRunning, config.StatusStopped) {
		log.Warnf("cleaner[%v] is not running, close operation ignored", c.name)
		return
	}
	c.exitChan <- struct{}{}
}

func (c *Cleaner) hasStopped() bool {
	return atomic.LoadInt32(&c.status) == config.StatusStopped
}

func (c *Cleaner) Name() string {
	return c.name
}

func (c *Cleaner) shouldClean(size, count int64) bool {
	if c.reserveNumber > 0 && count > c.reserveNumber {
		return true
	}
	if c.reserveSize > 0 && size > c.reserveSize {
		return true
	}
	return false
}

func (c *Cleaner) checkBelong(path string) bool {
	dir := filepath.Dir(path)
	dir, _, err := GetRealPath(dir)
	if err != nil {
		log.Errorf("GetRealPath for %v error %v", path, err)
		return false
	}

	switch c.meta.GetMode() {
	case config.ModeTailx:
		matched, err := filepath.Match(filepath.Dir(c.logdir), filepath.Dir(path))
		if err != nil {
			log.Errorf("Failed to check if %q belongs to %q: %v", path, c.logdir, err)
			return false
		}
		return matched

	case config.ModeDirx:
		matched, err := doublestar.Match(c.logdir, filepath.Dir(path))
		if err != nil {
			log.Errorf("Failed to check if %q belongs to %q: %v", path, c.logdir, err)
			return false
		}
		return matched
	}

	if dir != c.logdir {
		return false
	}
	return true
}

func (c *Cleaner) Clean() (err error) {
	defer func() {
		if rec := recover(); rec != nil {
			log.Errorf("cleaner %q was panicked and recovered from %v\nstack: %s", c.Name(), rec, debug.Stack())
		}
	}()
	if c.hasStopped() {
		log.Warnf("cleaner[%v] reader %s has stopped, skip clean operation", c.name)
		return
	}
	var size int64 = 0
	var count int64 = 0
	beginClean := false
	doneFiles, err := c.meta.GetDoneFiles()
	if err != nil {
		return err
	}
	checked := make(map[string]struct{})
	for _, f := range doneFiles {
		logFiles := GetLogFiles(f.Path)
		allremoved := true
		for _, logf := range logFiles {
			if !c.checkBelong(logf.Path) {
				continue
			}
			if _, ok := checked[logf.Path]; ok {
				continue
			}
			checked[logf.Path] = struct{}{}
			size += logf.Info.Size()
			count++
			// 一旦符合条件，更老的文件必然都要删除
			if beginClean || c.shouldClean(size, count) {
				beginClean = true
				sig := CleanSignal{
					Logdir:   filepath.Dir(logf.Path),
					Filename: logf.Info.Name(),
					Cleaner:  c.name,
					ReadMode: c.meta.GetMode(),
				}
				log.Infof("send clean signal %v", sig)
				c.cleanChan <- sig
				if err = c.meta.AppendDeleteFile(logf.Path); err != nil {
					log.Error(err)
				}
			} else {
				allremoved = false
			}
		}
		if allremoved {
			if err = c.meta.DeleteDoneFile(f.Path); err != nil {
				log.Error(err)
			}
		}
	}
	return nil
}

func (c *Cleaner) LogDir() string {
	return c.logdir
}
<|endoftext|>


<commit_before>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package complete_chassis_reboot_test

import (
	"context"
	"sort"
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"
	"github.com/openconfig/featureprofiles/internal/fptest"
	spb "github.com/openconfig/gnoi/system"
	"github.com/openconfig/ondatra"
	"github.com/openconfig/testt"
)

const (
	oneMinuteInNanoSecond = 6e10
	oneSecondInNanoSecond = 1e9
	rebootDelay           = 120
	// Maximum reboot time is 900 seconds (15 minutes).
	maxRebootTime = 900
)

func TestMain(m *testing.M) {
	fptest.RunTests(m)
}

// Test cases:
//  1) Send gNOI reboot request using the method COLD with the delay of N seconds.
//     - method: Only the COLD method is required to be supported by all targets.
//     - Delay: In nanoseconds before issuing reboot.
//     - message: Informational reason for the reboot.
//     - force: Force reboot if basic checks fail. (ex. uncommitted configuration).
//   - Verify the following items.
//     - DUT remains reachable for N seconds by checking DUT current time is updated.
//     - DUT boot time is updated after reboot.
//     - DUT software version is the same after the reboot.
//  2) Send gNOI reboot request using the method COLD without delay.
//     - method: Only the COLD method is required to be supported by all targets.
//     - Delay: 0 - no delay.
//     - message: Informational reason for the reboot.
//     - force: Force reboot if basic checks fail. (ex. uncommitted configuration).
//   - Verify the following items.
//     - DUT boot time is updated after reboot.
//     - DUT software version is the same after the reboot.
//
// Topology:
//   dut:port1 <--> ate:port1
//
// Test notes:
//  - A RebootRequest requests the specified target be rebooted using the specified
//    method after the specified delay.  Only the DEFAULT method with a delay of 0
//    is guaranteed to be accepted for all target types.
//  - A RebootMethod determines what should be done with a target when a Reboot is
//    requested.  Only the COLD method is required to be supported by all
//    targets.  Methods the target does not support should result in failure.
//
//  - gnoi operation commands can be sent and tested using CLI command grpcurl.
//    https://github.com/fullstorydev/grpcurl
//

func TestChassisReboot(t *testing.T) {
	dut := ondatra.DUT(t, "dut")

	cases := []struct {
		desc          string
		rebootRequest *spb.RebootRequest
	}{
		{
			desc: "with delay",
			rebootRequest: &spb.RebootRequest{
				Method:  spb.RebootMethod_COLD,
				Delay:   rebootDelay * oneSecondInNanoSecond,
				Message: "Reboot chassis with delay",
				Force:   true,
			}},
		{
			desc: "without delay",
			rebootRequest: &spb.RebootRequest{
				Method:  spb.RebootMethod_COLD,
				Delay:   0,
				Message: "Reboot chassis without delay",
				Force:   true,
			}},
	}

	expectedVersion := dut.Telemetry().ComponentAny().SoftwareVersion().Get(t)
	sort.Strings(expectedVersion)
	t.Logf("DUT software version: %v", expectedVersion)
	gnoiClient := dut.RawAPIs().GNOI().Default(t)

	for _, tc := range cases {
		t.Run(tc.desc, func(t *testing.T) {
			// TODO: Remove t.Skipf() after reboot with no delay issue is supported.
			if tc.rebootRequest.GetDelay() == 0 {
				t.Skipf("delay 0 option is not working due to known bug.")
			}

			bootTimeBeforeReboot := dut.Telemetry().System().BootTime().Get(t)
			t.Logf("DUT boot time before reboot: %v", bootTimeBeforeReboot)
			prevTime, err := time.Parse(time.RFC3339, dut.Telemetry().System().CurrentDatetime().Get(t))
			if err != nil {
				t.Fatalf("Failed parsing current-datetime: %s", err)
			}
			start := time.Now()

			t.Logf("Send reboot request: %v", tc.rebootRequest)
			rebootResponse, err := gnoiClient.System().Reboot(context.Background(), tc.rebootRequest)
			t.Logf("Got reboot response: %v, err: %v", rebootResponse, err)
			if err != nil {
				t.Fatalf("Failed to reboot chassis with unexpected err: %v", err)
			}

			if tc.rebootRequest.GetDelay() > 1 {
				t.Logf("Validating DUT remains reachable for at least %d seconds", rebootDelay)
				for {
					time.Sleep(10 * time.Second)
					t.Logf("Time elapsed %.2f seconds since reboot was requested.", time.Since(start).Seconds())
					if uint64(time.Since(start).Seconds()) > rebootDelay {
						t.Logf("Time elapsed %.2f seconds > %d reboot delay", time.Since(start).Seconds(), rebootDelay)
						break
					}
					latestTime, err := time.Parse(time.RFC3339, dut.Telemetry().System().CurrentDatetime().Get(t))
					if err != nil {
						t.Fatalf("Failed parsing current-datetime: %s", err)
					}
					if latestTime.Before(prevTime) || latestTime.Equal(prevTime) {
						t.Errorf("Get latest system time: got %v, want newer time than %v", latestTime, prevTime)
					}
					prevTime = latestTime
				}
			}

			startReboot := time.Now()
			t.Logf("Wait for DUT to boot up by polling the telemetry output.")
			for {
				var currentTime string
				t.Logf("Time elapsed %.2f seconds since reboot started.", time.Since(startReboot).Seconds())
				time.Sleep(30 * time.Second)
				if errMsg := testt.CaptureFatal(t, func(t testing.TB) {
					currentTime = dut.Telemetry().System().CurrentDatetime().Get(t)
				}); errMsg != nil {
					t.Logf("Got testt.CaptureFatal errMsg: %s, keep polling ...", *errMsg)
				} else {
					t.Logf("Device rebooted successfully with received time: %v", currentTime)
					break
				}

				if uint64(time.Since(startReboot).Seconds()) > maxRebootTime {
					t.Errorf("Check boot time: got %v, want < %v", time.Since(startReboot), maxRebootTime)
				}
			}
			t.Logf("Device boot time: %.2f seconds", time.Since(startReboot).Seconds())

			bootTimeAfterReboot := dut.Telemetry().System().BootTime().Get(t)
			t.Logf("DUT boot time after reboot: %v", bootTimeAfterReboot)
			if bootTimeAfterReboot <= bootTimeBeforeReboot {
				t.Errorf("Get boot time: got %v, want > %v", bootTimeAfterReboot, bootTimeBeforeReboot)
			}

			swVersion := dut.Telemetry().ComponentAny().SoftwareVersion().Get(t)
			sort.Strings(swVersion)
			t.Logf("DUT software version after reboot: %v", swVersion)
			if diff := cmp.Diff(expectedVersion, swVersion); diff != "" {
				t.Errorf("Software version differed (-want +got):\n%v", diff)
			}
		})
	}
}
<commit_msg>Update complete_chassis_reboot_test.go (#430)<commit_after>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package complete_chassis_reboot_test

import (
	"context"
	"sort"
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"
	"github.com/openconfig/featureprofiles/internal/fptest"
	spb "github.com/openconfig/gnoi/system"
	"github.com/openconfig/ondatra"
	"github.com/openconfig/testt"
)

const (
	oneMinuteInNanoSecond = 6e10
	oneSecondInNanoSecond = 1e9
	rebootDelay           = 120
	// Maximum reboot time is 900 seconds (15 minutes).
	maxRebootTime = 900
)

func TestMain(m *testing.M) {
	fptest.RunTests(m)
}

// Test cases:
//  1) Send gNOI reboot request using the method COLD with the delay of N seconds.
//     - method: Only the COLD method is required to be supported by all targets.
//     - Delay: In nanoseconds before issuing reboot.
//     - message: Informational reason for the reboot.
//     - force: Force reboot if basic checks fail. (ex. uncommitted configuration).
//   - Verify the following items.
//     - DUT remains reachable for N seconds by checking DUT current time is updated.
//     - DUT boot time is updated after reboot.
//     - DUT software version is the same after the reboot.
//  2) Send gNOI reboot request using the method COLD without delay.
//     - method: Only the COLD method is required to be supported by all targets.
//     - Delay: 0 - no delay.
//     - message: Informational reason for the reboot.
//     - force: Force reboot if basic checks fail. (ex. uncommitted configuration).
//   - Verify the following items.
//     - DUT boot time is updated after reboot.
//     - DUT software version is the same after the reboot.
//
// Topology:
//   dut:port1 <--> ate:port1
//
// Test notes:
//  - A RebootRequest requests the specified target be rebooted using the specified
//    method after the specified delay.  Only the DEFAULT method with a delay of 0
//    is guaranteed to be accepted for all target types.
//  - A RebootMethod determines what should be done with a target when a Reboot is
//    requested.  Only the COLD method is required to be supported by all
//    targets.  Methods the target does not support should result in failure.
//
//  - gnoi operation commands can be sent and tested using CLI command grpcurl.
//    https://github.com/fullstorydev/grpcurl
//

func TestChassisReboot(t *testing.T) {
	dut := ondatra.DUT(t, "dut")

	cases := []struct {
		desc          string
		rebootRequest *spb.RebootRequest
	}{
		{
			desc: "with delay",
			rebootRequest: &spb.RebootRequest{
				Method:  spb.RebootMethod_COLD,
				Delay:   rebootDelay * oneSecondInNanoSecond,
				Message: "Reboot chassis with delay",
				Force:   true,
			}},
		{
			desc: "without delay",
			rebootRequest: &spb.RebootRequest{
				Method:  spb.RebootMethod_COLD,
				Delay:   0,
				Message: "Reboot chassis without delay",
				Force:   true,
			}},
	}

	expectedVersion := dut.Telemetry().ComponentAny().SoftwareVersion().Get(t)
	sort.Strings(expectedVersion)
	t.Logf("DUT software version: %v", expectedVersion)
	gnoiClient := dut.RawAPIs().GNOI().Default(t)

	for _, tc := range cases {
		t.Run(tc.desc, func(t *testing.T) {
			bootTimeBeforeReboot := dut.Telemetry().System().BootTime().Get(t)
			t.Logf("DUT boot time before reboot: %v", bootTimeBeforeReboot)
			prevTime, err := time.Parse(time.RFC3339, dut.Telemetry().System().CurrentDatetime().Get(t))
			if err != nil {
				t.Fatalf("Failed parsing current-datetime: %s", err)
			}
			start := time.Now()

			t.Logf("Send reboot request: %v", tc.rebootRequest)
			rebootResponse, err := gnoiClient.System().Reboot(context.Background(), tc.rebootRequest)
			t.Logf("Got reboot response: %v, err: %v", rebootResponse, err)
			if err != nil {
				t.Fatalf("Failed to reboot chassis with unexpected err: %v", err)
			}

			if tc.rebootRequest.GetDelay() > 1 {
				t.Logf("Validating DUT remains reachable for at least %d seconds", rebootDelay)
				for {
					time.Sleep(10 * time.Second)
					t.Logf("Time elapsed %.2f seconds since reboot was requested.", time.Since(start).Seconds())
					if uint64(time.Since(start).Seconds()) > rebootDelay {
						t.Logf("Time elapsed %.2f seconds > %d reboot delay", time.Since(start).Seconds(), rebootDelay)
						break
					}
					latestTime, err := time.Parse(time.RFC3339, dut.Telemetry().System().CurrentDatetime().Get(t))
					if err != nil {
						t.Fatalf("Failed parsing current-datetime: %s", err)
					}
					if latestTime.Before(prevTime) || latestTime.Equal(prevTime) {
						t.Errorf("Get latest system time: got %v, want newer time than %v", latestTime, prevTime)
					}
					prevTime = latestTime
				}
			}

			startReboot := time.Now()
			t.Logf("Wait for DUT to boot up by polling the telemetry output.")
			for {
				var currentTime string
				t.Logf("Time elapsed %.2f seconds since reboot started.", time.Since(startReboot).Seconds())
				time.Sleep(30 * time.Second)
				if errMsg := testt.CaptureFatal(t, func(t testing.TB) {
					currentTime = dut.Telemetry().System().CurrentDatetime().Get(t)
				}); errMsg != nil {
					t.Logf("Got testt.CaptureFatal errMsg: %s, keep polling ...", *errMsg)
				} else {
					t.Logf("Device rebooted successfully with received time: %v", currentTime)
					break
				}

				if uint64(time.Since(startReboot).Seconds()) > maxRebootTime {
					t.Errorf("Check boot time: got %v, want < %v", time.Since(startReboot), maxRebootTime)
				}
			}
			t.Logf("Device boot time: %.2f seconds", time.Since(startReboot).Seconds())

			bootTimeAfterReboot := dut.Telemetry().System().BootTime().Get(t)
			t.Logf("DUT boot time after reboot: %v", bootTimeAfterReboot)
			if bootTimeAfterReboot <= bootTimeBeforeReboot {
				t.Errorf("Get boot time: got %v, want > %v", bootTimeAfterReboot, bootTimeBeforeReboot)
			}

			swVersion := dut.Telemetry().ComponentAny().SoftwareVersion().Get(t)
			sort.Strings(swVersion)
			t.Logf("DUT software version after reboot: %v", swVersion)
			if diff := cmp.Diff(expectedVersion, swVersion); diff != "" {
				t.Errorf("Software version differed (-want +got):\n%v", diff)
			}
		})
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/keybase/go/libcmdline"
	"github.com/keybase/go/libkb"
	"io"
)

func NewCmdSign(cl *libcmdline.CommandLine) cli.Command {
	return cli.Command{
		Name:        "sign",
		Usage:       "keybase sign [-a] [-o <outfile>] [<infile>]",
		Description: "sign a clear document",
		Action: func(c *cli.Context) {
			cl.ChooseCommand(&CmdSign{}, "sign", c)
		},
		Flags: []cli.Flag{
			cli.BoolFlag{
				Name:  "b, binary",
				Usage: "output binary message (armored by default",
			},
			cli.StringFlag{
				Name:  "m, message",
				Usage: "provide the message to sign on the command line",
			},
			cli.StringFlag{
				Name:  "o, outfile",
				Usage: "specify an outfile (stdout by default",
			},
		},
	}
}

type CmdSign struct {
	UnixFilter
	binary bool
	msg    string
}

func (s *CmdSign) ParseArgv(ctx *cli.Context) error {
	nargs := len(ctx.Args())
	var err error

	s.binary = ctx.Bool("binary")
	msg := ctx.String("message")
	outfile := ctx.String("outfile")
	var infile string

	if nargs == 1 {
		infile = ctx.Args()[0]
	} else if nargs > 1 {
		err = fmt.Errorf("sign takes at most 1 arg, an infile")
	}

	if err == nil {
		err = s.FilterInit(msg, infile, outfile)
	}

	return err
}

func (s *CmdSign) RunClient() (err error) { return s.Run() }

func (s *CmdSign) Run() (err error) {
	var key libkb.GenericKey
	var pgp *libkb.PgpKeyBundle
	var ok bool
	var dumpTo io.WriteCloser
	var written int64

	if err = s.FilterOpen(); err != nil {
		return
	}

	defer func() {
		if dumpTo != nil {
			dumpTo.Close()
		}
		s.Close(err)
	}()

	key, err = G.Keyrings.GetSecretKey("command-line signature", nil, false)
	if err != nil {
		return
	} else if pgp, ok = key.(*libkb.PgpKeyBundle); !ok {
		err = fmt.Errorf("Can only sign with PGP keys (for now)")
		return
	} else if key == nil {
		err = fmt.Errorf("No secret key available")
		return
	}

	dumpTo, err = libkb.AttachedSignWrapper(s.sink, *pgp, !s.binary)
	if err != nil {
		return
	}

	written, err = io.Copy(dumpTo, s.source)
	if err == nil && written == 0 {
		err = fmt.Errorf("Empty source file, nothing to sign")
	}

	return
}

func (v *CmdSign) GetUsage() libkb.Usage {
	return libkb.Usage{
		Config:    true,
		API:       true,
		Terminal:  true,
		KbKeyring: true,
	}
}
<commit_msg>fix compile<commit_after>package main

import (
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/keybase/go/libcmdline"
	"github.com/keybase/go/libkb"
	"io"
)

func NewCmdSign(cl *libcmdline.CommandLine) cli.Command {
	return cli.Command{
		Name:        "sign",
		Usage:       "keybase sign [-a] [-o <outfile>] [<infile>]",
		Description: "sign a clear document",
		Action: func(c *cli.Context) {
			cl.ChooseCommand(&CmdSign{}, "sign", c)
		},
		Flags: []cli.Flag{
			cli.BoolFlag{
				Name:  "b, binary",
				Usage: "output binary message (armored by default",
			},
			cli.StringFlag{
				Name:  "m, message",
				Usage: "provide the message to sign on the command line",
			},
			cli.StringFlag{
				Name:  "o, outfile",
				Usage: "specify an outfile (stdout by default",
			},
		},
	}
}

type CmdSign struct {
	UnixFilter
	binary bool
	msg    string
}

func (s *CmdSign) ParseArgv(ctx *cli.Context) error {
	nargs := len(ctx.Args())
	var err error

	s.binary = ctx.Bool("binary")
	msg := ctx.String("message")
	outfile := ctx.String("outfile")
	var infile string

	if nargs == 1 {
		infile = ctx.Args()[0]
	} else if nargs > 1 {
		err = fmt.Errorf("sign takes at most 1 arg, an infile")
	}

	if err == nil {
		err = s.FilterInit(msg, infile, outfile)
	}

	return err
}

func (s *CmdSign) RunClient() (err error) { return s.Run() }

func (s *CmdSign) Run() (err error) {
	var key libkb.GenericKey
	var pgp *libkb.PgpKeyBundle
	var ok bool
	var dumpTo io.WriteCloser
	var written int64

	if err = s.FilterOpen(); err != nil {
		return
	}

	defer func() {
		if dumpTo != nil {
			dumpTo.Close()
		}
		s.Close(err)
	}()

	key, err = G.Keyrings.GetSecretKey("command-line signature", nil, nil, false)
	if err != nil {
		return
	} else if pgp, ok = key.(*libkb.PgpKeyBundle); !ok {
		err = fmt.Errorf("Can only sign with PGP keys (for now)")
		return
	} else if key == nil {
		err = fmt.Errorf("No secret key available")
		return
	}

	dumpTo, err = libkb.AttachedSignWrapper(s.sink, *pgp, !s.binary)
	if err != nil {
		return
	}

	written, err = io.Copy(dumpTo, s.source)
	if err == nil && written == 0 {
		err = fmt.Errorf("Empty source file, nothing to sign")
	}

	return
}

func (v *CmdSign) GetUsage() libkb.Usage {
	return libkb.Usage{
		Config:    true,
		API:       true,
		Terminal:  true,
		KbKeyring: true,
	}
}
<|endoftext|>


<commit_before>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2017 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package cloudapi

import (
	"encoding/json"
	"time"

	"gopkg.in/guregu/null.v3"

	"github.com/loadimpact/k6/lib/types"
)

// Config holds all the necessary data and options for sending metrics to the Load Impact cloud.
//nolint: lll
type Config struct {
	// TODO: refactor common stuff between cloud execution and output
	Token           null.String `json:"token" envconfig:"K6_CLOUD_TOKEN"`
	DeprecatedToken null.String `json:"-" envconfig:"K6CLOUD_TOKEN"`
	ProjectID       null.Int    `json:"projectID" envconfig:"K6_CLOUD_PROJECT_ID"`
	Name            null.String `json:"name" envconfig:"K6_CLOUD_NAME"`

	Host        null.String `json:"host" envconfig:"K6_CLOUD_HOST"`
	LogsTailURL null.String `json:"-" envconfig:"K6_CLOUD_LOGS_TAIL_URL"`
	PushRefID   null.String `json:"pushRefID" envconfig:"K6_CLOUD_PUSH_REF_ID"`
	WebAppURL   null.String `json:"webAppURL" envconfig:"K6_CLOUD_WEB_APP_URL"`
	NoCompress  null.Bool   `json:"noCompress" envconfig:"K6_CLOUD_NO_COMPRESS"`

	MaxMetricSamplesPerPackage null.Int `json:"maxMetricSamplesPerPackage" envconfig:"K6_CLOUD_MAX_METRIC_SAMPLES_PER_PACKAGE"`

	// The time interval between periodic API calls for sending samples to the cloud ingest service.
	MetricPushInterval types.NullDuration `json:"metricPushInterval" envconfig:"K6_CLOUD_METRIC_PUSH_INTERVAL"`

	// This is how many concurrent pushes will be done at the same time to the cloud
	MetricPushConcurrency null.Int `json:"metricPushConcurrency" envconfig:"K6_CLOUD_METRIC_PUSH_CONCURRENCY"`

	// Aggregation docs:
	//
	// If AggregationPeriod is specified and if it is greater than 0, HTTP metric aggregation
	// with that period will be enabled. The general algorithm is this:
	// - HTTP trail samples will be collected separately and not
	//   included in the default sample buffer (which is directly sent
	//   to the cloud service every MetricPushInterval).
	// - On every AggregationCalcInterval, all collected HTTP Trails will be
	//   split into AggregationPeriod-sized time buckets (time slots) and
	//   then into sub-buckets according to their tags (each sub-bucket
	//   will contain only HTTP trails with the same sample tags -
	//   proto, staus, URL, method, etc.).
	// - If at that time the specified AggregationWaitPeriod has not passed
	//   for a particular time bucket, it will be left undisturbed until the next
	//   AggregationCalcInterval tick comes along.
	// - If AggregationWaitPeriod has passed for a time bucket, all of its
	//   sub-buckets will be traversed. Any sub-buckets that have less than
	//   AggregationMinSamples HTTP trails in them will not be aggregated.
	//   Instead the HTTP trails in them will just be individually added
	//   to the default sample buffer, like they would be if there was no
	//   aggregation.
	// - Sub-buckets with at least AggregationMinSamples HTTP trails on the
	//   other hand will be aggregated according to the algorithm below:
	//     - If AggregationSkipOutlierDetection is enabled, all of the collected
	//       HTTP trails in that sub-bucket will be directly aggregated into a single
	//       compoind metric sample, without any attempt at outlier detection.
	//       IMPORTANT: This is intended only for testing purposes only or, in
	//       extreme cases, when the resulting metrics' precision isn't very important,
	//       since it could lead to a huge loss of granularity and the masking
	//       of any outlier samples in the data.
	//     - By default (since AggregationSkipOutlierDetection is not enabled),
	//       the collected HTTP trails will be checked for outliers, so we don't loose
	//       granularity by accidentally aggregating them. That happens by finding
	//       the "quartiles" (by default the 75th and 25th percentiles) in the
	//       sub-bucket datapoints and using the inter-quartile range (IQR) to find
	//       any outliers (https://en.wikipedia.org/wiki/Interquartile_range#Outliers,
	//       though the specific parameters and coefficients can be customized
	//       by the AggregationOutlier{Radius,CoefLower,CoefUpper} options)
	//     - Depending on the number of samples in the sub-bucket, two different
	//       algorithms could be used to calculate the quartiles. If there are
	//       fewer samples (between AggregationMinSamples and AggregationOutlierAlgoThreshold),
	//       then a more precise but also more computationally-heavy sorting-based algorithm
	//       will be used. For sub-buckets with more samples, a lighter quickselect-based
	//       algorithm will be used, potentially with a very minor loss of precision.
	//     - Regardless of the used algorithm, once the quartiles for that sub-bucket
	//       are found and the IQR is calculated, every HTTP trail in the sub-bucket will
	//       be checked if it seems like an outlier. HTTP trails are evaluated by two different
	//       criteria whether they seem like outliers - by their total connection time (i.e.
	//       http_req_connecting + http_req_tls_handshaking) and by their total request time
	//       (i.e. http_req_sending + http_req_waiting + http_req_receiving). If any of those
	//       properties of an HTTP trail is out of the calculated "normal" bounds for the
	//       sub-bucket, it will be considered an outlier and will be sent to the cloud
	//       individually - it's simply added to the default sample buffer, like it would
	//       be if there was no aggregation.
	//     - Finally, all non-outliers are aggregated and the resultig single metric is also
	//       added to the default sample buffer for sending to the cloud ingest service
	//       on the next MetricPushInterval event.

	// If specified and is greater than 0, sample aggregation with that period is enabled
	AggregationPeriod types.NullDuration `json:"aggregationPeriod" envconfig:"K6_CLOUD_AGGREGATION_PERIOD"`

	// If aggregation is enabled, this is how often new HTTP trails will be sorted into buckets and sub-buckets and aggregated.
	AggregationCalcInterval types.NullDuration `json:"aggregationCalcInterval" envconfig:"K6_CLOUD_AGGREGATION_CALC_INTERVAL"`

	// If aggregation is enabled, this specifies how long we'll wait for period samples to accumulate before trying to aggregate them.
	AggregationWaitPeriod types.NullDuration `json:"aggregationWaitPeriod" envconfig:"K6_CLOUD_AGGREGATION_WAIT_PERIOD"`

	// If aggregation is enabled, but the collected samples for a certain AggregationPeriod after AggregationPushDelay has passed are less than this number, they won't be aggregated.
	AggregationMinSamples null.Int `json:"aggregationMinSamples" envconfig:"K6_CLOUD_AGGREGATION_MIN_SAMPLES"`

	// If this is enabled and a sub-bucket has more than AggregationMinSamples HTTP trails in it, they would all be
	// aggregated without attempting to find and separate any outlier metrics first.
	// IMPORTANT: This is intended for testing purposes only or, in extreme cases, when the result precision
	// isn't very important and the improved aggregation percentage would be worth the potentially huge loss
	// of metric granularity and possible masking of any outlier samples.
	AggregationSkipOutlierDetection null.Bool `json:"aggregationSkipOutlierDetection" envconfig:"K6_CLOUD_AGGREGATION_SKIP_OUTLIER_DETECTION"`

	// If aggregation and outlier detection are enabled, this option specifies the
	// number of HTTP trails in a sub-bucket that determine which quartile-calculating
	// algorithm would be used:
	// - for fewer samples (between MinSamples and OutlierAlgoThreshold), a more precise
	//   (i.e. supporting interpolation), but also more computationally-heavy sorting
	//   algorithm will be used to find the quartiles.
	// - if there are more samples than OutlierAlgoThreshold in the sub-bucket, a
	//   QuickSelect-based (https://en.wikipedia.org/wiki/Quickselect) algorithm will
	//   be used. It doesn't support interpolation, so there's a small loss of precision
	//   in the outlier detection, but it's not as resource-heavy as the sorting algorithm.
	AggregationOutlierAlgoThreshold null.Int `json:"aggregationOutlierAlgoThreshold" envconfig:"K6_CLOUD_AGGREGATION_OUTLIER_ALGO_THRESHOLD"`

	// The radius (as a fraction) from the median at which to sample Q1 and Q3.
	// By default it's one quarter (0.25) and if set to something different, the Q in IQR
	// won't make much sense... But this would allow us to select tighter sample groups for
	// aggregation if we want.
	AggregationOutlierIqrRadius null.Float `json:"aggregationOutlierIqrRadius" envconfig:"K6_CLOUD_AGGREGATION_OUTLIER_IQR_RADIUS"`

	// Connection or request times with how many IQRs below Q1 to consier as non-aggregatable outliers.
	AggregationOutlierIqrCoefLower null.Float `json:"aggregationOutlierIqrCoefLower" envconfig:"K6_CLOUD_AGGREGATION_OUTLIER_IQR_COEF_LOWER"`

	// Connection or request times with how many IQRs above Q3 to consier as non-aggregatable outliers.
	AggregationOutlierIqrCoefUpper null.Float `json:"aggregationOutlierIqrCoefUpper" envconfig:"K6_CLOUD_AGGREGATION_OUTLIER_IQR_COEF_UPPER"`
}

// NewConfig creates a new Config instance with default values for some fields.
func NewConfig() Config {
	return Config{
		Host:                       null.NewString("https://ingest.k6.io", false),
		LogsTailURL:                null.NewString("wss://cloudlogs.k6.io/api/v1/tail", false),
		WebAppURL:                  null.NewString("https://app.k6.io", false),
		MetricPushInterval:         types.NewNullDuration(1*time.Second, false),
		MetricPushConcurrency:      null.NewInt(1, false),
		MaxMetricSamplesPerPackage: null.NewInt(100000, false),
		// Aggregation is disabled by default, since AggregationPeriod has no default value
		// but if it's enabled manually or from the cloud service, those are the default values it will use:
		AggregationCalcInterval:         types.NewNullDuration(3*time.Second, false),
		AggregationWaitPeriod:           types.NewNullDuration(5*time.Second, false),
		AggregationMinSamples:           null.NewInt(25, false),
		AggregationOutlierAlgoThreshold: null.NewInt(75, false),
		AggregationOutlierIqrRadius:     null.NewFloat(0.25, false),

		// Since we're measuring durations, the upper coefficient is slightly
		// lower, since outliers from that side are more interesting than ones
		// close to zero.
		AggregationOutlierIqrCoefLower: null.NewFloat(1.5, false),
		AggregationOutlierIqrCoefUpper: null.NewFloat(1.3, false),
	}
}

// Apply saves config non-zero config values from the passed config in the receiver.
func (c Config) Apply(cfg Config) Config {
	if cfg.Token.Valid {
		c.Token = cfg.Token
	}
	if cfg.DeprecatedToken.Valid {
		c.DeprecatedToken = cfg.DeprecatedToken
	}
	if cfg.Name.Valid && cfg.Name.String != "" {
		c.Name = cfg.Name
	}
	if cfg.Host.Valid && cfg.Host.String != "" {
		c.Host = cfg.Host
	}
	if cfg.LogsTailURL.Valid && cfg.LogsTailURL.String != "" {
		c.LogsTailURL = cfg.LogsTailURL
	}
	if cfg.WebAppURL.Valid {
		c.WebAppURL = cfg.WebAppURL
	}
	if cfg.NoCompress.Valid {
		c.NoCompress = cfg.NoCompress
	}
	if cfg.ProjectID.Valid && cfg.ProjectID.Int64 > 0 {
		c.ProjectID = cfg.ProjectID
	}
	if cfg.MetricPushInterval.Valid {
		c.MetricPushInterval = cfg.MetricPushInterval
	}
	if cfg.MaxMetricSamplesPerPackage.Valid {
		c.MaxMetricSamplesPerPackage = cfg.MaxMetricSamplesPerPackage
	}
	if cfg.AggregationPeriod.Valid {
		c.AggregationPeriod = cfg.AggregationPeriod
	}
	if cfg.AggregationCalcInterval.Valid {
		c.AggregationCalcInterval = cfg.AggregationCalcInterval
	}
	if cfg.AggregationWaitPeriod.Valid {
		c.AggregationWaitPeriod = cfg.AggregationWaitPeriod
	}
	if cfg.AggregationMinSamples.Valid {
		c.AggregationMinSamples = cfg.AggregationMinSamples
	}
	if cfg.AggregationSkipOutlierDetection.Valid {
		c.AggregationSkipOutlierDetection = cfg.AggregationSkipOutlierDetection
	}
	if cfg.AggregationOutlierAlgoThreshold.Valid {
		c.AggregationOutlierAlgoThreshold = cfg.AggregationOutlierAlgoThreshold
	}
	if cfg.AggregationOutlierIqrRadius.Valid {
		c.AggregationOutlierIqrRadius = cfg.AggregationOutlierIqrRadius
	}
	if cfg.AggregationOutlierIqrCoefLower.Valid {
		c.AggregationOutlierIqrCoefLower = cfg.AggregationOutlierIqrCoefLower
	}
	if cfg.AggregationOutlierIqrCoefUpper.Valid {
		c.AggregationOutlierIqrCoefUpper = cfg.AggregationOutlierIqrCoefUpper
	}
	return c
}

// MergeFromExternal merges three fields from json in a loadimact key of the provided external map
func MergeFromExternal(external map[string]json.RawMessage, conf *Config) error {
	if val, ok := external["loadimpact"]; ok {
		// TODO: Important! Separate configs and fix the whole 2 configs mess!
		tmpConfig := Config{}
		if err := json.Unmarshal(val, &tmpConfig); err != nil {
			return err
		}
		// Only take out the ProjectID, Name and Token from the options.ext.loadimpact map:
		if tmpConfig.ProjectID.Valid {
			conf.ProjectID = tmpConfig.ProjectID
		}
		if tmpConfig.Name.Valid {
			conf.Name = tmpConfig.Name
		}
		if tmpConfig.Token.Valid {
			conf.Token = tmpConfig.Token
		}
	}
	return nil
}
<commit_msg>Fix a comment typo<commit_after>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2017 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package cloudapi

import (
	"encoding/json"
	"time"

	"gopkg.in/guregu/null.v3"

	"github.com/loadimpact/k6/lib/types"
)

// Config holds all the necessary data and options for sending metrics to the Load Impact cloud.
//nolint: lll
type Config struct {
	// TODO: refactor common stuff between cloud execution and output
	Token           null.String `json:"token" envconfig:"K6_CLOUD_TOKEN"`
	DeprecatedToken null.String `json:"-" envconfig:"K6CLOUD_TOKEN"`
	ProjectID       null.Int    `json:"projectID" envconfig:"K6_CLOUD_PROJECT_ID"`
	Name            null.String `json:"name" envconfig:"K6_CLOUD_NAME"`

	Host        null.String `json:"host" envconfig:"K6_CLOUD_HOST"`
	LogsTailURL null.String `json:"-" envconfig:"K6_CLOUD_LOGS_TAIL_URL"`
	PushRefID   null.String `json:"pushRefID" envconfig:"K6_CLOUD_PUSH_REF_ID"`
	WebAppURL   null.String `json:"webAppURL" envconfig:"K6_CLOUD_WEB_APP_URL"`
	NoCompress  null.Bool   `json:"noCompress" envconfig:"K6_CLOUD_NO_COMPRESS"`

	MaxMetricSamplesPerPackage null.Int `json:"maxMetricSamplesPerPackage" envconfig:"K6_CLOUD_MAX_METRIC_SAMPLES_PER_PACKAGE"`

	// The time interval between periodic API calls for sending samples to the cloud ingest service.
	MetricPushInterval types.NullDuration `json:"metricPushInterval" envconfig:"K6_CLOUD_METRIC_PUSH_INTERVAL"`

	// This is how many concurrent pushes will be done at the same time to the cloud
	MetricPushConcurrency null.Int `json:"metricPushConcurrency" envconfig:"K6_CLOUD_METRIC_PUSH_CONCURRENCY"`

	// Aggregation docs:
	//
	// If AggregationPeriod is specified and if it is greater than 0, HTTP metric aggregation
	// with that period will be enabled. The general algorithm is this:
	// - HTTP trail samples will be collected separately and not
	//   included in the default sample buffer (which is directly sent
	//   to the cloud service every MetricPushInterval).
	// - On every AggregationCalcInterval, all collected HTTP Trails will be
	//   split into AggregationPeriod-sized time buckets (time slots) and
	//   then into sub-buckets according to their tags (each sub-bucket
	//   will contain only HTTP trails with the same sample tags -
	//   proto, staus, URL, method, etc.).
	// - If at that time the specified AggregationWaitPeriod has not passed
	//   for a particular time bucket, it will be left undisturbed until the next
	//   AggregationCalcInterval tick comes along.
	// - If AggregationWaitPeriod has passed for a time bucket, all of its
	//   sub-buckets will be traversed. Any sub-buckets that have less than
	//   AggregationMinSamples HTTP trails in them will not be aggregated.
	//   Instead the HTTP trails in them will just be individually added
	//   to the default sample buffer, like they would be if there was no
	//   aggregation.
	// - Sub-buckets with at least AggregationMinSamples HTTP trails on the
	//   other hand will be aggregated according to the algorithm below:
	//     - If AggregationSkipOutlierDetection is enabled, all of the collected
	//       HTTP trails in that sub-bucket will be directly aggregated into a single
	//       compoind metric sample, without any attempt at outlier detection.
	//       IMPORTANT: This is intended only for testing purposes only or, in
	//       extreme cases, when the resulting metrics' precision isn't very important,
	//       since it could lead to a huge loss of granularity and the masking
	//       of any outlier samples in the data.
	//     - By default (since AggregationSkipOutlierDetection is not enabled),
	//       the collected HTTP trails will be checked for outliers, so we don't loose
	//       granularity by accidentally aggregating them. That happens by finding
	//       the "quartiles" (by default the 75th and 25th percentiles) in the
	//       sub-bucket datapoints and using the inter-quartile range (IQR) to find
	//       any outliers (https://en.wikipedia.org/wiki/Interquartile_range#Outliers,
	//       though the specific parameters and coefficients can be customized
	//       by the AggregationOutlier{Radius,CoefLower,CoefUpper} options)
	//     - Depending on the number of samples in the sub-bucket, two different
	//       algorithms could be used to calculate the quartiles. If there are
	//       fewer samples (between AggregationMinSamples and AggregationOutlierAlgoThreshold),
	//       then a more precise but also more computationally-heavy sorting-based algorithm
	//       will be used. For sub-buckets with more samples, a lighter quickselect-based
	//       algorithm will be used, potentially with a very minor loss of precision.
	//     - Regardless of the used algorithm, once the quartiles for that sub-bucket
	//       are found and the IQR is calculated, every HTTP trail in the sub-bucket will
	//       be checked if it seems like an outlier. HTTP trails are evaluated by two different
	//       criteria whether they seem like outliers - by their total connection time (i.e.
	//       http_req_connecting + http_req_tls_handshaking) and by their total request time
	//       (i.e. http_req_sending + http_req_waiting + http_req_receiving). If any of those
	//       properties of an HTTP trail is out of the calculated "normal" bounds for the
	//       sub-bucket, it will be considered an outlier and will be sent to the cloud
	//       individually - it's simply added to the default sample buffer, like it would
	//       be if there was no aggregation.
	//     - Finally, all non-outliers are aggregated and the resultig single metric is also
	//       added to the default sample buffer for sending to the cloud ingest service
	//       on the next MetricPushInterval event.

	// If specified and is greater than 0, sample aggregation with that period is enabled
	AggregationPeriod types.NullDuration `json:"aggregationPeriod" envconfig:"K6_CLOUD_AGGREGATION_PERIOD"`

	// If aggregation is enabled, this is how often new HTTP trails will be sorted into buckets and sub-buckets and aggregated.
	AggregationCalcInterval types.NullDuration `json:"aggregationCalcInterval" envconfig:"K6_CLOUD_AGGREGATION_CALC_INTERVAL"`

	// If aggregation is enabled, this specifies how long we'll wait for period samples to accumulate before trying to aggregate them.
	AggregationWaitPeriod types.NullDuration `json:"aggregationWaitPeriod" envconfig:"K6_CLOUD_AGGREGATION_WAIT_PERIOD"`

	// If aggregation is enabled, but the collected samples for a certain AggregationPeriod after AggregationPushDelay has passed are less than this number, they won't be aggregated.
	AggregationMinSamples null.Int `json:"aggregationMinSamples" envconfig:"K6_CLOUD_AGGREGATION_MIN_SAMPLES"`

	// If this is enabled and a sub-bucket has more than AggregationMinSamples HTTP trails in it, they would all be
	// aggregated without attempting to find and separate any outlier metrics first.
	// IMPORTANT: This is intended for testing purposes only or, in extreme cases, when the result precision
	// isn't very important and the improved aggregation percentage would be worth the potentially huge loss
	// of metric granularity and possible masking of any outlier samples.
	AggregationSkipOutlierDetection null.Bool `json:"aggregationSkipOutlierDetection" envconfig:"K6_CLOUD_AGGREGATION_SKIP_OUTLIER_DETECTION"`

	// If aggregation and outlier detection are enabled, this option specifies the
	// number of HTTP trails in a sub-bucket that determine which quartile-calculating
	// algorithm would be used:
	// - for fewer samples (between MinSamples and OutlierAlgoThreshold), a more precise
	//   (i.e. supporting interpolation), but also more computationally-heavy sorting
	//   algorithm will be used to find the quartiles.
	// - if there are more samples than OutlierAlgoThreshold in the sub-bucket, a
	//   QuickSelect-based (https://en.wikipedia.org/wiki/Quickselect) algorithm will
	//   be used. It doesn't support interpolation, so there's a small loss of precision
	//   in the outlier detection, but it's not as resource-heavy as the sorting algorithm.
	AggregationOutlierAlgoThreshold null.Int `json:"aggregationOutlierAlgoThreshold" envconfig:"K6_CLOUD_AGGREGATION_OUTLIER_ALGO_THRESHOLD"`

	// The radius (as a fraction) from the median at which to sample Q1 and Q3.
	// By default it's one quarter (0.25) and if set to something different, the Q in IQR
	// won't make much sense... But this would allow us to select tighter sample groups for
	// aggregation if we want.
	AggregationOutlierIqrRadius null.Float `json:"aggregationOutlierIqrRadius" envconfig:"K6_CLOUD_AGGREGATION_OUTLIER_IQR_RADIUS"`

	// Connection or request times with how many IQRs below Q1 to consier as non-aggregatable outliers.
	AggregationOutlierIqrCoefLower null.Float `json:"aggregationOutlierIqrCoefLower" envconfig:"K6_CLOUD_AGGREGATION_OUTLIER_IQR_COEF_LOWER"`

	// Connection or request times with how many IQRs above Q3 to consier as non-aggregatable outliers.
	AggregationOutlierIqrCoefUpper null.Float `json:"aggregationOutlierIqrCoefUpper" envconfig:"K6_CLOUD_AGGREGATION_OUTLIER_IQR_COEF_UPPER"`
}

// NewConfig creates a new Config instance with default values for some fields.
func NewConfig() Config {
	return Config{
		Host:                       null.NewString("https://ingest.k6.io", false),
		LogsTailURL:                null.NewString("wss://cloudlogs.k6.io/api/v1/tail", false),
		WebAppURL:                  null.NewString("https://app.k6.io", false),
		MetricPushInterval:         types.NewNullDuration(1*time.Second, false),
		MetricPushConcurrency:      null.NewInt(1, false),
		MaxMetricSamplesPerPackage: null.NewInt(100000, false),
		// Aggregation is disabled by default, since AggregationPeriod has no default value
		// but if it's enabled manually or from the cloud service, those are the default values it will use:
		AggregationCalcInterval:         types.NewNullDuration(3*time.Second, false),
		AggregationWaitPeriod:           types.NewNullDuration(5*time.Second, false),
		AggregationMinSamples:           null.NewInt(25, false),
		AggregationOutlierAlgoThreshold: null.NewInt(75, false),
		AggregationOutlierIqrRadius:     null.NewFloat(0.25, false),

		// Since we're measuring durations, the upper coefficient is slightly
		// lower, since outliers from that side are more interesting than ones
		// close to zero.
		AggregationOutlierIqrCoefLower: null.NewFloat(1.5, false),
		AggregationOutlierIqrCoefUpper: null.NewFloat(1.3, false),
	}
}

// Apply saves config non-zero config values from the passed config in the receiver.
func (c Config) Apply(cfg Config) Config {
	if cfg.Token.Valid {
		c.Token = cfg.Token
	}
	if cfg.DeprecatedToken.Valid {
		c.DeprecatedToken = cfg.DeprecatedToken
	}
	if cfg.Name.Valid && cfg.Name.String != "" {
		c.Name = cfg.Name
	}
	if cfg.Host.Valid && cfg.Host.String != "" {
		c.Host = cfg.Host
	}
	if cfg.LogsTailURL.Valid && cfg.LogsTailURL.String != "" {
		c.LogsTailURL = cfg.LogsTailURL
	}
	if cfg.WebAppURL.Valid {
		c.WebAppURL = cfg.WebAppURL
	}
	if cfg.NoCompress.Valid {
		c.NoCompress = cfg.NoCompress
	}
	if cfg.ProjectID.Valid && cfg.ProjectID.Int64 > 0 {
		c.ProjectID = cfg.ProjectID
	}
	if cfg.MetricPushInterval.Valid {
		c.MetricPushInterval = cfg.MetricPushInterval
	}
	if cfg.MaxMetricSamplesPerPackage.Valid {
		c.MaxMetricSamplesPerPackage = cfg.MaxMetricSamplesPerPackage
	}
	if cfg.AggregationPeriod.Valid {
		c.AggregationPeriod = cfg.AggregationPeriod
	}
	if cfg.AggregationCalcInterval.Valid {
		c.AggregationCalcInterval = cfg.AggregationCalcInterval
	}
	if cfg.AggregationWaitPeriod.Valid {
		c.AggregationWaitPeriod = cfg.AggregationWaitPeriod
	}
	if cfg.AggregationMinSamples.Valid {
		c.AggregationMinSamples = cfg.AggregationMinSamples
	}
	if cfg.AggregationSkipOutlierDetection.Valid {
		c.AggregationSkipOutlierDetection = cfg.AggregationSkipOutlierDetection
	}
	if cfg.AggregationOutlierAlgoThreshold.Valid {
		c.AggregationOutlierAlgoThreshold = cfg.AggregationOutlierAlgoThreshold
	}
	if cfg.AggregationOutlierIqrRadius.Valid {
		c.AggregationOutlierIqrRadius = cfg.AggregationOutlierIqrRadius
	}
	if cfg.AggregationOutlierIqrCoefLower.Valid {
		c.AggregationOutlierIqrCoefLower = cfg.AggregationOutlierIqrCoefLower
	}
	if cfg.AggregationOutlierIqrCoefUpper.Valid {
		c.AggregationOutlierIqrCoefUpper = cfg.AggregationOutlierIqrCoefUpper
	}
	return c
}

// MergeFromExternal merges three fields from json in a loadimpact key of the provided external map
func MergeFromExternal(external map[string]json.RawMessage, conf *Config) error {
	if val, ok := external["loadimpact"]; ok {
		// TODO: Important! Separate configs and fix the whole 2 configs mess!
		tmpConfig := Config{}
		if err := json.Unmarshal(val, &tmpConfig); err != nil {
			return err
		}
		// Only take out the ProjectID, Name and Token from the options.ext.loadimpact map:
		if tmpConfig.ProjectID.Valid {
			conf.ProjectID = tmpConfig.ProjectID
		}
		if tmpConfig.Name.Valid {
			conf.Name = tmpConfig.Name
		}
		if tmpConfig.Token.Valid {
			conf.Token = tmpConfig.Token
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"github.com/matm/go-cloudinary"
	"github.com/outofpluto/goconfig/config"
	"net/url"
	"os"
)

type Config struct {
	CloudinaryURI *url.URL
	MongoURI      *url.URL
}

var service *cloudinary.Service

// LoadConfig parses a config file and sets global settings
// variables to be used at runtime. Note that returning an error
// will cause the application to exit with code error 1.
func LoadConfig(path string) (*Config, error) {
	settings := &Config{}

	c, err := config.ReadDefault(path)
	if err != nil {
		return nil, err
	}

	// Cloudinary settings
	var cURI *url.URL
	var uri string

	if uri, err = c.String("cloudinary", "uri"); err != nil {
		return nil, err
	}
	if cURI, err = url.Parse(uri); err != nil {
		return nil, errors.New(fmt.Sprint("cloudinary URI: ", err.Error()))
	}
	settings.CloudinaryURI = cURI
	return settings, nil
}

func fatal(msg string) {
	fmt.Fprintf(os.Stderr, "Error: %s\n", msg)
	os.Exit(1)
}

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, fmt.Sprintf("Usage: %s [options] settings.conf \n", os.Args[0]))
		fmt.Fprintf(os.Stderr, `
Without any option supplied, it will read the config file and check
ressource (cloudinary, mongodb) availability.

`)
		fmt.Fprintf(os.Stderr, "Options:\n")
		flag.PrintDefaults()
		os.Exit(2)
	}

	uploadPath := flag.String("u", "", "path to the file or directory to upload")
	deleteId := flag.String("d", "", "delete remote file by upload_id")
	dropAll := flag.Bool("dropall", false, "delete all (images and raw) remote files")
	dropAllImages := flag.Bool("dropallimages", false, "delete all remote images files")
	dropAllRaws := flag.Bool("dropallraws", false, "delete all remote raw files")
	flag.Parse()

	if len(flag.Args()) != 1 {
		fmt.Fprint(os.Stderr, "Missing config file\n")
		flag.Usage()
	}

	settings, err := LoadConfig(flag.Arg(0))
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s: %s\n", flag.Arg(0), err.Error())
		os.Exit(1)
	}

	service, err = cloudinary.Dial(settings.CloudinaryURI.String())
	if err != nil {
		fatal(err.Error())
	}

	// Upload file
	if *uploadPath != "" {
		fmt.Println("Uploading ...")
		if err := service.Upload(*uploadPath, false); err != nil {
			fatal(err.Error())
		}
	} else if *deleteId != "" {
		fmt.Printf("Deleting %s ...\n", *deleteId)
		if err := service.Delete(*deleteId, cloudinary.ImageType); err != nil {
			fatal(err.Error())
		}
	} else if *dropAll {
		fmt.Println("Drop all")
		if err := service.DropAll(os.Stdout); err != nil {
			fatal(err.Error())
		}
	} else if *dropAllImages {
		fmt.Println("Drop all images")
		if err := service.DropAllImages(os.Stdout); err != nil {
			fatal(err.Error())
		}
	} else if *dropAllRaws {
		fmt.Println("Drop all raw files")
		if err := service.DropAllRaws(os.Stdout); err != nil {
			fatal(err.Error())
		}
	}
}
<commit_msg>Longer option names<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"github.com/matm/go-cloudinary"
	"github.com/outofpluto/goconfig/config"
	"net/url"
	"os"
)

type Config struct {
	CloudinaryURI *url.URL
	MongoURI      *url.URL
}

var service *cloudinary.Service

// LoadConfig parses a config file and sets global settings
// variables to be used at runtime. Note that returning an error
// will cause the application to exit with code error 1.
func LoadConfig(path string) (*Config, error) {
	settings := &Config{}

	c, err := config.ReadDefault(path)
	if err != nil {
		return nil, err
	}

	// Cloudinary settings
	var cURI *url.URL
	var uri string

	if uri, err = c.String("cloudinary", "uri"); err != nil {
		return nil, err
	}
	if cURI, err = url.Parse(uri); err != nil {
		return nil, errors.New(fmt.Sprint("cloudinary URI: ", err.Error()))
	}
	settings.CloudinaryURI = cURI
	return settings, nil
}

func fatal(msg string) {
	fmt.Fprintf(os.Stderr, "Error: %s\n", msg)
	os.Exit(1)
}

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, fmt.Sprintf("Usage: %s [options] settings.conf \n", os.Args[0]))
		fmt.Fprintf(os.Stderr, `
Without any option supplied, it will read the config file and check
ressource (cloudinary, mongodb) availability.

`)
		fmt.Fprintf(os.Stderr, "Options:\n")
		flag.PrintDefaults()
		os.Exit(2)
	}

	uploadPath := flag.String("upload", "", "path to the file or directory to upload")
	deleteId := flag.String("drop", "", "delete remote file by upload_id")
	dropAll := flag.Bool("dropall", false, "delete all (images and raw) remote files")
	dropAllImages := flag.Bool("dropallimages", false, "delete all remote images files")
	dropAllRaws := flag.Bool("dropallraws", false, "delete all remote raw files")
	flag.Parse()

	if len(flag.Args()) != 1 {
		fmt.Fprint(os.Stderr, "Missing config file\n")
		flag.Usage()
	}

	settings, err := LoadConfig(flag.Arg(0))
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s: %s\n", flag.Arg(0), err.Error())
		os.Exit(1)
	}

	service, err = cloudinary.Dial(settings.CloudinaryURI.String())
	if err != nil {
		fatal(err.Error())
	}

	// Upload file
	if *uploadPath != "" {
		fmt.Println("Uploading ...")
		if err := service.Upload(*uploadPath, false); err != nil {
			fatal(err.Error())
		}
	} else if *deleteId != "" {
		fmt.Printf("Deleting %s ...\n", *deleteId)
		if err := service.Delete(*deleteId, cloudinary.ImageType); err != nil {
			fatal(err.Error())
		}
	} else if *dropAll {
		fmt.Println("Drop all")
		if err := service.DropAll(os.Stdout); err != nil {
			fatal(err.Error())
		}
	} else if *dropAllImages {
		fmt.Println("Drop all images")
		if err := service.DropAllImages(os.Stdout); err != nil {
			fatal(err.Error())
		}
	} else if *dropAllRaws {
		fmt.Println("Drop all raw files")
		if err := service.DropAllRaws(os.Stdout); err != nil {
			fatal(err.Error())
		}
	}
}
<|endoftext|>


<commit_before>package version

// Version contains the LXD version number
var Version = "3.17"
<commit_msg>Release LXD 3.18<commit_after>package version

// Version contains the LXD version number
var Version = "3.18"
<|endoftext|>


<commit_before>package mount // import "github.com/docker/docker/pkg/mount"

// MakeShared ensures a mounted filesystem has the SHARED mount option enabled.
// See the supported options in flags.go for further reference.
func MakeShared(mountPoint string) error {
	return ensureMountedAs(mountPoint, "shared")
}

// MakeRShared ensures a mounted filesystem has the RSHARED mount option enabled.
// See the supported options in flags.go for further reference.
func MakeRShared(mountPoint string) error {
	return ensureMountedAs(mountPoint, "rshared")
}

// MakePrivate ensures a mounted filesystem has the PRIVATE mount option enabled.
// See the supported options in flags.go for further reference.
func MakePrivate(mountPoint string) error {
	return ensureMountedAs(mountPoint, "private")
}

// MakeRPrivate ensures a mounted filesystem has the RPRIVATE mount option
// enabled. See the supported options in flags.go for further reference.
func MakeRPrivate(mountPoint string) error {
	return ensureMountedAs(mountPoint, "rprivate")
}

// MakeSlave ensures a mounted filesystem has the SLAVE mount option enabled.
// See the supported options in flags.go for further reference.
func MakeSlave(mountPoint string) error {
	return ensureMountedAs(mountPoint, "slave")
}

// MakeRSlave ensures a mounted filesystem has the RSLAVE mount option enabled.
// See the supported options in flags.go for further reference.
func MakeRSlave(mountPoint string) error {
	return ensureMountedAs(mountPoint, "rslave")
}

// MakeUnbindable ensures a mounted filesystem has the UNBINDABLE mount option
// enabled. See the supported options in flags.go for further reference.
func MakeUnbindable(mountPoint string) error {
	return ensureMountedAs(mountPoint, "unbindable")
}

// MakeRUnbindable ensures a mounted filesystem has the RUNBINDABLE mount
// option enabled. See the supported options in flags.go for further reference.
func MakeRUnbindable(mountPoint string) error {
	return ensureMountedAs(mountPoint, "runbindable")
}

// MakeMount ensures that the file or directory given is a mount point,
// bind mounting it to itself it case it is not.
func MakeMount(mnt string) error {
	mounted, err := Mounted(mnt)
	if err != nil {
		return err
	}
	if mounted {
		return nil
	}

	return Mount(mnt, mnt, "none", "bind")
}

func ensureMountedAs(mountPoint, options string) error {
	if err := MakeMount(mountPoint); err != nil {
		return err
	}

	return ForceMount("", mountPoint, "none", options)
}
<commit_msg>pkg/mount: MakeMount: minor optimization<commit_after>package mount // import "github.com/docker/docker/pkg/mount"

// MakeShared ensures a mounted filesystem has the SHARED mount option enabled.
// See the supported options in flags.go for further reference.
func MakeShared(mountPoint string) error {
	return ensureMountedAs(mountPoint, "shared")
}

// MakeRShared ensures a mounted filesystem has the RSHARED mount option enabled.
// See the supported options in flags.go for further reference.
func MakeRShared(mountPoint string) error {
	return ensureMountedAs(mountPoint, "rshared")
}

// MakePrivate ensures a mounted filesystem has the PRIVATE mount option enabled.
// See the supported options in flags.go for further reference.
func MakePrivate(mountPoint string) error {
	return ensureMountedAs(mountPoint, "private")
}

// MakeRPrivate ensures a mounted filesystem has the RPRIVATE mount option
// enabled. See the supported options in flags.go for further reference.
func MakeRPrivate(mountPoint string) error {
	return ensureMountedAs(mountPoint, "rprivate")
}

// MakeSlave ensures a mounted filesystem has the SLAVE mount option enabled.
// See the supported options in flags.go for further reference.
func MakeSlave(mountPoint string) error {
	return ensureMountedAs(mountPoint, "slave")
}

// MakeRSlave ensures a mounted filesystem has the RSLAVE mount option enabled.
// See the supported options in flags.go for further reference.
func MakeRSlave(mountPoint string) error {
	return ensureMountedAs(mountPoint, "rslave")
}

// MakeUnbindable ensures a mounted filesystem has the UNBINDABLE mount option
// enabled. See the supported options in flags.go for further reference.
func MakeUnbindable(mountPoint string) error {
	return ensureMountedAs(mountPoint, "unbindable")
}

// MakeRUnbindable ensures a mounted filesystem has the RUNBINDABLE mount
// option enabled. See the supported options in flags.go for further reference.
func MakeRUnbindable(mountPoint string) error {
	return ensureMountedAs(mountPoint, "runbindable")
}

// MakeMount ensures that the file or directory given is a mount point,
// bind mounting it to itself it case it is not.
func MakeMount(mnt string) error {
	mounted, err := Mounted(mnt)
	if err != nil {
		return err
	}
	if mounted {
		return nil
	}

	return ForceMount(mnt, mnt, "none", "bind")
}

func ensureMountedAs(mountPoint, options string) error {
	if err := MakeMount(mountPoint); err != nil {
		return err
	}

	return ForceMount("", mountPoint, "none", options)
}
<|endoftext|>


<commit_before>package build

import (
	"bufio"
	"encoding/binary"
	"fmt"
	"io"
	"math"
	"net"
	"net/http"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"text/template"

	"github.com/codegangsta/cli"
)

type Node struct {
	IP, Mask, Mask2 uint32
}

type Graph []Node

func (g Graph) Len() int           { return len(g) }
func (g Graph) Swap(i, j int)      { g[i], g[j] = g[j], g[i] }
func (g Graph) Less(i, j int) bool { return g[i].IP < g[j].IP }

var (
	F      uint32 = 0xffffffff
	EOL           = byte('\n')
	URL           = "http://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-latest"
	CNIPV4        = regexp.MustCompile(`apnic\|(CN|cn)\|ipv4\|([0-9\.]+)\|([0-9]+)\|([0-9]+)\|a.*`)
)

func ip2long(ipstr string) uint32 {
	ip := net.ParseIP(ipstr)
	if ip == nil {
		return 0
	}
	ip = ip.To4()
	return binary.BigEndian.Uint32(ip)
}

func long2ip(ipLong uint32) string {
	ipByte := make([]byte, 4)
	binary.BigEndian.PutUint32(ipByte, ipLong)
	ip := net.IP(ipByte)
	return ip.String()
}

func fetchIPData(results *Graph) (err error) {
	var (
		n   int
		buf string
		r   *bufio.Reader

		startIP, prevIP, smask string
		numIP                  int
		imask                  uint32
	)

	fmt.Println("Fetching data from apnic.net, it might take a few minutes, please wait...")

	res, err := http.Get(URL)
	if err != nil {
		panic(err)
	}
	r = bufio.NewReader(res.Body)
	defer res.Body.Close()
	for {
		// read line by line
		buf, err = r.ReadString(EOL)
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		matches := CNIPV4.FindStringSubmatch(buf)
		if len(matches) > 0 {
			n++
			startIP = matches[2]
			numIP, _ = strconv.Atoi(matches[3])

			imask = F ^ uint32(numIP-1)
			smask = fmt.Sprintf("%02x", imask)
			mask := [4]string{}
			mask[0] = smask[0:2]
			mask[1] = smask[2:4]
			mask[2] = "0"
			mask[3] = "0"

			for i, s := range mask[:2] {
				num, _ := strconv.ParseInt(s, 16, 10)
				mask[i] = fmt.Sprintf("%d", num)
			}

			mask2 := 32 - uint32(math.Log2(float64(numIP)))

			ip := strings.Split(startIP, ".")
			ip[2] = "0"
			ip[3] = "0"
			startIP = strings.Join(ip, ".")

			maskIP := fmt.Sprintf("%s.%s.%s.%s", mask[0], mask[1], mask[2], mask[3])

			if startIP != prevIP {
				*results = append(*results, Node{ip2long(startIP), ip2long(maskIP), mask2})
				prevIP = startIP
			}
		}
	}
	return nil
}

func Action(c *cli.Context) {
	var pacfile = "go.pac"
	var results = make(Graph, 0)
	results = append(results, Node{ip2long("127.0.0.1"), ip2long("255.0.0.0"), 0})
	results = append(results, Node{ip2long("10.0.0.0"), ip2long("255.0.0.0"), 0})
	results = append(results, Node{ip2long("127.0.0.1"), ip2long("255.240.0.0"), 0})
	results = append(results, Node{ip2long("192.168.0.0"), ip2long("255.255.0.0"), 0})
	fetchIPData(&results)
	sort.Sort(results)

	file, err := os.Create(pacfile)
	if err != nil {
		panic(err)
	}

	t, err := template.ParseFiles("templates/pac.tmpl")
	data := make(map[string]interface{}, 0)
	data["Graph"] = results
	data["Proxy"] = c.String("proxy")
	err = t.Execute(file, data)
	if err != nil {
		panic(err)
	}

	fmt.Printf("Rules: %d items.\n", len(results))
	fmt.Printf("Usage: Use the newly created %s as your web browser's automatic \n", pacfile)
	fmt.Printf("PAC(Proxy auto-config) file.\n")
}
<commit_msg>update<commit_after>package build

import (
	"bufio"
	"fmt"
	"io"
	"math"
	"net/http"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"text/template"

	"github.com/codegangsta/cli"
	"github.com/go-libs/iputils"
)

type Node struct {
	IP, Mask, Mask2 uint32
}

type Graph []Node

func (g Graph) Len() int           { return len(g) }
func (g Graph) Swap(i, j int)      { g[i], g[j] = g[j], g[i] }
func (g Graph) Less(i, j int) bool { return g[i].IP < g[j].IP }

var (
	F      uint32 = 0xffffffff
	EOL           = byte('\n')
	URL           = "http://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-latest"
	CNIPV4        = regexp.MustCompile(`apnic\|(CN|cn)\|ipv4\|([0-9\.]+)\|([0-9]+)\|([0-9]+)\|a.*`)
)

func fetchIPData(results *Graph) (err error) {
	var (
		n   int
		buf string
		r   *bufio.Reader

		startIP, prevIP, smask string
		numIP                  int
		imask                  uint32
	)

	fmt.Println("Fetching data from apnic.net, it might take a few minutes, please wait...")

	res, err := http.Get(URL)
	if err != nil {
		panic(err)
	}
	r = bufio.NewReader(res.Body)
	defer res.Body.Close()
	for {
		// read line by line
		buf, err = r.ReadString(EOL)
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		matches := CNIPV4.FindStringSubmatch(buf)
		if len(matches) > 0 {
			n++
			startIP = matches[2]
			numIP, _ = strconv.Atoi(matches[3])

			imask = F ^ uint32(numIP-1)
			smask = fmt.Sprintf("%02x", imask)
			mask := [4]string{}
			mask[0] = smask[0:2]
			mask[1] = smask[2:4]
			mask[2] = "0"
			mask[3] = "0"

			for i, s := range mask[:2] {
				num, _ := strconv.ParseInt(s, 16, 10)
				mask[i] = fmt.Sprintf("%d", num)
			}

			mask2 := 32 - uint32(math.Log2(float64(numIP)))

			ip := strings.Split(startIP, ".")
			ip[2] = "0"
			ip[3] = "0"
			startIP = strings.Join(ip, ".")

			maskIP := fmt.Sprintf("%s.%s.%s.%s", mask[0], mask[1], mask[2], mask[3])

			if startIP != prevIP {
				*results = append(*results, Node{iputils.IP2Long(startIP), iputils.IP2Long(maskIP), mask2})
				prevIP = startIP
			}
		}
	}
	return nil
}

func Action(c *cli.Context) {
	var pacfile = "go.pac"
	var results = make(Graph, 0)
	results = append(results, Node{iputils.IP2Long("127.0.0.1"), iputils.IP2Long("255.0.0.0"), 0})
	results = append(results, Node{iputils.IP2Long("10.0.0.0"), iputils.IP2Long("255.0.0.0"), 0})
	results = append(results, Node{iputils.IP2Long("127.0.0.1"), iputils.IP2Long("255.240.0.0"), 0})
	results = append(results, Node{iputils.IP2Long("192.168.0.0"), iputils.IP2Long("255.255.0.0"), 0})
	fetchIPData(&results)
	sort.Sort(results)

	file, err := os.Create(pacfile)
	if err != nil {
		panic(err)
	}

	t, err := template.ParseFiles("templates/pac.tmpl")
	data := make(map[string]interface{}, 0)
	data["Graph"] = results
	data["Proxy"] = c.String("proxy")
	err = t.Execute(file, data)
	if err != nil {
		panic(err)
	}

	fmt.Printf("Rules: %d items.\n", len(results))
	fmt.Printf("Usage: Use the newly created %s as your web browser's automatic \n", pacfile)
	fmt.Printf("PAC(Proxy auto-config) file.\n")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"

	"gopkg.in/alecthomas/kingpin.v2"
)

// #cgo windows LDFLAGS: -Wl,--allow-multiple-definition -static
import "C"

var (
	version = "head" // set by command-line on CI release builds
	app     = kingpin.New("butler", "Your very own itch.io helper")

	dlCmd = app.Command("dl", "Download a file (resumes if can, checks hashes)")

	pushCmd = app.Command("push", "Upload a new version of something to itch.io")
)

var appArgs = struct {
	json       *bool
	quiet      *bool
	timestamps *bool
}{
	app.Flag("json", "Enable machine-readable JSON-lines output").Short('j').Bool(),
	app.Flag("quiet", "Hide progress indicators & other extra info").Short('q').Bool(),
	app.Flag("timestamps", "Prefix all output by timestamps (for logging purposes)").Bool(),
}

var dlArgs = struct {
	url  *string
	dest *string
}{
	dlCmd.Arg("url", "Address to download from").Required().String(),
	dlCmd.Arg("dest", "File to write downloaded data to").Required().String(),
}

var pushArgs = struct {
	identity *string
	address  *string
	src      *string
	repo     *string
}{
	pushCmd.Flag("identity", "Path to the private key used for public key authentication.").Default(fmt.Sprintf("%s/%s", os.Getenv("HOME"), ".ssh/id_rsa")).Short('i').ExistingFile(),
	pushCmd.Flag("address", "Specify wharf address (advanced)").Default("wharf.itch.zone").Short('a').Hidden().String(),
	pushCmd.Arg("src", "Directory or zip archive to upload, e.g.").Required().ExistingFileOrDir(),
	pushCmd.Arg("repo", "Repository to push to, e.g. leafo/xmoon:win64").Required().String(),
}

func main() {
	log.Println("branch cita")
	app.HelpFlag.Short('h')
	app.Version(fmt.Sprintf("that log is going to be rubbish... %s", version))
	app.VersionFlag.Short('V')

	cmd, err := app.Parse(os.Args[1:])
	if *appArgs.timestamps {
		log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)
	} else {
		log.SetFlags(0)
	}

	switch kingpin.MustParse(cmd, err) {
	case dlCmd.FullCommand():
		dl(*dlArgs.url, *dlArgs.dest)

	case pushCmd.FullCommand():
		push(*pushArgs.src, *pushArgs.repo)
	}
}
<commit_msg>pushing a branch now<commit_after>package main

import (
	"fmt"
	"log"
	"os"

	"gopkg.in/alecthomas/kingpin.v2"
)

// #cgo windows LDFLAGS: -Wl,--allow-multiple-definition -static
import "C"

var (
	version = "head" // set by command-line on CI release builds
	app     = kingpin.New("butler", "Your very own itch.io helper")

	dlCmd = app.Command("dl", "Download a file (resumes if can, checks hashes)")

	pushCmd = app.Command("push", "Upload a new version of something to itch.io")
)

var appArgs = struct {
	json       *bool
	quiet      *bool
	timestamps *bool
}{
	app.Flag("json", "Enable machine-readable JSON-lines output").Short('j').Bool(),
	app.Flag("quiet", "Hide progress indicators & other extra info").Short('q').Bool(),
	app.Flag("timestamps", "Prefix all output by timestamps (for logging purposes)").Bool(),
}

var dlArgs = struct {
	url  *string
	dest *string
}{
	dlCmd.Arg("url", "Address to download from").Required().String(),
	dlCmd.Arg("dest", "File to write downloaded data to").Required().String(),
}

var pushArgs = struct {
	identity *string
	address  *string
	src      *string
	repo     *string
}{
	pushCmd.Flag("identity", "Path to the private key used for public key authentication.").Default(fmt.Sprintf("%s/%s", os.Getenv("HOME"), ".ssh/id_rsa")).Short('i').ExistingFile(),
	pushCmd.Flag("address", "Specify wharf address (advanced)").Default("wharf.itch.zone").Short('a').Hidden().String(),
	pushCmd.Arg("src", "Directory or zip archive to upload, e.g.").Required().ExistingFileOrDir(),
	pushCmd.Arg("repo", "Repository to push to, e.g. leafo/xmoon:win64").Required().String(),
}

func main() {
	app.HelpFlag.Short('h')
	app.Version(fmt.Sprintf("ah yes the cita branch... %s", version))
	app.VersionFlag.Short('V')

	cmd, err := app.Parse(os.Args[1:])
	if *appArgs.timestamps {
		log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)
	} else {
		log.SetFlags(0)
	}

	switch kingpin.MustParse(cmd, err) {
	case dlCmd.FullCommand():
		dl(*dlArgs.url, *dlArgs.dest)

	case pushCmd.FullCommand():
		push(*pushArgs.src, *pushArgs.repo)
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"

	"gopkg.in/mgo.v2"

	"github.com/juju/charmstore/config"
	"github.com/juju/charmstore/internal/charmstore"
	"github.com/juju/charmstore/internal/elasticsearch"
	"github.com/juju/loggo"
	"gopkg.in/errgo.v1"
)

var logger = loggo.GetLogger("essync")

var (
	index         = flag.String("index", "charmstore", "Name of index to populate.")
	loggingConfig = flag.String("logging-config", "", "specify log levels for modules e.g. <root>=TRACE")
)

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "usage: %s <config path>\n", filepath.Base(os.Args[0]))
		flag.PrintDefaults()
		os.Exit(2)
	}
	flag.Parse()
	if flag.NArg() != 1 {
		flag.Usage()
	}
	if *loggingConfig != "" {
		if err := loggo.ConfigureLoggers(*loggingConfig); err != nil {
			return errgo.Notef(err, "cannot configure loggers")
		}
	}
	if err := populate(flag.Arg(0)); err != nil {
		logger.Errorf("cannot populate elasticsearch: %v", err)
		os.Exit(1)
	}
}

func populate(confPath string) error {
	conf, err := config.Read(confPath)
	if err != nil {
		return errgo.Notef(err, "cannot read config file %q", confPath)
	}
	if conf.ESAddr == "" {
		return errgo.Newf("no elasticsearch-addr specified in config file %q", confPath)
	}
	es := &elasticsearch.Database{conf.ESAddr}

	logger.Infof("config: %#v", conf)

	session, err := mgo.Dial(conf.MongoURL)
	if err != nil {
		return errgo.Notef(err, "cannot dial mongo at %q", conf.MongoURL)
	}
	defer session.Close()
	db := session.DB("juju")
	store, err := charmstore.NewStore(db, &charmstore.StoreElasticSearch{es, *index})
	if err != nil {
		return errgo.Notef(err, "unable to create store for ESSync")
	}
	logger.Debugf("starting export to Elastic Search")
	return store.ExportToElasticSearch()
}
<commit_msg>errgo.v1 in right place, modify the cannot configure logger<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"

	"gopkg.in/errgo.v1"
	"gopkg.in/mgo.v2"

	"github.com/juju/charmstore/config"
	"github.com/juju/charmstore/internal/charmstore"
	"github.com/juju/charmstore/internal/elasticsearch"
	"github.com/juju/loggo"
)

var logger = loggo.GetLogger("essync")

var (
	index         = flag.String("index", "charmstore", "Name of index to populate.")
	loggingConfig = flag.String("logging-config", "", "specify log levels for modules e.g. <root>=TRACE")
)

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "usage: %s <config path>\n", filepath.Base(os.Args[0]))
		flag.PrintDefaults()
		os.Exit(2)
	}
	flag.Parse()
	if flag.NArg() != 1 {
		flag.Usage()
	}
	if *loggingConfig != "" {
		if err := loggo.ConfigureLoggers(*loggingConfig); err != nil {
			fmt.Fprintf(os.Stderr, "cannot configure loggers: %v", err)
			os.Exit(1)
		}
	}
	if err := populate(flag.Arg(0)); err != nil {
		logger.Errorf("cannot populate elasticsearch: %v", err)
		os.Exit(1)
	}
}

func populate(confPath string) error {
	logger.Debugf("reading config file %q", confPath)
	conf, err := config.Read(confPath)
	if err != nil {
		return errgo.Notef(err, "cannot read config file %q", confPath)
	}
	if conf.ESAddr == "" {
		return errgo.Newf("no elasticsearch-addr specified in config file %q", confPath)
	}
	es := &elasticsearch.Database{conf.ESAddr}
	session, err := mgo.Dial(conf.MongoURL)
	if err != nil {
		return errgo.Notef(err, "cannot dial mongo at %q", conf.MongoURL)
	}
	defer session.Close()
	db := session.DB("juju")
	store, err := charmstore.NewStore(db, &charmstore.StoreElasticSearch{es, *index})
	if err != nil {
		return errgo.Notef(err, "unable to create store for ESSync")
	}
	logger.Debugf("starting export to Elastic Search")
	return store.ExportToElasticSearch()
}
<|endoftext|>