<commit_before>// Copyright (c) 2014 The SurgeMQ Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"fmt"
	"io"
	"net"
	"time"

	"github.com/surgemq/message"
	"runtime"
	"cmd/internal/obj"
)

type netReader interface {
	io.Reader
	SetReadDeadline(t time.Time) error
}

type timeoutReader struct {
	d    time.Duration
	conn netReader
}

func (r timeoutReader) Read(b []byte) (int, error) {
	if err := r.conn.SetReadDeadline(time.Now().Add(r.d)); err != nil {
		return 0, err
	}
	return r.conn.Read(b)
}

// receiver() reads data from the network, and writes the data into the incoming buffer
func (this *service) receiver() {
	defer func() {
		// Let's recover from panic
		if r := recover(); r != nil {
			Log.Errorc(func() string {
				return fmt.Sprintf("(%s) Recovering from panic(receiver): %v", this.cid(), r)
			})
		}

		this.wgStopped.Done()

		Log.Debugc(func() string {
			return fmt.Sprintf("(%s) Stopping receiver", this.cid())
		})
	}()

	//   Log.Debugc(func() string{ return fmt.Sprintf("(%s) Starting receiver", this.cid())})

	this.wgStarted.Done()

	switch conn := this.conn.(type) {
	case net.Conn:
		//Log.Debugc(func() string{ return fmt.Sprintf("server/handleConnection: Setting read deadline to %d", time.Second*time.Duration(this.keepAlive))})
		keepAlive := time.Second * time.Duration(this.keepAlive)
		r := timeoutReader{
			d:    keepAlive + (keepAlive / 2),
			conn: conn,
		}

		for {
			_, err := this.in.ReadFrom(r)
			//       Log.Errorc(func() string{ return fmt.Sprintf("this.sess is: %v", this.sess)})
			//       Log.Errorc(func() string{ return fmt.Sprintf("this.sessMgr is: %v", this.sessMgr)})

			if err != nil {
				Log.Infoc(func() string {
					return fmt.Sprintf("(%s) error reading from connection: %v", this.cid(), err)
				})
				//         if err != io.EOF {
				//         }
				return
			}
		}

	//case *websocket.Conn:
	//	Log.Errorc(func() string{ return fmt.Sprintf("(%s) Websocket: %v", this.cid(), ErrInvalidConnectionType)})

	default:
		Log.Errorc(func() string {
			return fmt.Sprintf("(%s) %v", this.cid(), ErrInvalidConnectionType)
		})
	}
}

// sender() writes data from the outgoing buffer to the network
func (this *service) sender() {
	defer func() {
		// Let's recover from panic
		if r := recover(); r != nil {
			Log.Errorc(func() string {
				return fmt.Sprintf("(%s) Recovering from panic(sender): %v", this.cid(), r)
			})
		}

		this.wgStopped.Done()

		Log.Debugc(func() string {
			return fmt.Sprintf("(%s) Stopping sender", this.cid())
		})
	}()

	//Log.Debugc(func() string {
	//	return fmt.Sprintf("(%s) Starting sender", this.cid())
	//})

	this.wgStarted.Done()
	//Log.Debugc(func() string {
	//	return fmt.Sprintf("sender_1(%s)", this.cid())
	//})
	switch conn := this.conn.(type) {
	case net.Conn:
		for {
			//Log.Debugc(func() string {
			//	return fmt.Sprintf("sender_2(%s)", this.cid())
			//})
			_, err := this.out.WriteTo(conn)
			Log.Debugc(func() string {
				return fmt.Sprintf("sender_3(%s)", this.cid())
			})
			if err != nil {
				Log.Debugc(func() string {
					return fmt.Sprintf("sender_4(%s)", this.cid())
				})
				if err != io.EOF {
					Log.Errorc(func() string {
						return fmt.Sprintf("(%s) error writing data: %v", this.cid(), err)
					})
				}
				return
			}else {
				Log.Debugc(func() string {
					return fmt.Sprintf("sender_5(%s)", this.cid())
				})
				Log.Infoc(func() string {
					return fmt.Sprintf("向conn些数据成功！")
				})
			}
		}

	//case *websocket.Conn:
	//	Log.Errorc(func() string{ return fmt.Sprintf("(%s) Websocket not supported", this.cid())})

	default:
		Log.Errorc(func() string {
			return fmt.Sprintf("(%s) Invalid connection type", this.cid())
		})
	}
	Log.Debugc(func() string {
		return fmt.Sprintf("sender_6(%s)", this.cid())
	})
}

// readMessage() reads and copies a message from the buffer. The buffer bytes are
// committed as a result of the read.
func (this *service) readMessage(mtype message.MessageType, total int) (message.Message, error) {
	var (
		err error
		msg message.Message
	)

	if this.in == nil {
		err = ErrBufferNotReady
		return nil, err
	}

	var b []byte
	var index int64
	var ok bool

	for i := 0; i < 99; i++ {
		b, index, ok = this.in.ReadBuffer()
		if ok {
			break
		}
		runtime.Gosched()
	}
	defer this.in.ReadCommit(index)
	msg, err = mtype.New()
	if err != nil {
		Log.Errorc(func() string {
			return fmt.Sprintf("(%s)NewMessage  Error processing: %v", this.cid(), err)
		})
		return nil, err
	}

	_, err = msg.Decode(b)
	if err != nil {
		Log.Errorc(func() string {
			return fmt.Sprintf("(%s) Decode Error processing: %v", this.cid(), err)
		})
		return nil, err
	}

	return msg, err
}

// writeMessage() writes a message to the outgoing buffer
func (this *service) writeMessage(msg message.Message) (error) {
	if this.out == nil {
		return ErrBufferNotReady
	}

	// This is to serialize writes to the underlying buffer. Multiple goroutines could
	// potentially get here because of calling Publish() or Subscribe() or other
	// functions that will send messages. For example, if a message is received in
	// another connetion, and the message needs to be published to this client, then
	// the Publish() function is called, and at the same time, another client could
	// do exactly the same thing.
	//
	// Not an ideal fix though. If possible we should remove mutex and be lockfree.
	// Mainly because when there's a large number of goroutines that want to publish
	// to this client, then they will all block. However, this will do for now.
	//
	// FIXME: Try to find a better way than a mutex...if possible.
	b := make([]byte, msg.Len())
	_, err := msg.Encode(b)
	if err != nil {
		return err
	}

	for i := 0; i < 100; i++ {
		if this.out.WriteBuffer(b); {
			break
		}
		runtime.Gosched()
	}

	this.outStat.increment(int64(1))

	return nil
}
<commit_msg>修改buffer读写限制<commit_after>// Copyright (c) 2014 The SurgeMQ Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"fmt"
	"io"
	"net"
	"time"

	"github.com/surgemq/message"
	"runtime"
)

type netReader interface {
	io.Reader
	SetReadDeadline(t time.Time) error
}

type timeoutReader struct {
	d    time.Duration
	conn netReader
}

func (r timeoutReader) Read(b []byte) (int, error) {
	if err := r.conn.SetReadDeadline(time.Now().Add(r.d)); err != nil {
		return 0, err
	}
	return r.conn.Read(b)
}

// receiver() reads data from the network, and writes the data into the incoming buffer
func (this *service) receiver() {
	defer func() {
		// Let's recover from panic
		if r := recover(); r != nil {
			Log.Errorc(func() string {
				return fmt.Sprintf("(%s) Recovering from panic(receiver): %v", this.cid(), r)
			})
		}

		this.wgStopped.Done()

		Log.Debugc(func() string {
			return fmt.Sprintf("(%s) Stopping receiver", this.cid())
		})
	}()

	//   Log.Debugc(func() string{ return fmt.Sprintf("(%s) Starting receiver", this.cid())})

	this.wgStarted.Done()

	switch conn := this.conn.(type) {
	case net.Conn:
		//Log.Debugc(func() string{ return fmt.Sprintf("server/handleConnection: Setting read deadline to %d", time.Second*time.Duration(this.keepAlive))})
		keepAlive := time.Second * time.Duration(this.keepAlive)
		r := timeoutReader{
			d:    keepAlive + (keepAlive / 2),
			conn: conn,
		}

		for {
			_, err := this.in.ReadFrom(r)
			//       Log.Errorc(func() string{ return fmt.Sprintf("this.sess is: %v", this.sess)})
			//       Log.Errorc(func() string{ return fmt.Sprintf("this.sessMgr is: %v", this.sessMgr)})

			if err != nil {
				Log.Infoc(func() string {
					return fmt.Sprintf("(%s) error reading from connection: %v", this.cid(), err)
				})
				//         if err != io.EOF {
				//         }
				return
			}
		}

	//case *websocket.Conn:
	//	Log.Errorc(func() string{ return fmt.Sprintf("(%s) Websocket: %v", this.cid(), ErrInvalidConnectionType)})

	default:
		Log.Errorc(func() string {
			return fmt.Sprintf("(%s) %v", this.cid(), ErrInvalidConnectionType)
		})
	}
}

// sender() writes data from the outgoing buffer to the network
func (this *service) sender() {
	defer func() {
		// Let's recover from panic
		if r := recover(); r != nil {
			Log.Errorc(func() string {
				return fmt.Sprintf("(%s) Recovering from panic(sender): %v", this.cid(), r)
			})
		}

		this.wgStopped.Done()

		Log.Debugc(func() string {
			return fmt.Sprintf("(%s) Stopping sender", this.cid())
		})
	}()

	//Log.Debugc(func() string {
	//	return fmt.Sprintf("(%s) Starting sender", this.cid())
	//})

	this.wgStarted.Done()
	//Log.Debugc(func() string {
	//	return fmt.Sprintf("sender_1(%s)", this.cid())
	//})
	switch conn := this.conn.(type) {
	case net.Conn:
		for {
			//Log.Debugc(func() string {
			//	return fmt.Sprintf("sender_2(%s)", this.cid())
			//})
			_, err := this.out.WriteTo(conn)
			Log.Debugc(func() string {
				return fmt.Sprintf("sender_3(%s)", this.cid())
			})
			if err != nil {
				Log.Debugc(func() string {
					return fmt.Sprintf("sender_4(%s)", this.cid())
				})
				if err != io.EOF {
					Log.Errorc(func() string {
						return fmt.Sprintf("(%s) error writing data: %v", this.cid(), err)
					})
				}
				return
			}else {
				Log.Debugc(func() string {
					return fmt.Sprintf("sender_5(%s)", this.cid())
				})
				Log.Infoc(func() string {
					return fmt.Sprintf("向conn些数据成功！")
				})
			}
		}

	//case *websocket.Conn:
	//	Log.Errorc(func() string{ return fmt.Sprintf("(%s) Websocket not supported", this.cid())})

	default:
		Log.Errorc(func() string {
			return fmt.Sprintf("(%s) Invalid connection type", this.cid())
		})
	}
	Log.Debugc(func() string {
		return fmt.Sprintf("sender_6(%s)", this.cid())
	})
}

// readMessage() reads and copies a message from the buffer. The buffer bytes are
// committed as a result of the read.
func (this *service) readMessage(mtype message.MessageType, total int) (message.Message, error) {
	var (
		err error
		msg message.Message
	)

	if this.in == nil {
		err = ErrBufferNotReady
		return nil, err
	}

	var b []byte
	var index int64
	var ok bool

	for i := 0; i < 99; i++ {
		b, index, ok = this.in.ReadBuffer()
		if ok {
			break
		}
		runtime.Gosched()
	}
	defer this.in.ReadCommit(index)
	msg, err = mtype.New()
	if err != nil {
		Log.Errorc(func() string {
			return fmt.Sprintf("(%s)NewMessage  Error processing: %v", this.cid(), err)
		})
		return nil, err
	}

	_, err = msg.Decode(b)
	if err != nil {
		Log.Errorc(func() string {
			return fmt.Sprintf("(%s) Decode Error processing: %v", this.cid(), err)
		})
		return nil, err
	}

	return msg, err
}

// writeMessage() writes a message to the outgoing buffer
func (this *service) writeMessage(msg message.Message) (error) {
	if this.out == nil {
		return ErrBufferNotReady
	}

	// This is to serialize writes to the underlying buffer. Multiple goroutines could
	// potentially get here because of calling Publish() or Subscribe() or other
	// functions that will send messages. For example, if a message is received in
	// another connetion, and the message needs to be published to this client, then
	// the Publish() function is called, and at the same time, another client could
	// do exactly the same thing.
	//
	// Not an ideal fix though. If possible we should remove mutex and be lockfree.
	// Mainly because when there's a large number of goroutines that want to publish
	// to this client, then they will all block. However, this will do for now.
	//
	// FIXME: Try to find a better way than a mutex...if possible.
	b := make([]byte, msg.Len())
	_, err := msg.Encode(b)
	if err != nil {
		return err
	}

	for i := 0; i < 100; i++ {
		if this.out.WriteBuffer(b); {
			break
		}
		runtime.Gosched()
	}

	this.outStat.increment(int64(1))

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"strings"
	"time"

	mp "github.com/mackerelio/go-mackerel-plugin"
)

var graphDef = map[string](mp.Graphs){
	"multicore.cpu.#": mp.Graphs{
		Label: "MultiCore CPU",
		Unit:  "percentage",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "user", Label: "user", Diff: false, Stacked: true},
			mp.Metrics{Name: "nice", Label: "nice", Diff: false, Stacked: true},
			mp.Metrics{Name: "system", Label: "system", Diff: false, Stacked: true},
			mp.Metrics{Name: "idle", Label: "idle", Diff: false, Stacked: true},
			mp.Metrics{Name: "iowait", Label: "ioWait", Diff: false, Stacked: true},
			mp.Metrics{Name: "irq", Label: "irq", Diff: false, Stacked: true},
			mp.Metrics{Name: "softirq", Label: "softirq", Diff: false, Stacked: true},
			mp.Metrics{Name: "steal", Label: "steal", Diff: false, Stacked: true},
			mp.Metrics{Name: "guest", Label: "guest", Diff: false, Stacked: true},
		},
	},
	"multicore.loadavg_per_core": mp.Graphs{
		Label: "MultiCore loadavg5 per core",
		Unit:  "float",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "loadavg5", Label: "loadavg5", Diff: false, Stacked: false},
		},
	},
}

type saveItem struct {
	LastTime       time.Time
	ProcStatsByCPU map[string]*procStats
}

type procStats struct {
	User    float64 `json:"user"`
	Nice    float64 `json:"nice"`
	System  float64 `json:"system"`
	Idle    float64 `json:"idle"`
	IoWait  float64 `json:"iowait"`
	Irq     float64 `json:"irq"`
	SoftIrq float64 `json:"softirq"`
	Steal   float64 `json:"steal"`
	Guest   float64 `json:"guest"`
	Total   float64 `json:"total"`
}

type cpuPercentages struct {
	GroupName string
	User      float64
	Nice      float64
	System    float64
	Idle      float64
	IoWait    float64
	Irq       float64
	SoftIrq   float64
	Steal     float64
	Guest     float64
}

func getProcStat() (string, error) {
	contentbytes, err := ioutil.ReadFile("/proc/stat")
	if err != nil {
		return "", err
	}
	return string(contentbytes), nil
}

func parseFloats(values []string) ([]float64, error) {
	var result []float64
	for _, v := range values {
		f, err := strconv.ParseFloat(v, 64)
		if err != nil {
			return nil, err
		}
		result = append(result, f)
	}
	return result, nil
}

func fill(arr []float64, elementCount int) []float64 {
	if len(arr) < elementCount {
		zeroArr := make([]float64, elementCount - len(arr))
		filled := append(arr, zeroArr...)
		return filled
	}
	return arr
}

func parseProcStat(str string) (map[string]*procStats, error) {
	var result = make(map[string]*procStats)
	for _, line := range strings.Split(str, "\n") {
		if strings.HasPrefix(line, "cpu") {
			fields := strings.Fields(line)
			key := fields[0]
			values := fields[1:]

			floatValues, err := parseFloats(values)
			if err != nil {
				return nil, err
			}
			filledValues := fill(floatValues, 9)

			total := 0.0
			for _, v := range floatValues {
				total += v
			}

			ps := &procStats{
				User:    filledValues[0],
				Nice:    filledValues[1],
				System:  filledValues[2],
				Idle:    filledValues[3],
				IoWait:  filledValues[4],
				Irq:     filledValues[5],
				SoftIrq: filledValues[6],
				Steal:   filledValues[7],
				Guest:   filledValues[8],
				Total:   total,
			}
			result[key] = ps

		} else {
			break
		}
	}
	return result, nil
}

func collectProcStatValues() (map[string]*procStats, error) {
	procStats, err := getProcStat()
	if err != nil {
		return nil, err
	}
	return parseProcStat(procStats)
}

func saveValues(tempFileName string, values map[string]*procStats, now time.Time) error {
	f, err := os.Create(tempFileName)
	if err != nil {
		return err
	}
	defer f.Close()

	s := saveItem{
		LastTime:       time.Now(),
		ProcStatsByCPU: values,
	}

	encoder := json.NewEncoder(f)
	err = encoder.Encode(s)
	if err != nil {
		return err
	}

	return nil
}

func fetchLastValues(tempFileName string) (map[string]*procStats, time.Time, error) {
	f, err := os.Open(tempFileName)
	if err != nil {
		return nil, time.Now(), err
	}
	defer f.Close()

	var stat saveItem
	decoder := json.NewDecoder(f)
	err = decoder.Decode(&stat)
	if err != nil {
		return stat.ProcStatsByCPU, stat.LastTime, err
	}
	return stat.ProcStatsByCPU, stat.LastTime, nil
}

func calcCPUUsage(currentValues map[string]*procStats, now time.Time, lastValues map[string]*procStats, lastTime time.Time) ([]*cpuPercentages, error) {

	var result []*cpuPercentages
	for key, current := range currentValues {
		last, ok := lastValues[key]
		if ok {
			user, err := calcPercentage(current.User, last.User, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			nice, err := calcPercentage(current.Nice, last.Nice, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			system, err := calcPercentage(current.System, last.System, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			idle, err := calcPercentage(current.Idle, last.Idle, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			iowait, err := calcPercentage(current.IoWait, last.IoWait, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			irq, err := calcPercentage(current.Irq, last.Irq, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			softirq, err := calcPercentage(current.SoftIrq, last.SoftIrq, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			steal, err := calcPercentage(current.Steal, last.Steal, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			guest, err := calcPercentage(current.Guest, last.Guest, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}

			p := &cpuPercentages{
				GroupName: key,
				User:      user,
				Nice:      nice,
				System:    system,
				Idle:      idle,
				IoWait:    iowait,
				Irq:       irq,
				SoftIrq:   softirq,
				Steal:     steal,
				Guest:     guest,
			}
			result = append(result, p)
		}

	}

	return result, nil
}

func calcPercentage(currentValue float64, lastValue float64, currentTotal float64, lastTotal float64, now time.Time, lastTime time.Time) (float64, error) {
	value, err := calcDiff(currentValue, now, lastValue, lastTime)
	if err != nil {
		return 0.0, err
	}

	total, err := calcDiff(currentTotal, now, lastTotal, lastTime)
	if err != nil {
		return 0.0, err
	}

	return (value / total * 100.0), nil
}

func calcDiff(value float64, now time.Time, lastValue float64, lastTime time.Time) (float64, error) {
	diffTime := now.Unix() - lastTime.Unix()
	if diffTime > 600 {
		return 0.0, errors.New("Too long duration")
	}

	diff := (value - lastValue) * 60 / float64(diffTime)

	if lastValue <= value {
		return diff, nil
	}
	return 0.0, nil
}

func fetchLoadavg5() (float64, error) {
	contentbytes, err := ioutil.ReadFile("/proc/loadavg")
	if err != nil {
		return 0.0, err
	}
	content := string(contentbytes)
	cols := strings.Fields(content)

	if len(cols) > 2 {
		f, err := strconv.ParseFloat(cols[1], 64)
		if err != nil {
			return 0.0, err
		}
		return f, nil
	}
	return 0.0, errors.New("cannot fetch loadavg5.")
}

func printValue(key string, value float64, time time.Time) {
	fmt.Printf("%s\t%f\t%d\n", key, value, time.Unix())
}

func outputCPUUsage(cpuUsage []*cpuPercentages, now time.Time) {
	if cpuUsage != nil {
		for _, u := range cpuUsage {
			if u.GroupName != "cpu" {
				printValue(fmt.Sprintf("multicore.cpu.%s.user", u.GroupName), u.User, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.nice", u.GroupName), u.Nice, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.system", u.GroupName), u.System, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.idle", u.GroupName), u.Idle, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.iowait", u.GroupName), u.IoWait, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.irq", u.GroupName), u.Irq, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.softirq", u.GroupName), u.SoftIrq, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.steal", u.GroupName), u.Steal, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.guest", u.GroupName), u.Guest, now)
			}
		}
	}
}

func outputLoadavgPerCore(loadavgPerCore float64, now time.Time) {
	printValue("multicore.loadavg_per_core.loadavg5", loadavgPerCore, now)
}

func outputDefinitions() {
	fmt.Println("# mackerel-agent-plugin")
	var graphs mp.GraphDef
	graphs.Graphs = graphDef

	b, err := json.Marshal(graphs)
	if err != nil {
		log.Fatalln("OutputDefinitions: ", err)
	}
	fmt.Println(string(b))
}

// main function
func main() {
	optTempfile := flag.String("tempfile", "", "Temp file name")
	var tempFileName string
	if *optTempfile != "" {
		tempFileName = *optTempfile
	} else {
		tempFileName = "/tmp/mackerel-plugin-multicore"
	}
	now := time.Now()

	currentValues, _ := collectProcStatValues()
	lastValues, lastTime, err := fetchLastValues(tempFileName)
	if currentValues != nil {
		saveValues(tempFileName, currentValues, now)
	}
	if err != nil {
		log.Fatalln("fetchLastValues: ", err)
	}

	var cpuUsage []*cpuPercentages
	if lastValues != nil {
		var err error
		cpuUsage, err = calcCPUUsage(currentValues, now, lastValues, lastTime)
		if err != nil {
			log.Fatalln("calcCPUUsage: ", err)
		}
	}

	loadavg5, err := fetchLoadavg5()
	if err != nil {
		log.Fatalln("fetchLoadavg5: ", err)
	}
	loadPerCPUCount := loadavg5 / (float64(len(cpuUsage) - 1))

	if os.Getenv("MACKEREL_AGENT_PLUGIN_META") != "" {
		outputDefinitions()
	} else {
		outputCPUUsage(cpuUsage, now)
		outputLoadavgPerCore(loadPerCPUCount, now)
	}
}
<commit_msg>gofmt<commit_after>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"strings"
	"time"

	mp "github.com/mackerelio/go-mackerel-plugin"
)

var graphDef = map[string](mp.Graphs){
	"multicore.cpu.#": mp.Graphs{
		Label: "MultiCore CPU",
		Unit:  "percentage",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "user", Label: "user", Diff: false, Stacked: true},
			mp.Metrics{Name: "nice", Label: "nice", Diff: false, Stacked: true},
			mp.Metrics{Name: "system", Label: "system", Diff: false, Stacked: true},
			mp.Metrics{Name: "idle", Label: "idle", Diff: false, Stacked: true},
			mp.Metrics{Name: "iowait", Label: "ioWait", Diff: false, Stacked: true},
			mp.Metrics{Name: "irq", Label: "irq", Diff: false, Stacked: true},
			mp.Metrics{Name: "softirq", Label: "softirq", Diff: false, Stacked: true},
			mp.Metrics{Name: "steal", Label: "steal", Diff: false, Stacked: true},
			mp.Metrics{Name: "guest", Label: "guest", Diff: false, Stacked: true},
		},
	},
	"multicore.loadavg_per_core": mp.Graphs{
		Label: "MultiCore loadavg5 per core",
		Unit:  "float",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "loadavg5", Label: "loadavg5", Diff: false, Stacked: false},
		},
	},
}

type saveItem struct {
	LastTime       time.Time
	ProcStatsByCPU map[string]*procStats
}

type procStats struct {
	User    float64 `json:"user"`
	Nice    float64 `json:"nice"`
	System  float64 `json:"system"`
	Idle    float64 `json:"idle"`
	IoWait  float64 `json:"iowait"`
	Irq     float64 `json:"irq"`
	SoftIrq float64 `json:"softirq"`
	Steal   float64 `json:"steal"`
	Guest   float64 `json:"guest"`
	Total   float64 `json:"total"`
}

type cpuPercentages struct {
	GroupName string
	User      float64
	Nice      float64
	System    float64
	Idle      float64
	IoWait    float64
	Irq       float64
	SoftIrq   float64
	Steal     float64
	Guest     float64
}

func getProcStat() (string, error) {
	contentbytes, err := ioutil.ReadFile("/proc/stat")
	if err != nil {
		return "", err
	}
	return string(contentbytes), nil
}

func parseFloats(values []string) ([]float64, error) {
	var result []float64
	for _, v := range values {
		f, err := strconv.ParseFloat(v, 64)
		if err != nil {
			return nil, err
		}
		result = append(result, f)
	}
	return result, nil
}

func fill(arr []float64, elementCount int) []float64 {
	if len(arr) < elementCount {
		zeroArr := make([]float64, elementCount-len(arr))
		filled := append(arr, zeroArr...)
		return filled
	}
	return arr
}

func parseProcStat(str string) (map[string]*procStats, error) {
	var result = make(map[string]*procStats)
	for _, line := range strings.Split(str, "\n") {
		if strings.HasPrefix(line, "cpu") {
			fields := strings.Fields(line)
			key := fields[0]
			values := fields[1:]

			floatValues, err := parseFloats(values)
			if err != nil {
				return nil, err
			}
			filledValues := fill(floatValues, 9)

			total := 0.0
			for _, v := range floatValues {
				total += v
			}

			ps := &procStats{
				User:    filledValues[0],
				Nice:    filledValues[1],
				System:  filledValues[2],
				Idle:    filledValues[3],
				IoWait:  filledValues[4],
				Irq:     filledValues[5],
				SoftIrq: filledValues[6],
				Steal:   filledValues[7],
				Guest:   filledValues[8],
				Total:   total,
			}
			result[key] = ps

		} else {
			break
		}
	}
	return result, nil
}

func collectProcStatValues() (map[string]*procStats, error) {
	procStats, err := getProcStat()
	if err != nil {
		return nil, err
	}
	return parseProcStat(procStats)
}

func saveValues(tempFileName string, values map[string]*procStats, now time.Time) error {
	f, err := os.Create(tempFileName)
	if err != nil {
		return err
	}
	defer f.Close()

	s := saveItem{
		LastTime:       time.Now(),
		ProcStatsByCPU: values,
	}

	encoder := json.NewEncoder(f)
	err = encoder.Encode(s)
	if err != nil {
		return err
	}

	return nil
}

func fetchLastValues(tempFileName string) (map[string]*procStats, time.Time, error) {
	f, err := os.Open(tempFileName)
	if err != nil {
		return nil, time.Now(), err
	}
	defer f.Close()

	var stat saveItem
	decoder := json.NewDecoder(f)
	err = decoder.Decode(&stat)
	if err != nil {
		return stat.ProcStatsByCPU, stat.LastTime, err
	}
	return stat.ProcStatsByCPU, stat.LastTime, nil
}

func calcCPUUsage(currentValues map[string]*procStats, now time.Time, lastValues map[string]*procStats, lastTime time.Time) ([]*cpuPercentages, error) {

	var result []*cpuPercentages
	for key, current := range currentValues {
		last, ok := lastValues[key]
		if ok {
			user, err := calcPercentage(current.User, last.User, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			nice, err := calcPercentage(current.Nice, last.Nice, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			system, err := calcPercentage(current.System, last.System, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			idle, err := calcPercentage(current.Idle, last.Idle, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			iowait, err := calcPercentage(current.IoWait, last.IoWait, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			irq, err := calcPercentage(current.Irq, last.Irq, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			softirq, err := calcPercentage(current.SoftIrq, last.SoftIrq, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			steal, err := calcPercentage(current.Steal, last.Steal, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			guest, err := calcPercentage(current.Guest, last.Guest, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}

			p := &cpuPercentages{
				GroupName: key,
				User:      user,
				Nice:      nice,
				System:    system,
				Idle:      idle,
				IoWait:    iowait,
				Irq:       irq,
				SoftIrq:   softirq,
				Steal:     steal,
				Guest:     guest,
			}
			result = append(result, p)
		}

	}

	return result, nil
}

func calcPercentage(currentValue float64, lastValue float64, currentTotal float64, lastTotal float64, now time.Time, lastTime time.Time) (float64, error) {
	value, err := calcDiff(currentValue, now, lastValue, lastTime)
	if err != nil {
		return 0.0, err
	}

	total, err := calcDiff(currentTotal, now, lastTotal, lastTime)
	if err != nil {
		return 0.0, err
	}

	return (value / total * 100.0), nil
}

func calcDiff(value float64, now time.Time, lastValue float64, lastTime time.Time) (float64, error) {
	diffTime := now.Unix() - lastTime.Unix()
	if diffTime > 600 {
		return 0.0, errors.New("Too long duration")
	}

	diff := (value - lastValue) * 60 / float64(diffTime)

	if lastValue <= value {
		return diff, nil
	}
	return 0.0, nil
}

func fetchLoadavg5() (float64, error) {
	contentbytes, err := ioutil.ReadFile("/proc/loadavg")
	if err != nil {
		return 0.0, err
	}
	content := string(contentbytes)
	cols := strings.Fields(content)

	if len(cols) > 2 {
		f, err := strconv.ParseFloat(cols[1], 64)
		if err != nil {
			return 0.0, err
		}
		return f, nil
	}
	return 0.0, errors.New("cannot fetch loadavg5.")
}

func printValue(key string, value float64, time time.Time) {
	fmt.Printf("%s\t%f\t%d\n", key, value, time.Unix())
}

func outputCPUUsage(cpuUsage []*cpuPercentages, now time.Time) {
	if cpuUsage != nil {
		for _, u := range cpuUsage {
			if u.GroupName != "cpu" {
				printValue(fmt.Sprintf("multicore.cpu.%s.user", u.GroupName), u.User, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.nice", u.GroupName), u.Nice, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.system", u.GroupName), u.System, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.idle", u.GroupName), u.Idle, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.iowait", u.GroupName), u.IoWait, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.irq", u.GroupName), u.Irq, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.softirq", u.GroupName), u.SoftIrq, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.steal", u.GroupName), u.Steal, now)
				printValue(fmt.Sprintf("multicore.cpu.%s.guest", u.GroupName), u.Guest, now)
			}
		}
	}
}

func outputLoadavgPerCore(loadavgPerCore float64, now time.Time) {
	printValue("multicore.loadavg_per_core.loadavg5", loadavgPerCore, now)
}

func outputDefinitions() {
	fmt.Println("# mackerel-agent-plugin")
	var graphs mp.GraphDef
	graphs.Graphs = graphDef

	b, err := json.Marshal(graphs)
	if err != nil {
		log.Fatalln("OutputDefinitions: ", err)
	}
	fmt.Println(string(b))
}

// main function
func main() {
	optTempfile := flag.String("tempfile", "", "Temp file name")
	var tempFileName string
	if *optTempfile != "" {
		tempFileName = *optTempfile
	} else {
		tempFileName = "/tmp/mackerel-plugin-multicore"
	}
	now := time.Now()

	currentValues, _ := collectProcStatValues()
	lastValues, lastTime, err := fetchLastValues(tempFileName)
	if currentValues != nil {
		saveValues(tempFileName, currentValues, now)
	}
	if err != nil {
		log.Fatalln("fetchLastValues: ", err)
	}

	var cpuUsage []*cpuPercentages
	if lastValues != nil {
		var err error
		cpuUsage, err = calcCPUUsage(currentValues, now, lastValues, lastTime)
		if err != nil {
			log.Fatalln("calcCPUUsage: ", err)
		}
	}

	loadavg5, err := fetchLoadavg5()
	if err != nil {
		log.Fatalln("fetchLoadavg5: ", err)
	}
	loadPerCPUCount := loadavg5 / (float64(len(cpuUsage) - 1))

	if os.Getenv("MACKEREL_AGENT_PLUGIN_META") != "" {
		outputDefinitions()
	} else {
		outputCPUUsage(cpuUsage, now)
		outputLoadavgPerCore(loadPerCPUCount, now)
	}
}
<|endoftext|>


<commit_before>package mpprocfd

import (
	"flag"
	"fmt"
	"os"
	"regexp"

	mp "github.com/mackerelio/go-mackerel-plugin-helper"
	"github.com/mackerelio/mackerel-agent/logging"
)

var logger = logging.GetLogger("metrics.plugin.proc-fd")

// ProcfdPlugin for fetching metrics
type ProcfdPlugin struct {
	Process           string
	NormalizedProcess string
	MetricName        string
}

// FetchMetrics fetch the metrics
func (p ProcfdPlugin) FetchMetrics() (map[string]interface{}, error) {
	fds, err := openFd.getNumOpenFileDesc()
	if err != nil {
		return nil, err
	}

	stat := make(map[string]interface{})

	// Compute maximum open file descriptor
	var maxFD uint64
	for _, fd := range fds {
		if fd > maxFD {
			maxFD = fd
		}
	}
	stat["max_fd"] = maxFD

	return stat, nil
}

// GraphDefinition Graph definition
func (p ProcfdPlugin) GraphDefinition() map[string]mp.Graphs {
	return map[string]mp.Graphs{
		fmt.Sprintf("proc-fd.%s", p.NormalizedProcess): {
			Label: fmt.Sprintf("Opening fd by %s", p.NormalizedProcess),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "max_fd", Label: "Maximum opening fd", Diff: false, Type: "uint64"},
			},
		},
	}
}

func normalizeForMetricName(process string) string {
	// Mackerel accepts following characters in custom metric names
	// [-a-zA-Z0-9_.]
	re := regexp.MustCompile("[^-a-zA-Z0-9_.]")
	return re.ReplaceAllString(process, "_")
}

// Do the plugin
func Do() {
	optProcess := flag.String("process", "", "Process name")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	flag.Parse()

	if *optProcess == "" {
		logger.Warningf("Process name is required")
		flag.PrintDefaults()
		os.Exit(1)
	}

	var fd ProcfdPlugin
	fd.Process = *optProcess
	openFd = RealOpenFd{fd.Process}
	fd.NormalizedProcess = normalizeForMetricName(*optProcess)

	helper := mp.NewMackerelPlugin(fd)
	if *optTempfile != "" {
		helper.Tempfile = *optTempfile
	} else {
		helper.Tempfile = fmt.Sprintf("/tmp/mackerel-plugin-proc-fd")
	}

	if os.Getenv("MACKEREL_AGENT_PLUGIN_META") != "" {
		helper.OutputDefinitions()
	} else {
		helper.OutputValues()
	}
}
<commit_msg>[proc-fd] don't set default tempfile name by plugin<commit_after>package mpprocfd

import (
	"flag"
	"fmt"
	"os"
	"regexp"

	mp "github.com/mackerelio/go-mackerel-plugin-helper"
	"github.com/mackerelio/mackerel-agent/logging"
)

var logger = logging.GetLogger("metrics.plugin.proc-fd")

// ProcfdPlugin for fetching metrics
type ProcfdPlugin struct {
	Process           string
	NormalizedProcess string
	MetricName        string
}

// FetchMetrics fetch the metrics
func (p ProcfdPlugin) FetchMetrics() (map[string]interface{}, error) {
	fds, err := openFd.getNumOpenFileDesc()
	if err != nil {
		return nil, err
	}

	stat := make(map[string]interface{})

	// Compute maximum open file descriptor
	var maxFD uint64
	for _, fd := range fds {
		if fd > maxFD {
			maxFD = fd
		}
	}
	stat["max_fd"] = maxFD

	return stat, nil
}

// GraphDefinition Graph definition
func (p ProcfdPlugin) GraphDefinition() map[string]mp.Graphs {
	return map[string]mp.Graphs{
		fmt.Sprintf("proc-fd.%s", p.NormalizedProcess): {
			Label: fmt.Sprintf("Opening fd by %s", p.NormalizedProcess),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "max_fd", Label: "Maximum opening fd", Diff: false, Type: "uint64"},
			},
		},
	}
}

func normalizeForMetricName(process string) string {
	// Mackerel accepts following characters in custom metric names
	// [-a-zA-Z0-9_.]
	re := regexp.MustCompile("[^-a-zA-Z0-9_.]")
	return re.ReplaceAllString(process, "_")
}

// Do the plugin
func Do() {
	optProcess := flag.String("process", "", "Process name")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	flag.Parse()

	if *optProcess == "" {
		logger.Warningf("Process name is required")
		flag.PrintDefaults()
		os.Exit(1)
	}

	var fd ProcfdPlugin
	fd.Process = *optProcess
	openFd = RealOpenFd{fd.Process}
	fd.NormalizedProcess = normalizeForMetricName(*optProcess)

	helper := mp.NewMackerelPlugin(fd)
	helper.Tempfile = *optTempfile

	if os.Getenv("MACKEREL_AGENT_PLUGIN_META") != "" {
		helper.OutputDefinitions()
	} else {
		helper.OutputValues()
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package elastigo

import ()

type BaseResponse struct {
	Ok      bool        `json:"ok"`
	Index   string      `json:"_index,omitempty"`
	Type    string      `json:"_type,omitempty"`
	Id      string      `json:"_id,omitempty"`
	Source  interface{} `json:"_source,omitempty"` // depends on the schema you've defined
	Version int         `json:"_version,omitempty"`
	Found   bool        `json:"found,omitempty"`
	Exists  bool        `json:"exists,omitempty"`
	Matches []string    `json:"matches,omitempty"` // percolate matches
}

type Status struct {
	Total      int `json:"total"`
	Successful int `json:"successful"`
	Failed     int `json:"failed"`
}

type ExtendedStatus struct {
	Ok           bool   `json:"ok"`
	ShardsStatus Status `json:"_shards"`
}

type Match struct {
	OK          bool         `json:"ok"`
	Matches     []string     `json:"matches"`
	Explanation *Explanation `json:"explanation,omitempty"`
}

type Explanation struct {
	Value       float32        `json:"value"`
	Description string         `json:"description"`
	Details     []*Explanation `json:"details,omitempty"`
}

func ScrollDuration(duration string) string {
	scrollString := ""
	if duration != "" {
		scrollString = "&scroll=" + duration
	}
	return scrollString
}

// http://www.elasticsearch.org/guide/reference/api/search/search-type/
<commit_msg>add index response field "created".<commit_after>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package elastigo

import ()

type BaseResponse struct {
	Ok      bool        `json:"ok"`
	Index   string      `json:"_index,omitempty"`
	Type    string      `json:"_type,omitempty"`
	Id      string      `json:"_id,omitempty"`
	Source  interface{} `json:"_source,omitempty"` // depends on the schema you've defined
	Version int         `json:"_version,omitempty"`
	Found   bool        `json:"found,omitempty"`
	Exists  bool        `json:"exists,omitempty"`
	Created bool        `json:"created,omitempty"`
	Matches []string    `json:"matches,omitempty"` // percolate matches
}

type Status struct {
	Total      int `json:"total"`
	Successful int `json:"successful"`
	Failed     int `json:"failed"`
}

type ExtendedStatus struct {
	Ok           bool   `json:"ok"`
	ShardsStatus Status `json:"_shards"`
}

type Match struct {
	OK          bool         `json:"ok"`
	Matches     []string     `json:"matches"`
	Explanation *Explanation `json:"explanation,omitempty"`
}

type Explanation struct {
	Value       float32        `json:"value"`
	Description string         `json:"description"`
	Details     []*Explanation `json:"details,omitempty"`
}

func ScrollDuration(duration string) string {
	scrollString := ""
	if duration != "" {
		scrollString = "&scroll=" + duration
	}
	return scrollString
}

// http://www.elasticsearch.org/guide/reference/api/search/search-type/
<|endoftext|>


<commit_before>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package elastigo

import (
	"encoding/json"
	"strconv"
)

type BaseResponse struct {
	Ok      bool        `json:"ok"`
	Index   string      `json:"_index,omitempty"`
	Type    string      `json:"_type,omitempty"`
	Id      string      `json:"_id,omitempty"`
	Source  interface{} `json:"_source,omitempty"` // depends on the schema you've defined
	Version int         `json:"_version,omitempty"`
	Found   bool        `json:"found,omitempty"`
	Exists  bool        `json:"exists,omitempty"`
	Matches []string    `json:"matches,omitempty"` // percolate matches
}

// StatusInt is required because /_optimize, at least, returns its status as
// strings instead of integers.
type StatusInt int

func (self *StatusInt) UnmarshalJSON(b []byte) error {
	s := ""
	if json.Unmarshal(b, &s) == nil {
		if i, err := strconv.Atoi(s); err == nil {
			*self = StatusInt(i)
		}
	}
	return json.Unmarshal(b, self)
}

func (self *StatusInt) MarshalJSON() ([]byte, error) {
	return json.Marshal(*self)
}

type Status struct {
	Total      StatusInt `json:"total"`
	Successful StatusInt `json:"successful"`
	Failed     StatusInt `json:"failed"`
}

type ExtendedStatus struct {
	Ok           bool   `json:"ok"`
	ShardsStatus Status `json:"_shards"`
}

type Match struct {
	OK          bool         `json:"ok"`
	Matches     []string     `json:"matches"`
	Explanation *Explanation `json:"explanation,omitempty"`
}

type Explanation struct {
	Value       float32        `json:"value"`
	Description string         `json:"description"`
	Details     []*Explanation `json:"details,omitempty"`
}

func ScrollDuration(duration string) string {
	scrollString := ""
	if duration != "" {
		scrollString = "&scroll=" + duration
	}
	return scrollString
}

// http://www.elasticsearch.org/guide/reference/api/search/search-type/
<commit_msg>fixes infinite recursion in UnmarshalJSON<commit_after>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package elastigo

import (
	"encoding/json"
	"strconv"
)

type BaseResponse struct {
	Ok      bool        `json:"ok"`
	Index   string      `json:"_index,omitempty"`
	Type    string      `json:"_type,omitempty"`
	Id      string      `json:"_id,omitempty"`
	Source  interface{} `json:"_source,omitempty"` // depends on the schema you've defined
	Version int         `json:"_version,omitempty"`
	Found   bool        `json:"found,omitempty"`
	Exists  bool        `json:"exists,omitempty"`
	Matches []string    `json:"matches,omitempty"` // percolate matches
}

// StatusInt is required because /_optimize, at least, returns its status as
// strings instead of integers.
type StatusInt int

func (self *StatusInt) UnmarshalJSON(b []byte) error {
	s := ""
	if json.Unmarshal(b, &s) == nil {
		if i, err := strconv.Atoi(s); err == nil {
			*self = StatusInt(i)
		}
	}
	i := 0
	err := json.Unmarshal(b, &i)
	if err == nil {
		*self = StatusInt(i)
	}
	return err
}

func (self *StatusInt) MarshalJSON() ([]byte, error) {
	return json.Marshal(*self)
}

type Status struct {
	Total      StatusInt `json:"total"`
	Successful StatusInt `json:"successful"`
	Failed     StatusInt `json:"failed"`
}

type ExtendedStatus struct {
	Ok           bool   `json:"ok"`
	ShardsStatus Status `json:"_shards"`
}

type Match struct {
	OK          bool         `json:"ok"`
	Matches     []string     `json:"matches"`
	Explanation *Explanation `json:"explanation,omitempty"`
}

type Explanation struct {
	Value       float32        `json:"value"`
	Description string         `json:"description"`
	Details     []*Explanation `json:"details,omitempty"`
}

func ScrollDuration(duration string) string {
	scrollString := ""
	if duration != "" {
		scrollString = "&scroll=" + duration
	}
	return scrollString
}

// http://www.elasticsearch.org/guide/reference/api/search/search-type/
<|endoftext|>


<commit_before>package volumeattachmentcommands

import (
	"github.com/jrperritt/rack/commandoptions"
	"github.com/jrperritt/rack/handler"
	"github.com/jrperritt/rack/internal/github.com/codegangsta/cli"
	"github.com/jrperritt/rack/internal/github.com/fatih/structs"
	osVolumeAttach "github.com/jrperritt/rack/internal/github.com/rackspace/gophercloud/openstack/compute/v2/extensions/volumeattach"
	osServers "github.com/jrperritt/rack/internal/github.com/rackspace/gophercloud/openstack/compute/v2/servers"
	"github.com/jrperritt/rack/util"
)

var get = cli.Command{
	Name:        "get",
	Usage:       util.Usage(commandPrefix, "get", "[--server-id <serverID> | --server-name <serverName>] [--id <volumeID> | --name <volumeName> | --stdin id]"),
	Description: "Gets a new volume attachment on the server",
	Action:      actionGet,
	Flags:       commandoptions.CommandFlags(flagsGet, keysGet),
	BashComplete: func(c *cli.Context) {
		commandoptions.CompleteFlags(commandoptions.CommandFlags(flagsGet, keysGet))
	},
}

func flagsGet() []cli.Flag {
	return []cli.Flag{
		cli.StringFlag{
			Name:  "id",
			Usage: "[required] The ID of the attachment.",
		},
		cli.StringFlag{
			Name:  "server-id",
			Usage: "[optional; required if `server-name` isn't provided] The server ID of the attachment.",
		},
		cli.StringFlag{
			Name:  "server-name",
			Usage: "[optional; required if `server-id` isn't provided] The server name of the attachment.",
		},
	}
}

var keysGet = []string{"ID", "Device", "VolumeID", "ServerID"}

type paramsGet struct {
	volumeID string
	serverID string
}

type commandGet handler.Command

func actionGet(c *cli.Context) {
	command := &commandGet{
		Ctx: &handler.Context{
			CLIContext: c,
		},
	}
	handler.Handle(command)
}

func (command *commandGet) Context() *handler.Context {
	return command.Ctx
}

func (command *commandGet) Keys() []string {
	return keysGet
}

func (command *commandGet) ServiceClientType() string {
	return serviceClientType
}

func (command *commandGet) HandleFlags(resource *handler.Resource) error {
	serverID, err := command.Ctx.IDOrName(osServers.IDFromName)
	if err != nil {
		return err
	}

	err = command.Ctx.CheckFlagsSet([]string{"id"})
	if err != nil {
		return err
	}

	resource.Params = &paramsGet{
		volumeID: command.Ctx.CLIContext.String("id"),
		serverID: serverID,
	}
	return nil
}

func (command *commandGet) Execute(resource *handler.Resource) {
	params := resource.Params.(*paramsGet)
	volumeAttachment, err := osVolumeAttach.Get(command.Ctx.ServiceClient, params.serverID, params.volumeID).Extract()
	if err != nil {
		resource.Err = err
		return
	}
	resource.Result = structs.Map(volumeAttachment)
}
<commit_msg>improve usage msg for 'volume-attachment get'<commit_after>package volumeattachmentcommands

import (
	"github.com/jrperritt/rack/commandoptions"
	"github.com/jrperritt/rack/handler"
	"github.com/jrperritt/rack/internal/github.com/codegangsta/cli"
	"github.com/jrperritt/rack/internal/github.com/fatih/structs"
	osVolumeAttach "github.com/jrperritt/rack/internal/github.com/rackspace/gophercloud/openstack/compute/v2/extensions/volumeattach"
	osServers "github.com/jrperritt/rack/internal/github.com/rackspace/gophercloud/openstack/compute/v2/servers"
	"github.com/jrperritt/rack/util"
)

var get = cli.Command{
	Name:        "get",
	Usage:       util.Usage(commandPrefix, "get", "[--server-id <serverID> | --server-name <serverName>] --id <attachmentID> "),
	Description: "Gets a new volume attachment on the server",
	Action:      actionGet,
	Flags:       commandoptions.CommandFlags(flagsGet, keysGet),
	BashComplete: func(c *cli.Context) {
		commandoptions.CompleteFlags(commandoptions.CommandFlags(flagsGet, keysGet))
	},
}

func flagsGet() []cli.Flag {
	return []cli.Flag{
		cli.StringFlag{
			Name:  "id",
			Usage: "[required] The ID of the attachment.",
		},
		cli.StringFlag{
			Name:  "server-id",
			Usage: "[optional; required if `server-name` isn't provided] The server ID of the attachment.",
		},
		cli.StringFlag{
			Name:  "server-name",
			Usage: "[optional; required if `server-id` isn't provided] The server name of the attachment.",
		},
	}
}

var keysGet = []string{"ID", "Device", "VolumeID", "ServerID"}

type paramsGet struct {
	volumeID string
	serverID string
}

type commandGet handler.Command

func actionGet(c *cli.Context) {
	command := &commandGet{
		Ctx: &handler.Context{
			CLIContext: c,
		},
	}
	handler.Handle(command)
}

func (command *commandGet) Context() *handler.Context {
	return command.Ctx
}

func (command *commandGet) Keys() []string {
	return keysGet
}

func (command *commandGet) ServiceClientType() string {
	return serviceClientType
}

func (command *commandGet) HandleFlags(resource *handler.Resource) error {
	serverID, err := command.Ctx.IDOrName(osServers.IDFromName)
	if err != nil {
		return err
	}

	err = command.Ctx.CheckFlagsSet([]string{"id"})
	if err != nil {
		return err
	}

	resource.Params = &paramsGet{
		volumeID: command.Ctx.CLIContext.String("id"),
		serverID: serverID,
	}
	return nil
}

func (command *commandGet) Execute(resource *handler.Resource) {
	params := resource.Params.(*paramsGet)
	volumeAttachment, err := osVolumeAttach.Get(command.Ctx.ServiceClient, params.serverID, params.volumeID).Extract()
	if err != nil {
		resource.Err = err
		return
	}
	resource.Result = structs.Map(volumeAttachment)
}
<|endoftext|>


<commit_before>package migration

import (
	"fmt"
	"io"
	"net/http"

	backupConfig "github.com/lxc/lxd/lxd/backup/config"
	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/ioprogress"
	"github.com/lxc/lxd/shared/units"
)

// Info represents the index frame sent if supported.
type Info struct {
	Config *backupConfig.Config `json:"config,omitempty" yaml:"config,omitempty"` // Equivalent of backup.yaml but embedded in index.
}

// InfoResponse represents the response to the index frame sent if supported.
// Right now this doesn't contain anything useful, its just used to indicate receipt of the index header.
// But in the future the itention is to use it allow the target to send back additional information to the source
// about which frames (such as snapshots) it needs for the migration after having inspected the Info index header.
type InfoResponse struct {
	StatusCode int
	Error      string
}

// Err returns the error of the response.
func (r *InfoResponse) Err() error {
	if r.StatusCode != http.StatusOK {
		return api.StatusErrorf(r.StatusCode, r.Error)
	}

	return nil
}

// Type represents the migration transport type. It indicates the method by which the migration can
// take place and what optional features are available.
type Type struct {
	FSType   MigrationFSType // Transport mode selected.
	Features []string        // Feature hints for selected FSType transport mode.
}

// VolumeSourceArgs represents the arguments needed to setup a volume migration source.
type VolumeSourceArgs struct {
	IndexHeaderVersion uint32
	Name               string
	Snapshots          []string
	MigrationType      Type
	TrackProgress      bool
	MultiSync          bool
	FinalSync          bool
	Data               any // Optional store to persist storage driver state between MultiSync phases.
	ContentType        string
	AllowInconsistent  bool
	Refresh            bool
	Info               *Info
}

// VolumeTargetArgs represents the arguments needed to setup a volume migration sink.
type VolumeTargetArgs struct {
	IndexHeaderVersion uint32
	Name               string
	Description        string
	Config             map[string]string // Only used for custom volume migration.
	Snapshots          []string
	MigrationType      Type
	TrackProgress      bool
	Refresh            bool
	Live               bool
	VolumeSize         int64
	ContentType        string
}

// TypesToHeader converts one or more Types to a MigrationHeader. It uses the first type argument
// supplied to indicate the preferred migration method and sets the MigrationHeader's Fs type
// to that. If the preferred type is ZFS then it will also set the header's optional ZfsFeatures.
// If the fallback Rsync type is present in any of the types even if it is not preferred, then its
// optional features are added to the header's RsyncFeatures, allowing for fallback negotiation to
// take place on the farside.
func TypesToHeader(types ...Type) *MigrationHeader {
	missingFeature := false
	hasFeature := true
	var preferredType Type

	if len(types) > 0 {
		preferredType = types[0]
	}

	header := MigrationHeader{Fs: &preferredType.FSType}

	// Add ZFS features if preferred type is ZFS.
	if preferredType.FSType == MigrationFSType_ZFS {
		features := ZfsFeatures{
			Compress: &missingFeature,
		}
		for _, feature := range preferredType.Features {
			if feature == "compress" {
				features.Compress = &hasFeature
			} else if feature == ZFSFeatureMigrationHeader {
				features.MigrationHeader = &hasFeature
			}
		}

		header.ZfsFeatures = &features
	}

	// Add BTRFS features if preferred type is BTRFS.
	if preferredType.FSType == MigrationFSType_BTRFS {
		features := BtrfsFeatures{
			MigrationHeader:  &missingFeature,
			HeaderSubvolumes: &missingFeature,
		}
		for _, feature := range preferredType.Features {
			if feature == BTRFSFeatureMigrationHeader {
				features.MigrationHeader = &hasFeature
			} else if feature == BTRFSFeatureSubvolumes {
				features.HeaderSubvolumes = &hasFeature
			} else if feature == BTRFSFeatureSubvolumeUUIDs {
				features.HeaderSubvolumeUuids = &hasFeature
			}
		}

		header.BtrfsFeatures = &features
	}

	// Check all the types for an Rsync method, if found add its features to the header's RsyncFeatures list.
	for _, t := range types {
		if t.FSType != MigrationFSType_RSYNC && t.FSType != MigrationFSType_BLOCK_AND_RSYNC {
			continue
		}

		features := RsyncFeatures{
			Xattrs:        &missingFeature,
			Delete:        &missingFeature,
			Compress:      &missingFeature,
			Bidirectional: &missingFeature,
		}

		for _, feature := range t.Features {
			if feature == "xattrs" {
				features.Xattrs = &hasFeature
			} else if feature == "delete" {
				features.Delete = &hasFeature
			} else if feature == "compress" {
				features.Compress = &hasFeature
			} else if feature == "bidirectional" {
				features.Bidirectional = &hasFeature
			}
		}

		header.RsyncFeatures = &features
		break // Only use the first rsync transport type found to generate rsync features list.
	}

	return &header
}

// MatchTypes attempts to find matching migration transport types between an offered type sent from a remote
// source and the types supported by a local storage pool. If matches are found then one or more Types are
// returned containing the method and the matching optional features present in both. The function also takes a
// fallback type which is used as an additional offer type preference in case the preferred remote type is not
// compatible with the local type available. It is expected that both sides of the migration will support the
// fallback type for the volume's content type that is being migrated.
func MatchTypes(offer *MigrationHeader, fallbackType MigrationFSType, ourTypes []Type) ([]Type, error) {
	// Generate an offer types slice from the preferred type supplied from remote and the
	// fallback type supplied based on the content type of the transfer.
	offeredFSTypes := []MigrationFSType{offer.GetFs(), fallbackType}

	matchedTypes := []Type{}

	// Find first matching type.
	for _, ourType := range ourTypes {
		for _, offerFSType := range offeredFSTypes {
			if offerFSType != ourType.FSType {
				continue // Not a match, try the next one.
			}

			// We got a match, now extract the relevant offered features.
			var offeredFeatures []string
			if offerFSType == MigrationFSType_ZFS {
				offeredFeatures = offer.GetZfsFeaturesSlice()
			} else if offerFSType == MigrationFSType_BTRFS {
				offeredFeatures = offer.GetBtrfsFeaturesSlice()
			} else if offerFSType == MigrationFSType_RSYNC {
				offeredFeatures = offer.GetRsyncFeaturesSlice()
				if !shared.StringInSlice("bidirectional", offeredFeatures) {
					// If no bi-directional support, this means we are getting a response from
					// an old LXD server that doesn't support bidirectional negotiation, so
					// assume LXD 3.7 level. NOTE: Do NOT extend this list of arguments.
					offeredFeatures = []string{"xattrs", "delete", "compress"}
				}
			}

			// Find common features in both our type and offered type.
			commonFeatures := []string{}
			for _, ourFeature := range ourType.Features {
				if shared.StringInSlice(ourFeature, offeredFeatures) {
					commonFeatures = append(commonFeatures, ourFeature)
				}
			}

			if offer.Refresh != nil && *offer.Refresh == true {
				// Optimized refresh with zfs only works if ZfsFeatureMigrationHeader is available.
				if ourType.FSType == MigrationFSType_ZFS && !shared.StringInSlice(ZFSFeatureMigrationHeader, commonFeatures) {
					continue
				}

				// Optimized refresh with btrfs only works if BtrfsFeatureSubvolumeUUIDs is available.
				if ourType.FSType == MigrationFSType_BTRFS && !shared.StringInSlice(BTRFSFeatureSubvolumeUUIDs, commonFeatures) {
					continue
				}
			}

			// Append type with combined features.
			matchedTypes = append(matchedTypes, Type{
				FSType:   ourType.FSType,
				Features: commonFeatures,
			})
		}
	}

	if len(matchedTypes) < 1 {
		// No matching transport type found, generate an error with offered types and our types.
		offeredTypeStrings := make([]string, 0, len(offeredFSTypes))
		for _, offerFSType := range offeredFSTypes {
			offeredTypeStrings = append(offeredTypeStrings, offerFSType.String())
		}

		ourTypeStrings := make([]string, 0, len(ourTypes))
		for _, ourType := range ourTypes {
			ourTypeStrings = append(ourTypeStrings, ourType.FSType.String())
		}

		return matchedTypes, fmt.Errorf("No matching migration types found. Offered types: %v, our types: %v", offeredTypeStrings, ourTypeStrings)
	}

	return matchedTypes, nil
}

func progressWrapperRender(op *operations.Operation, key string, description string, progressInt int64, speedInt int64) {
	meta := op.Metadata()
	if meta == nil {
		meta = make(map[string]any)
	}

	progress := fmt.Sprintf("%s (%s/s)", units.GetByteSizeString(progressInt, 2), units.GetByteSizeString(speedInt, 2))
	if description != "" {
		progress = fmt.Sprintf("%s: %s (%s/s)", description, units.GetByteSizeString(progressInt, 2), units.GetByteSizeString(speedInt, 2))
	}

	if meta[key] != progress {
		meta[key] = progress
		_ = op.UpdateMetadata(meta)
	}
}

// ProgressReader reports the read progress.
func ProgressReader(op *operations.Operation, key string, description string) func(io.ReadCloser) io.ReadCloser {
	return func(reader io.ReadCloser) io.ReadCloser {
		if op == nil {
			return reader
		}

		progress := func(progressInt int64, speedInt int64) {
			progressWrapperRender(op, key, description, progressInt, speedInt)
		}

		readPipe := &ioprogress.ProgressReader{
			ReadCloser: reader,
			Tracker: &ioprogress.ProgressTracker{
				Handler: progress,
			},
		}

		return readPipe
	}
}

// ProgressWriter reports the write progress.
func ProgressWriter(op *operations.Operation, key string, description string) func(io.WriteCloser) io.WriteCloser {
	return func(writer io.WriteCloser) io.WriteCloser {
		if op == nil {
			return writer
		}

		progress := func(progressInt int64, speedInt int64) {
			progressWrapperRender(op, key, description, progressInt, speedInt)
		}

		writePipe := &ioprogress.ProgressWriter{
			WriteCloser: writer,
			Tracker: &ioprogress.ProgressTracker{
				Handler: progress,
			},
		}

		return writePipe
	}
}

// ProgressTracker returns a migration I/O tracker
func ProgressTracker(op *operations.Operation, key string, description string) *ioprogress.ProgressTracker {
	progress := func(progressInt int64, speedInt int64) {
		progressWrapperRender(op, key, description, progressInt, speedInt)
	}

	tracker := &ioprogress.ProgressTracker{
		Handler: progress,
	}

	return tracker
}
<commit_msg>lxd/migration: Add Refresh to InfoResponse<commit_after>package migration

import (
	"fmt"
	"io"
	"net/http"

	backupConfig "github.com/lxc/lxd/lxd/backup/config"
	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/ioprogress"
	"github.com/lxc/lxd/shared/units"
)

// Info represents the index frame sent if supported.
type Info struct {
	Config *backupConfig.Config `json:"config,omitempty" yaml:"config,omitempty"` // Equivalent of backup.yaml but embedded in index.
}

// InfoResponse represents the response to the index frame sent if supported.
// Right now this doesn't contain anything useful, its just used to indicate receipt of the index header.
// But in the future the itention is to use it allow the target to send back additional information to the source
// about which frames (such as snapshots) it needs for the migration after having inspected the Info index header.
type InfoResponse struct {
	StatusCode int
	Error      string
	Refresh    *bool // This is used to let the source know whether to actually refresh a volume.
}

// Err returns the error of the response.
func (r *InfoResponse) Err() error {
	if r.StatusCode != http.StatusOK {
		return api.StatusErrorf(r.StatusCode, r.Error)
	}

	return nil
}

// Type represents the migration transport type. It indicates the method by which the migration can
// take place and what optional features are available.
type Type struct {
	FSType   MigrationFSType // Transport mode selected.
	Features []string        // Feature hints for selected FSType transport mode.
}

// VolumeSourceArgs represents the arguments needed to setup a volume migration source.
type VolumeSourceArgs struct {
	IndexHeaderVersion uint32
	Name               string
	Snapshots          []string
	MigrationType      Type
	TrackProgress      bool
	MultiSync          bool
	FinalSync          bool
	Data               any // Optional store to persist storage driver state between MultiSync phases.
	ContentType        string
	AllowInconsistent  bool
	Refresh            bool
	Info               *Info
}

// VolumeTargetArgs represents the arguments needed to setup a volume migration sink.
type VolumeTargetArgs struct {
	IndexHeaderVersion uint32
	Name               string
	Description        string
	Config             map[string]string // Only used for custom volume migration.
	Snapshots          []string
	MigrationType      Type
	TrackProgress      bool
	Refresh            bool
	Live               bool
	VolumeSize         int64
	ContentType        string
}

// TypesToHeader converts one or more Types to a MigrationHeader. It uses the first type argument
// supplied to indicate the preferred migration method and sets the MigrationHeader's Fs type
// to that. If the preferred type is ZFS then it will also set the header's optional ZfsFeatures.
// If the fallback Rsync type is present in any of the types even if it is not preferred, then its
// optional features are added to the header's RsyncFeatures, allowing for fallback negotiation to
// take place on the farside.
func TypesToHeader(types ...Type) *MigrationHeader {
	missingFeature := false
	hasFeature := true
	var preferredType Type

	if len(types) > 0 {
		preferredType = types[0]
	}

	header := MigrationHeader{Fs: &preferredType.FSType}

	// Add ZFS features if preferred type is ZFS.
	if preferredType.FSType == MigrationFSType_ZFS {
		features := ZfsFeatures{
			Compress: &missingFeature,
		}
		for _, feature := range preferredType.Features {
			if feature == "compress" {
				features.Compress = &hasFeature
			} else if feature == ZFSFeatureMigrationHeader {
				features.MigrationHeader = &hasFeature
			}
		}

		header.ZfsFeatures = &features
	}

	// Add BTRFS features if preferred type is BTRFS.
	if preferredType.FSType == MigrationFSType_BTRFS {
		features := BtrfsFeatures{
			MigrationHeader:  &missingFeature,
			HeaderSubvolumes: &missingFeature,
		}
		for _, feature := range preferredType.Features {
			if feature == BTRFSFeatureMigrationHeader {
				features.MigrationHeader = &hasFeature
			} else if feature == BTRFSFeatureSubvolumes {
				features.HeaderSubvolumes = &hasFeature
			} else if feature == BTRFSFeatureSubvolumeUUIDs {
				features.HeaderSubvolumeUuids = &hasFeature
			}
		}

		header.BtrfsFeatures = &features
	}

	// Check all the types for an Rsync method, if found add its features to the header's RsyncFeatures list.
	for _, t := range types {
		if t.FSType != MigrationFSType_RSYNC && t.FSType != MigrationFSType_BLOCK_AND_RSYNC {
			continue
		}

		features := RsyncFeatures{
			Xattrs:        &missingFeature,
			Delete:        &missingFeature,
			Compress:      &missingFeature,
			Bidirectional: &missingFeature,
		}

		for _, feature := range t.Features {
			if feature == "xattrs" {
				features.Xattrs = &hasFeature
			} else if feature == "delete" {
				features.Delete = &hasFeature
			} else if feature == "compress" {
				features.Compress = &hasFeature
			} else if feature == "bidirectional" {
				features.Bidirectional = &hasFeature
			}
		}

		header.RsyncFeatures = &features
		break // Only use the first rsync transport type found to generate rsync features list.
	}

	return &header
}

// MatchTypes attempts to find matching migration transport types between an offered type sent from a remote
// source and the types supported by a local storage pool. If matches are found then one or more Types are
// returned containing the method and the matching optional features present in both. The function also takes a
// fallback type which is used as an additional offer type preference in case the preferred remote type is not
// compatible with the local type available. It is expected that both sides of the migration will support the
// fallback type for the volume's content type that is being migrated.
func MatchTypes(offer *MigrationHeader, fallbackType MigrationFSType, ourTypes []Type) ([]Type, error) {
	// Generate an offer types slice from the preferred type supplied from remote and the
	// fallback type supplied based on the content type of the transfer.
	offeredFSTypes := []MigrationFSType{offer.GetFs(), fallbackType}

	matchedTypes := []Type{}

	// Find first matching type.
	for _, ourType := range ourTypes {
		for _, offerFSType := range offeredFSTypes {
			if offerFSType != ourType.FSType {
				continue // Not a match, try the next one.
			}

			// We got a match, now extract the relevant offered features.
			var offeredFeatures []string
			if offerFSType == MigrationFSType_ZFS {
				offeredFeatures = offer.GetZfsFeaturesSlice()
			} else if offerFSType == MigrationFSType_BTRFS {
				offeredFeatures = offer.GetBtrfsFeaturesSlice()
			} else if offerFSType == MigrationFSType_RSYNC {
				offeredFeatures = offer.GetRsyncFeaturesSlice()
				if !shared.StringInSlice("bidirectional", offeredFeatures) {
					// If no bi-directional support, this means we are getting a response from
					// an old LXD server that doesn't support bidirectional negotiation, so
					// assume LXD 3.7 level. NOTE: Do NOT extend this list of arguments.
					offeredFeatures = []string{"xattrs", "delete", "compress"}
				}
			}

			// Find common features in both our type and offered type.
			commonFeatures := []string{}
			for _, ourFeature := range ourType.Features {
				if shared.StringInSlice(ourFeature, offeredFeatures) {
					commonFeatures = append(commonFeatures, ourFeature)
				}
			}

			if offer.Refresh != nil && *offer.Refresh == true {
				// Optimized refresh with zfs only works if ZfsFeatureMigrationHeader is available.
				if ourType.FSType == MigrationFSType_ZFS && !shared.StringInSlice(ZFSFeatureMigrationHeader, commonFeatures) {
					continue
				}

				// Optimized refresh with btrfs only works if BtrfsFeatureSubvolumeUUIDs is available.
				if ourType.FSType == MigrationFSType_BTRFS && !shared.StringInSlice(BTRFSFeatureSubvolumeUUIDs, commonFeatures) {
					continue
				}
			}

			// Append type with combined features.
			matchedTypes = append(matchedTypes, Type{
				FSType:   ourType.FSType,
				Features: commonFeatures,
			})
		}
	}

	if len(matchedTypes) < 1 {
		// No matching transport type found, generate an error with offered types and our types.
		offeredTypeStrings := make([]string, 0, len(offeredFSTypes))
		for _, offerFSType := range offeredFSTypes {
			offeredTypeStrings = append(offeredTypeStrings, offerFSType.String())
		}

		ourTypeStrings := make([]string, 0, len(ourTypes))
		for _, ourType := range ourTypes {
			ourTypeStrings = append(ourTypeStrings, ourType.FSType.String())
		}

		return matchedTypes, fmt.Errorf("No matching migration types found. Offered types: %v, our types: %v", offeredTypeStrings, ourTypeStrings)
	}

	return matchedTypes, nil
}

func progressWrapperRender(op *operations.Operation, key string, description string, progressInt int64, speedInt int64) {
	meta := op.Metadata()
	if meta == nil {
		meta = make(map[string]any)
	}

	progress := fmt.Sprintf("%s (%s/s)", units.GetByteSizeString(progressInt, 2), units.GetByteSizeString(speedInt, 2))
	if description != "" {
		progress = fmt.Sprintf("%s: %s (%s/s)", description, units.GetByteSizeString(progressInt, 2), units.GetByteSizeString(speedInt, 2))
	}

	if meta[key] != progress {
		meta[key] = progress
		_ = op.UpdateMetadata(meta)
	}
}

// ProgressReader reports the read progress.
func ProgressReader(op *operations.Operation, key string, description string) func(io.ReadCloser) io.ReadCloser {
	return func(reader io.ReadCloser) io.ReadCloser {
		if op == nil {
			return reader
		}

		progress := func(progressInt int64, speedInt int64) {
			progressWrapperRender(op, key, description, progressInt, speedInt)
		}

		readPipe := &ioprogress.ProgressReader{
			ReadCloser: reader,
			Tracker: &ioprogress.ProgressTracker{
				Handler: progress,
			},
		}

		return readPipe
	}
}

// ProgressWriter reports the write progress.
func ProgressWriter(op *operations.Operation, key string, description string) func(io.WriteCloser) io.WriteCloser {
	return func(writer io.WriteCloser) io.WriteCloser {
		if op == nil {
			return writer
		}

		progress := func(progressInt int64, speedInt int64) {
			progressWrapperRender(op, key, description, progressInt, speedInt)
		}

		writePipe := &ioprogress.ProgressWriter{
			WriteCloser: writer,
			Tracker: &ioprogress.ProgressTracker{
				Handler: progress,
			},
		}

		return writePipe
	}
}

// ProgressTracker returns a migration I/O tracker
func ProgressTracker(op *operations.Operation, key string, description string) *ioprogress.ProgressTracker {
	progress := func(progressInt int64, speedInt int64) {
		progressWrapperRender(op, key, description, progressInt, speedInt)
	}

	tracker := &ioprogress.ProgressTracker{
		Handler: progress,
	}

	return tracker
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This example demonstrates a priority queue built using the heap interface.
package heap_test

import (
	"container/heap"
	"fmt"
)

// An Item is something we manage in a priority queue.
type Item struct {
	value    string // The value of the item; arbitrary.
	priority int    // The priority of the item in the queue.
	// The index is needed by update and is maintained by the heap.Interface methods.
	index int // The index of the item in the heap.
}

// A PriorityQueue implements heap.Interface and holds Items.
type PriorityQueue []*Item

func (pq PriorityQueue) Len() int { return len(pq) }

func (pq PriorityQueue) Less(i, j int) bool {
	// We want Pop to give us the highest, not lowest, priority so we use greater than here.
	return pq[i].priority > pq[j].priority
}

func (pq PriorityQueue) Swap(i, j int) {
	pq[i], pq[j] = pq[j], pq[i]
	pq[i].index = i
	pq[j].index = j
}

func (pq *PriorityQueue) Push(x interface{}) {
	n := len(*pq)
	item := x.(*Item)
	item.index = n
	*pq = append(*pq, item)
}

func (pq *PriorityQueue) Pop() interface{} {
	old := *pq
	n := len(old)
	item := old[n-1]
	item.index = -1 // for safety
	*pq = old[0 : n-1]
	return item
}

// update modifies the priority and value of an Item in the queue.
func (pq *PriorityQueue) update(item *Item, value string, priority int) {
	item.value = value
	item.priority = priority
	heap.Fix(pq, item.index)
}

// This example creates a PriorityQueue with some items, adds and manipulates an item,
// and then removes the items in priority order.
func Example_priorityQueue() {
	// Some items and their priorities.
	items := map[string]int{
		"banana": 3, "apple": 2, "pear": 4,
	}

	// Create a priority queue, put the items in it, and
	// establish the priority queue (heap) invariants.
	pq := make(PriorityQueue, len(items))
	i := 0
	for value, priority := range items {
		pq[i] = &Item{
			value:    value,
			priority: priority,
			index:    i,
		}
		i++
	}
	heap.Init(&pq)

	// Insert a new item and then modify its priority.
	item := &Item{
		value:    "orange",
		priority: 1,
	}
	heap.Push(&pq, item)
	pq.update(item, item.value, 5)

	// Take the items out; they arrive in decreasing priority order.
	for pq.Len() > 0 {
		item := heap.Pop(&pq).(*Item)
		fmt.Printf("%.2d:%s ", item.priority, item.value)
	}
	// Output:
	// 05:orange 04:pear 03:banana 02:apple
}
<commit_msg>container/heap: avoid memory leak in example<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This example demonstrates a priority queue built using the heap interface.
package heap_test

import (
	"container/heap"
	"fmt"
)

// An Item is something we manage in a priority queue.
type Item struct {
	value    string // The value of the item; arbitrary.
	priority int    // The priority of the item in the queue.
	// The index is needed by update and is maintained by the heap.Interface methods.
	index int // The index of the item in the heap.
}

// A PriorityQueue implements heap.Interface and holds Items.
type PriorityQueue []*Item

func (pq PriorityQueue) Len() int { return len(pq) }

func (pq PriorityQueue) Less(i, j int) bool {
	// We want Pop to give us the highest, not lowest, priority so we use greater than here.
	return pq[i].priority > pq[j].priority
}

func (pq PriorityQueue) Swap(i, j int) {
	pq[i], pq[j] = pq[j], pq[i]
	pq[i].index = i
	pq[j].index = j
}

func (pq *PriorityQueue) Push(x interface{}) {
	n := len(*pq)
	item := x.(*Item)
	item.index = n
	*pq = append(*pq, item)
}

func (pq *PriorityQueue) Pop() interface{} {
	old := *pq
	n := len(old)
	item := old[n-1]
	old[n-1] = nil  // avoid memory leak
	item.index = -1 // for safety
	*pq = old[0 : n-1]
	return item
}

// update modifies the priority and value of an Item in the queue.
func (pq *PriorityQueue) update(item *Item, value string, priority int) {
	item.value = value
	item.priority = priority
	heap.Fix(pq, item.index)
}

// This example creates a PriorityQueue with some items, adds and manipulates an item,
// and then removes the items in priority order.
func Example_priorityQueue() {
	// Some items and their priorities.
	items := map[string]int{
		"banana": 3, "apple": 2, "pear": 4,
	}

	// Create a priority queue, put the items in it, and
	// establish the priority queue (heap) invariants.
	pq := make(PriorityQueue, len(items))
	i := 0
	for value, priority := range items {
		pq[i] = &Item{
			value:    value,
			priority: priority,
			index:    i,
		}
		i++
	}
	heap.Init(&pq)

	// Insert a new item and then modify its priority.
	item := &Item{
		value:    "orange",
		priority: 1,
	}
	heap.Push(&pq, item)
	pq.update(item, item.value, 5)

	// Take the items out; they arrive in decreasing priority order.
	for pq.Len() > 0 {
		item := heap.Pop(&pq).(*Item)
		fmt.Printf("%.2d:%s ", item.priority, item.value)
	}
	// Output:
	// 05:orange 04:pear 03:banana 02:apple
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// godoc: Go Documentation Server

// Web server tree:
//
//	http://godoc/		main landing page
//	http://godoc/doc/	serve from $GOROOT/doc - spec, mem, tutorial, etc.
//	http://godoc/src/	serve files from $GOROOT/src; .go gets pretty-printed
//	http://godoc/cmd/	serve documentation about commands (TODO)
//	http://godoc/pkg/	serve documentation about packages
//				(idea is if you say import "compress/zlib", you go to
//				http://godoc/pkg/compress/zlib)
//
// Command-line interface:
//
//	godoc packagepath [name ...]
//
//	godoc compress/zlib
//		- prints doc for package compress/zlib
//	godoc crypto/block Cipher NewCMAC
//		- prints doc for Cipher and NewCMAC in package crypto/block

package main

import (
	"bytes"
	"flag"
	"fmt"
	"http"
	"io"
	"log"
	"os"
	"time"
)

var (
	// periodic sync
	syncCmd             = flag.String("sync", "", "sync command; disabled if empty")
	syncMin             = flag.Int("sync_minutes", 0, "sync interval in minutes; disabled if <= 0")
	syncDelay delayTime // actual sync delay in minutes; usually syncDelay == syncMin, but delay may back off exponentially

	// server control
	httpaddr = flag.String("http", "", "HTTP service address (e.g., ':6060')")

	// layout control
	html = flag.Bool("html", false, "print HTML in command-line mode")
)

func serveError(c *http.Conn, r *http.Request, relpath string, err os.Error) {
	contents := applyTemplate(errorHTML, "errorHTML", err) // err may contain an absolute path!
	servePage(c, "File "+relpath, "", contents)
}

func exec(c *http.Conn, args []string) (status int) {
	r, w, err := os.Pipe()
	if err != nil {
		log.Stderrf("os.Pipe(): %v\n", err)
		return 2
	}

	bin := args[0]
	fds := []*os.File{nil, w, w}
	if *verbose {
		log.Stderrf("executing %v", args)
	}
	pid, err := os.ForkExec(bin, args, os.Environ(), goroot, fds)
	defer r.Close()
	w.Close()
	if err != nil {
		log.Stderrf("os.ForkExec(%q): %v\n", bin, err)
		return 2
	}

	var buf bytes.Buffer
	io.Copy(&buf, r)
	wait, err := os.Wait(pid, 0)
	if err != nil {
		os.Stderr.Write(buf.Bytes())
		log.Stderrf("os.Wait(%d, 0): %v\n", pid, err)
		return 2
	}
	status = wait.ExitStatus()
	if !wait.Exited() || status > 1 {
		os.Stderr.Write(buf.Bytes())
		log.Stderrf("executing %v failed (exit status = %d)", args, status)
		return
	}

	if *verbose {
		os.Stderr.Write(buf.Bytes())
	}
	if c != nil {
		c.SetHeader("content-type", "text/plain; charset=utf-8")
		c.Write(buf.Bytes())
	}

	return
}

// Maximum directory depth, adjust as needed.
const maxDirDepth = 24

func dosync(c *http.Conn, r *http.Request) {
	args := []string{"/bin/sh", "-c", *syncCmd}
	switch exec(c, args) {
	case 0:
		// sync succeeded and some files have changed;
		// update package tree.
		// TODO(gri): The directory tree may be temporarily out-of-sync.
		//            Consider keeping separate time stamps so the web-
		//            page can indicate this discrepancy.
		fsTree.set(newDirectory(goroot, maxDirDepth))
		fallthrough
	case 1:
		// sync failed because no files changed;
		// don't change the package tree
		syncDelay.set(*syncMin) //  revert to regular sync schedule
	default:
		// sync failed because of an error - back off exponentially, but try at least once a day
		syncDelay.backoff(24 * 60)
	}
}

func usage() {
	fmt.Fprintf(os.Stderr,
		"usage: godoc package [name ...]\n"+
			"	godoc -http=:6060\n")
	flag.PrintDefaults()
	os.Exit(2)
}

func loggingHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(c *http.Conn, req *http.Request) {
		log.Stderrf("%s\t%s", c.RemoteAddr, req.URL)
		h.ServeHTTP(c, req)
	})
}

func main() {
	flag.Usage = usage
	flag.Parse()

	// Check usage: either server and no args, or command line and args
	if (*httpaddr != "") != (flag.NArg() == 0) {
		usage()
	}

	if *tabwidth < 0 {
		log.Exitf("negative tabwidth %d", *tabwidth)
	}

	initHandlers()
	readTemplates()

	if *httpaddr != "" {
		// HTTP server mode.
		var handler http.Handler = http.DefaultServeMux
		if *verbose {
			log.Stderrf("Go Documentation Server\n")
			log.Stderrf("address = %s\n", *httpaddr)
			log.Stderrf("goroot = %s\n", goroot)
			log.Stderrf("tabwidth = %d\n", *tabwidth)
			if !fsMap.IsEmpty() {
				log.Stderr("user-defined mapping:")
				fsMap.Fprint(os.Stderr)
			}
			handler = loggingHandler(handler)
		}

		registerPublicHandlers(http.DefaultServeMux)
		if *syncCmd != "" {
			http.Handle("/debug/sync", http.HandlerFunc(dosync))
		}

		// Initialize directory tree with corresponding timestamp.
		// Do it in two steps:
		// 1) set timestamp right away so that the indexer is kicked on
		fsTree.set(nil)
		// 2) compute initial directory tree in a goroutine so that launch is quick
		go func() { fsTree.set(newDirectory(goroot, maxDirDepth)) }()

		// Start sync goroutine, if enabled.
		if *syncCmd != "" && *syncMin > 0 {
			syncDelay.set(*syncMin) // initial sync delay
			go func() {
				for {
					dosync(nil, nil)
					delay, _ := syncDelay.get()
					if *verbose {
						log.Stderrf("next sync in %dmin", delay.(int))
					}
					time.Sleep(int64(delay.(int)) * 60e9)
				}
			}()
		}

		// Start indexing goroutine.
		go indexer()

		// The server may have been restarted; always wait 1sec to
		// give the forking server a chance to shut down and release
		// the http port.
		// TODO(gri): Do we still need this?
		time.Sleep(1e9)

		// Start http server.
		if err := http.ListenAndServe(*httpaddr, handler); err != nil {
			log.Exitf("ListenAndServe %s: %v", *httpaddr, err)
		}
		return
	}

	// Command line mode.
	if *html {
		packageText = packageHTML
	}

	info := pkgHandler.getPageInfo(flag.Arg(0), flag.Arg(0), true)

	if info.PDoc == nil && info.Dirs == nil {
		// try again, this time assume it's a command
		info = cmdHandler.getPageInfo(flag.Arg(0), flag.Arg(0), false)
	}

	if info.PDoc != nil && flag.NArg() > 1 {
		args := flag.Args()
		info.PDoc.Filter(args[1:])
	}

	if err := packageText.Execute(info, os.Stdout); err != nil {
		log.Stderrf("packageText.Execute: %s", err)
	}
}
<commit_msg>godoc: fix path resolution for command-line one more time (sigh...)<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// godoc: Go Documentation Server

// Web server tree:
//
//	http://godoc/		main landing page
//	http://godoc/doc/	serve from $GOROOT/doc - spec, mem, tutorial, etc.
//	http://godoc/src/	serve files from $GOROOT/src; .go gets pretty-printed
//	http://godoc/cmd/	serve documentation about commands (TODO)
//	http://godoc/pkg/	serve documentation about packages
//				(idea is if you say import "compress/zlib", you go to
//				http://godoc/pkg/compress/zlib)
//
// Command-line interface:
//
//	godoc packagepath [name ...]
//
//	godoc compress/zlib
//		- prints doc for package compress/zlib
//	godoc crypto/block Cipher NewCMAC
//		- prints doc for Cipher and NewCMAC in package crypto/block

package main

import (
	"bytes"
	"flag"
	"fmt"
	"http"
	"io"
	"log"
	"os"
	pathutil "path"
	"time"
)

var (
	// periodic sync
	syncCmd             = flag.String("sync", "", "sync command; disabled if empty")
	syncMin             = flag.Int("sync_minutes", 0, "sync interval in minutes; disabled if <= 0")
	syncDelay delayTime // actual sync delay in minutes; usually syncDelay == syncMin, but delay may back off exponentially

	// server control
	httpaddr = flag.String("http", "", "HTTP service address (e.g., ':6060')")

	// layout control
	html = flag.Bool("html", false, "print HTML in command-line mode")
)

func serveError(c *http.Conn, r *http.Request, relpath string, err os.Error) {
	contents := applyTemplate(errorHTML, "errorHTML", err) // err may contain an absolute path!
	servePage(c, "File "+relpath, "", contents)
}

func exec(c *http.Conn, args []string) (status int) {
	r, w, err := os.Pipe()
	if err != nil {
		log.Stderrf("os.Pipe(): %v\n", err)
		return 2
	}

	bin := args[0]
	fds := []*os.File{nil, w, w}
	if *verbose {
		log.Stderrf("executing %v", args)
	}
	pid, err := os.ForkExec(bin, args, os.Environ(), goroot, fds)
	defer r.Close()
	w.Close()
	if err != nil {
		log.Stderrf("os.ForkExec(%q): %v\n", bin, err)
		return 2
	}

	var buf bytes.Buffer
	io.Copy(&buf, r)
	wait, err := os.Wait(pid, 0)
	if err != nil {
		os.Stderr.Write(buf.Bytes())
		log.Stderrf("os.Wait(%d, 0): %v\n", pid, err)
		return 2
	}
	status = wait.ExitStatus()
	if !wait.Exited() || status > 1 {
		os.Stderr.Write(buf.Bytes())
		log.Stderrf("executing %v failed (exit status = %d)", args, status)
		return
	}

	if *verbose {
		os.Stderr.Write(buf.Bytes())
	}
	if c != nil {
		c.SetHeader("content-type", "text/plain; charset=utf-8")
		c.Write(buf.Bytes())
	}

	return
}

// Maximum directory depth, adjust as needed.
const maxDirDepth = 24

func dosync(c *http.Conn, r *http.Request) {
	args := []string{"/bin/sh", "-c", *syncCmd}
	switch exec(c, args) {
	case 0:
		// sync succeeded and some files have changed;
		// update package tree.
		// TODO(gri): The directory tree may be temporarily out-of-sync.
		//            Consider keeping separate time stamps so the web-
		//            page can indicate this discrepancy.
		fsTree.set(newDirectory(goroot, maxDirDepth))
		fallthrough
	case 1:
		// sync failed because no files changed;
		// don't change the package tree
		syncDelay.set(*syncMin) //  revert to regular sync schedule
	default:
		// sync failed because of an error - back off exponentially, but try at least once a day
		syncDelay.backoff(24 * 60)
	}
}

func usage() {
	fmt.Fprintf(os.Stderr,
		"usage: godoc package [name ...]\n"+
			"	godoc -http=:6060\n")
	flag.PrintDefaults()
	os.Exit(2)
}

func loggingHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(c *http.Conn, req *http.Request) {
		log.Stderrf("%s\t%s", c.RemoteAddr, req.URL)
		h.ServeHTTP(c, req)
	})
}

func main() {
	flag.Usage = usage
	flag.Parse()

	// Check usage: either server and no args, or command line and args
	if (*httpaddr != "") != (flag.NArg() == 0) {
		usage()
	}

	if *tabwidth < 0 {
		log.Exitf("negative tabwidth %d", *tabwidth)
	}

	initHandlers()
	readTemplates()

	if *httpaddr != "" {
		// HTTP server mode.
		var handler http.Handler = http.DefaultServeMux
		if *verbose {
			log.Stderrf("Go Documentation Server\n")
			log.Stderrf("address = %s\n", *httpaddr)
			log.Stderrf("goroot = %s\n", goroot)
			log.Stderrf("tabwidth = %d\n", *tabwidth)
			if !fsMap.IsEmpty() {
				log.Stderr("user-defined mapping:")
				fsMap.Fprint(os.Stderr)
			}
			handler = loggingHandler(handler)
		}

		registerPublicHandlers(http.DefaultServeMux)
		if *syncCmd != "" {
			http.Handle("/debug/sync", http.HandlerFunc(dosync))
		}

		// Initialize directory tree with corresponding timestamp.
		// Do it in two steps:
		// 1) set timestamp right away so that the indexer is kicked on
		fsTree.set(nil)
		// 2) compute initial directory tree in a goroutine so that launch is quick
		go func() { fsTree.set(newDirectory(goroot, maxDirDepth)) }()

		// Start sync goroutine, if enabled.
		if *syncCmd != "" && *syncMin > 0 {
			syncDelay.set(*syncMin) // initial sync delay
			go func() {
				for {
					dosync(nil, nil)
					delay, _ := syncDelay.get()
					if *verbose {
						log.Stderrf("next sync in %dmin", delay.(int))
					}
					time.Sleep(int64(delay.(int)) * 60e9)
				}
			}()
		}

		// Start indexing goroutine.
		go indexer()

		// The server may have been restarted; always wait 1sec to
		// give the forking server a chance to shut down and release
		// the http port.
		// TODO(gri): Do we still need this?
		time.Sleep(1e9)

		// Start http server.
		if err := http.ListenAndServe(*httpaddr, handler); err != nil {
			log.Exitf("ListenAndServe %s: %v", *httpaddr, err)
		}
		return
	}

	// Command line mode.
	if *html {
		packageText = packageHTML
	}

	// determine paths
	path := flag.Arg(0)
	if len(path) > 0 && path[0] == '.' {
		// assume cwd; don't assume -goroot
		cwd, _ := os.Getwd() // ignore errors
		path = pathutil.Join(cwd, path)
	}
	relpath := path
	abspath := path
	if len(path) > 0 && path[0] != '/' {
		abspath = absolutePath(path, pkgHandler.fsRoot)
	} else {
		relpath = relativePath(path)
	}

	info := pkgHandler.getPageInfo(abspath, relpath, true)

	if info.PDoc == nil && info.Dirs == nil {
		// try again, this time assume it's a command
		if len(path) > 0 && path[0] != '/' {
			abspath = absolutePath(path, cmdHandler.fsRoot)
		}
		info = cmdHandler.getPageInfo(abspath, relpath, false)
	}

	if info.PDoc != nil && flag.NArg() > 1 {
		args := flag.Args()
		info.PDoc.Filter(args[1:])
	}

	if err := packageText.Execute(info, os.Stdout); err != nil {
		log.Stderrf("packageText.Execute: %s", err)
	}
}
<|endoftext|>