<commit_before>// This package implements a provisioner for Packer that executes a
// saltstack highstate within the remote machine
package saltmasterless

import (
	"errors"
	"fmt"
	"github.com/mitchellh/packer/common"
	"github.com/mitchellh/packer/packer"
	"os"
)

const DefaultTempConfigDir = "/tmp/salt"

type Config struct {
	common.PackerConfig `mapstructure:",squash"`

	// If true, run the salt-bootstrap script
	SkipBootstrap bool   `mapstructure:"skip_bootstrap"`
	BootstrapArgs string `mapstructure:"bootstrap_args"`

	// Local path to the minion config
	MinionConfig string `mapstructure:"minion_config"`

	// Local path to the salt state tree
	LocalStateTree string `mapstructure:"local_state_tree"`

	// Local path to the salt pillar roots
	LocalPillarRoots string `mapstructure:"local_pillar_roots"`

	// Where files will be copied before moving to the /srv/salt directory
	TempConfigDir string `mapstructure:"temp_config_dir"`

	tpl *packer.ConfigTemplate
}

type Provisioner struct {
	config Config
}

func (p *Provisioner) Prepare(raws ...interface{}) error {
	md, err := common.DecodeConfig(&p.config, raws...)
	if err != nil {
		return err
	}

	p.config.tpl, err = packer.NewConfigTemplate()
	if err != nil {
		return err
	}
	p.config.tpl.UserVars = p.config.PackerUserVars

	if p.config.TempConfigDir == "" {
		p.config.TempConfigDir = DefaultTempConfigDir
	}

	// Accumulate any errors
	errs := common.CheckUnusedConfig(md)

	templates := map[string]*string{
		"bootstrap_args":     &p.config.BootstrapArgs,
		"minion_config":      &p.config.MinionConfig,
		"local_state_tree":   &p.config.LocalStateTree,
		"local_pillar_roots": &p.config.LocalPillarRoots,
		"temp_config_dir":    &p.config.TempConfigDir,
	}

	for n, ptr := range templates {
		var err error
		*ptr, err = p.config.tpl.Process(*ptr, nil)
		if err != nil {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("Error processing %s: %s", n, err))
		}
	}

	// require a salt state tree
	if p.config.LocalStateTree == "" {
		errs = packer.MultiErrorAppend(errs,
			errors.New("local_state_tree must be supplied"))
	} else {
		if _, err := os.Stat(p.config.LocalStateTree); err != nil {
			errs = packer.MultiErrorAppend(errs,
				errors.New("local_state_tree must exist and be accessible"))
		}
	}

	if p.config.LocalPillarRoots != "" {
		if _, err := os.Stat(p.config.LocalPillarRoots); err != nil {
			errs = packer.MultiErrorAppend(errs,
				errors.New("local_pillar_roots must exist and be accessible"))
		}
	}

	if p.config.MinionConfig != "" {
		if _, err := os.Stat(p.config.MinionConfig); err != nil {
			errs = packer.MultiErrorAppend(errs,
				errors.New("minion_config must exist and be accessible"))
		}
	}

	if errs != nil && len(errs.Errors) > 0 {
		return errs
	}

	return nil
}

func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error {
	var err error

	ui.Say("Provisioning with Salt...")
	if !p.config.SkipBootstrap {
		cmd := &packer.RemoteCmd{
			Command: fmt.Sprintf("wget -O - http://bootstrap.saltstack.org | sudo sh -s %s", p.config.BootstrapArgs),
		}
		ui.Message(fmt.Sprintf("Installing Salt with command %s", cmd))
		if err = cmd.StartWithUi(comm, ui); err != nil {
			return fmt.Errorf("Unable to install Salt: %d", err)
		}
	}

	ui.Message(fmt.Sprintf("Creating remote directory: %s", p.config.TempConfigDir))
	cmd := &packer.RemoteCmd{Command: fmt.Sprintf("mkdir -p %s", p.config.TempConfigDir)}
	if err = cmd.StartWithUi(comm, ui); err != nil || cmd.ExitStatus != 0 {
		if err == nil {
			err = fmt.Errorf("Bad exit status: %d", cmd.ExitStatus)
		}

		return fmt.Errorf("Error creating remote salt state directory: %s", err)
	}

	if p.config.MinionConfig != "" {
		ui.Message(fmt.Sprintf("Uploading minion config: %s", p.config.MinionConfig))
		if err = uploadMinionConfig(comm, fmt.Sprintf("%s/minion", p.config.TempConfigDir), p.config.MinionConfig); err != nil {
			return fmt.Errorf("Error uploading local minion config file to remote: %s", err)
		}

		ui.Message(fmt.Sprintf("Moving %s/minion to /etc/salt/minion", p.config.TempConfigDir))
		cmd = &packer.RemoteCmd{Command: fmt.Sprintf("sudo mv %s/minion /etc/salt/minion", p.config.TempConfigDir)}
		if err = cmd.StartWithUi(comm, ui); err != nil || cmd.ExitStatus != 0 {
			if err == nil {
				err = fmt.Errorf("Bad exit status: %d", cmd.ExitStatus)
			}

			return fmt.Errorf("Unable to move %s/minion to /etc/salt/minion: %d", p.config.TempConfigDir, err)
		}
	}

	ui.Message(fmt.Sprintf("Uploading local state tree: %s", p.config.LocalStateTree))
	if err = comm.UploadDir(fmt.Sprintf("%s/states", p.config.TempConfigDir),
		p.config.LocalStateTree, []string{".git"}); err != nil {
		return fmt.Errorf("Error uploading local state tree to remote: %s", err)
	}

	ui.Message(fmt.Sprintf("Moving %s/states to /srv/salt", p.config.TempConfigDir))
	cmd = &packer.RemoteCmd{Command: fmt.Sprintf("sudo mv %s/states /srv/salt", p.config.TempConfigDir)}
	if err = cmd.StartWithUi(comm, ui); err != nil || cmd.ExitStatus != 0 {
		if err == nil {
			err = fmt.Errorf("Bad exit status: %d", cmd.ExitStatus)
		}

		return fmt.Errorf("Unable to move %s/states to /srv/salt: %d", p.config.TempConfigDir, err)
	}

	if p.config.LocalPillarRoots != "" {
		ui.Message(fmt.Sprintf("Uploading local pillar roots: %s", p.config.LocalPillarRoots))
		if err = comm.UploadDir(fmt.Sprintf("%s/pillar", p.config.TempConfigDir),
			p.config.LocalPillarRoots, []string{".git"}); err != nil {
			return fmt.Errorf("Error uploading local pillar roots to remote: %s", err)
		}

		ui.Message(fmt.Sprintf("Moving %s/pillar to /srv/pillar", p.config.TempConfigDir))
		cmd = &packer.RemoteCmd{Command: fmt.Sprintf("sudo mv %s/pillar /srv/pillar", p.config.TempConfigDir)}
		if err = cmd.StartWithUi(comm, ui); err != nil || cmd.ExitStatus != 0 {
			if err == nil {
				err = fmt.Errorf("Bad exit status: %d", cmd.ExitStatus)
			}

			return fmt.Errorf("Unable to move %s/pillar to /srv/pillar: %d", p.config.TempConfigDir, err)
		}
	}

	ui.Message("Running highstate")
	cmd = &packer.RemoteCmd{Command: "sudo salt-call --local state.highstate -l info"}
	if err = cmd.StartWithUi(comm, ui); err != nil || cmd.ExitStatus != 0 {
		if err == nil {
			err = fmt.Errorf("Bad exit status: %d", cmd.ExitStatus)
		}

		return fmt.Errorf("Error executing highstate: %s", err)
	}

	return nil
}

func (p *Provisioner) Cancel() {
	// Just hard quit. It isn't a big deal if what we're doing keeps
	// running on the other side.
	os.Exit(0)
}

func uploadMinionConfig(comm packer.Communicator, dst string, src string) error {
	f, err := os.Open(src)
	if err != nil {
		return fmt.Errorf("Error opening minion config: %s", err)
	}
	defer f.Close()

	if err = comm.Upload(dst, f, nil); err != nil {
		return fmt.Errorf("Error uploading minion config: %s", err)
	}

	return nil
}
<commit_msg>salt provisioner: tidy up and refactor into functions<commit_after>// This package implements a provisioner for Packer that executes a
// saltstack highstate within the remote machine
package saltmasterless

import (
	"errors"
	"fmt"
	"github.com/mitchellh/packer/common"
	"github.com/mitchellh/packer/packer"
	"os"
	"path/filepath"
)

const DefaultTempConfigDir = "/tmp/salt"

type Config struct {
	common.PackerConfig `mapstructure:",squash"`

	// If true, run the salt-bootstrap script
	SkipBootstrap bool   `mapstructure:"skip_bootstrap"`
	BootstrapArgs string `mapstructure:"bootstrap_args"`

	// Local path to the minion config
	MinionConfig string `mapstructure:"minion_config"`

	// Local path to the salt state tree
	LocalStateTree string `mapstructure:"local_state_tree"`

	// Local path to the salt pillar roots
	LocalPillarRoots string `mapstructure:"local_pillar_roots"`

	// Where files will be copied before moving to the /srv/salt directory
	TempConfigDir string `mapstructure:"temp_config_dir"`

	tpl *packer.ConfigTemplate
}

type Provisioner struct {
	config Config
}

func (p *Provisioner) Prepare(raws ...interface{}) error {
	md, err := common.DecodeConfig(&p.config, raws...)
	if err != nil {
		return err
	}

	p.config.tpl, err = packer.NewConfigTemplate()
	if err != nil {
		return err
	}
	p.config.tpl.UserVars = p.config.PackerUserVars

	if p.config.TempConfigDir == "" {
		p.config.TempConfigDir = DefaultTempConfigDir
	}

	// Accumulate any errors
	errs := common.CheckUnusedConfig(md)

	templates := map[string]*string{
		"bootstrap_args":     &p.config.BootstrapArgs,
		"minion_config":      &p.config.MinionConfig,
		"local_state_tree":   &p.config.LocalStateTree,
		"local_pillar_roots": &p.config.LocalPillarRoots,
		"temp_config_dir":    &p.config.TempConfigDir,
	}

	for n, ptr := range templates {
		var err error
		*ptr, err = p.config.tpl.Process(*ptr, nil)
		if err != nil {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("Error processing %s: %s", n, err))
		}
	}

	// require a salt state tree
	if p.config.LocalStateTree == "" {
		errs = packer.MultiErrorAppend(errs,
			errors.New("local_state_tree must be supplied"))
	} else {
		if _, err := os.Stat(p.config.LocalStateTree); err != nil {
			errs = packer.MultiErrorAppend(errs,
				errors.New("local_state_tree must exist and be accessible"))
		}
	}

	if p.config.LocalPillarRoots != "" {
		if _, err := os.Stat(p.config.LocalPillarRoots); err != nil {
			errs = packer.MultiErrorAppend(errs,
				errors.New("local_pillar_roots must exist and be accessible"))
		}
	}

	if p.config.MinionConfig != "" {
		if _, err := os.Stat(p.config.MinionConfig); err != nil {
			errs = packer.MultiErrorAppend(errs,
				errors.New("minion_config must exist and be accessible"))
		}
	}

	if errs != nil && len(errs.Errors) > 0 {
		return errs
	}

	return nil
}

func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error {
	var err error
	var src, dst string

	ui.Say("Provisioning with Salt...")
	if !p.config.SkipBootstrap {
		cmd := &packer.RemoteCmd{
			Command: fmt.Sprintf("wget -O - http://bootstrap.saltstack.org | sudo sh -s %s", p.config.BootstrapArgs),
		}
		ui.Message(fmt.Sprintf("Installing Salt with command %s", cmd))
		if err = cmd.StartWithUi(comm, ui); err != nil {
			return fmt.Errorf("Unable to install Salt: %d", err)
		}
	}

	ui.Message(fmt.Sprintf("Creating remote directory: %s", p.config.TempConfigDir))
	if err := p.createDir(ui, comm, p.config.TempConfigDir); err != nil {
		return fmt.Errorf("Error creating remote salt state directory: %s", err)
	}

	if p.config.MinionConfig != "" {
		ui.Message(fmt.Sprintf("Uploading minion config: %s", p.config.MinionConfig))
		src = p.config.MinionConfig
		dst = filepath.ToSlash(filepath.Join(p.config.TempConfigDir, "minion"))
		if err = p.uploadFile(ui, comm, dst, src); err != nil {
			return fmt.Errorf("Error uploading local minion config file to remote: %s", err)
		}

		// move minion config into /etc/salt
		src = filepath.ToSlash(filepath.Join(p.config.TempConfigDir, "minion"))
		dst = "/etc/salt/minion"
		if err = p.moveFile(ui, comm, dst, src); err != nil {
			return fmt.Errorf("Unable to move %s/minion to /etc/salt/minion: %d", p.config.TempConfigDir, err)
		}
	}

	ui.Message(fmt.Sprintf("Uploading local state tree: %s", p.config.LocalStateTree))
	src = p.config.LocalStateTree
	dst = filepath.ToSlash(filepath.Join(p.config.TempConfigDir, "states"))
	if err = p.uploadDir(ui, comm, dst, src, []string{".git"}); err != nil {
		return fmt.Errorf("Error uploading local state tree to remote: %s", err)
	}

	// move state tree into /srv/salt
	src = filepath.ToSlash(filepath.Join(p.config.TempConfigDir, "states"))
	dst = "/srv/salt"
	if err = p.moveFile(ui, comm, dst, src); err != nil {
		return fmt.Errorf("Unable to move %s/states to /srv/salt: %d", p.config.TempConfigDir, err)
	}

	if p.config.LocalPillarRoots != "" {
		ui.Message(fmt.Sprintf("Uploading local pillar roots: %s", p.config.LocalPillarRoots))
		src = p.config.LocalPillarRoots
		dst = filepath.ToSlash(filepath.Join(p.config.TempConfigDir, "pillar"))
		if err = p.uploadDir(ui, comm, dst, src, []string{".git"}); err != nil {
			return fmt.Errorf("Error uploading local pillar roots to remote: %s", err)
		}

		// move pillar tree into /srv/pillar
		src = filepath.ToSlash(filepath.Join(p.config.TempConfigDir, "pillar"))
		dst = "/srv/pillar"
		if err = p.moveFile(ui, comm, dst, src); err != nil {
			return fmt.Errorf("Unable to move %s/pillar to /srv/pillar: %d", p.config.TempConfigDir, err)
		}
	}

	ui.Message("Running highstate")
	cmd := &packer.RemoteCmd{Command: "sudo salt-call --local state.highstate -l info"}
	if err = cmd.StartWithUi(comm, ui); err != nil || cmd.ExitStatus != 0 {
		if err == nil {
			err = fmt.Errorf("Bad exit status: %d", cmd.ExitStatus)
		}

		return fmt.Errorf("Error executing highstate: %s", err)
	}

	return nil
}

func (p *Provisioner) Cancel() {
	// Just hard quit. It isn't a big deal if what we're doing keeps
	// running on the other side.
	os.Exit(0)
}

func (p *Provisioner) uploadFile(ui packer.Ui, comm packer.Communicator, dst, src string) error {
	f, err := os.Open(src)
	if err != nil {
		return fmt.Errorf("Error opening: %s", err)
	}
	defer f.Close()

	if err = comm.Upload(dst, f, nil); err != nil {
		return fmt.Errorf("Error uploading %s: %s", src, err)
	}
	return nil
}

func (p *Provisioner) moveFile(ui packer.Ui, comm packer.Communicator, dst, src string) error {
	ui.Message(fmt.Sprintf("Moving %s to %s", src, dst))
	cmd := &packer.RemoteCmd{Command: fmt.Sprintf("sudo mv %s %s", src, dst)}
	if err := cmd.StartWithUi(comm, ui); err != nil || cmd.ExitStatus != 0 {
		if err == nil {
			err = fmt.Errorf("Bad exit status: %d", cmd.ExitStatus)
		}

		return fmt.Errorf("Unable to move %s/minion to /etc/salt/minion: %d", p.config.TempConfigDir, err)
	}
	return nil
}

func (p *Provisioner) createDir(ui packer.Ui, comm packer.Communicator, dir string) error {
	ui.Message(fmt.Sprintf("Creating directory: %s", dir))
	cmd := &packer.RemoteCmd{
		Command: fmt.Sprintf("mkdir -p '%s'", dir),
	}
	if err := cmd.StartWithUi(comm, ui); err != nil {
		return err
	}
	if cmd.ExitStatus != 0 {
		return fmt.Errorf("Non-zero exit status.")
	}
	return nil
}

func (p *Provisioner) uploadDir(ui packer.Ui, comm packer.Communicator, dst, src string, ignore []string) error {
	if err := p.createDir(ui, comm, dst); err != nil {
		return err
	}

	// Make sure there is a trailing "/" so that the directory isn't
	// created on the other side.
	if src[len(src)-1] != '/' {
		src = src + "/"
	}
	return comm.UploadDir(dst, src, ignore)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"hash"
	"io"
	"sort"
	"strconv"
	"strings"
	"sync/atomic"

	"github.com/montanaflynn/stats"
	"github.com/pkg/errors"
)

// SegmentStat holds stats for a single segment
type SegmentStat struct {
	SeqNum uint64 // First occurrence of this segment in the file
	Length int    // Length of segment
	Freq   int    // How many times this segment occurred in the file
}

// ParseStats holds stats about the parsed file
type ParseStats struct {
	Cutpoints   []uint64               // indices at which we have cutpoints
	SegLengths  []uint64               // lengths of segments (between cutpoints)
	SegHashes   map[string]SegmentStat // map[crypto hash of seg] -> SegmentStat
	BytesParsed uint64                 // number of bytes parsed

	// internal:
	segNum    uint64    // tracks the segment numbers we've issued
	segHasher hash.Hash // used to generate the crypto hash of segments
}

// NewParseStats returns an initialized ParseStats struct
func NewParseStats(hasher hash.Hash) *ParseStats {
	return &ParseStats{
		SegHashes: make(map[string]SegmentStat),
		segNum:    uint64(0),
		segHasher: hasher,
	}
}

// UpdateStats updates the ParseStats with stats for the specified chunk
func (s *ParseStats) UpdateStats(segment []byte) string {

	// Sprint'ing the hash sum causes an allocation that is unnecessary and
	// is completely avoidable.
	//segHash := fmt.Sprintf("%X", s.segHasher.Sum(segment))
	segHash := string(s.segHasher.Sum(segment))
	segStat, there := s.SegHashes[segHash]
	if there {
		segStat.Freq++
	} else {
		//segStat = SegmentStat{SeqNum: s.segNum, Length: len(segment), Freq: 1}
		segStat.Freq = 1
		segStat.Length = len(segment)
		segStat.SeqNum = atomic.AddUint64(&s.segNum, 1)
	}
	s.SegHashes[segHash] = segStat

	// Additional book keeping (TODO: allow this to be disabled)
	s.Cutpoints = append(s.Cutpoints, s.BytesParsed+uint64(len(segment)))
	s.SegLengths = append(s.SegLengths, uint64(len(segment)))
	s.BytesParsed += uint64(len(segment))

	return segHash
}

// Print prints the specified ParseStats on the given output (io.Writer)
//
func (s ParseStats) Print(out io.Writer) error {
	segLens := make([]float64, 0, len(s.SegLengths))
	for _, s := range s.SegLengths {
		segLens = append(segLens, float64(s))
	}

	med, err := stats.Median(segLens)
	if err != nil {
		return errors.Wrapf(err, "Failed compute median")
	}
	max, err := stats.Max(segLens)
	if err != nil {
		return errors.Wrapf(err, "Failed to compute max")
	}
	min, err := stats.Min(segLens)
	if err != nil {
		return errors.Wrapf(err, "Failed to compute min")
	}
	mea, err := stats.Mean(segLens)
	if err != nil {
		return errors.Wrapf(err, "Failed to compute mean")
	}

	dupCount := 0
	dupBytes := 0
	lenUnique := uint64(0)
	for _, segStat := range s.SegHashes {
		if segStat.Freq > 1 {
			dupCount += (segStat.Freq - 1)
			dupBytes += (segStat.Length * (segStat.Freq - 1))
		}
		lenUnique += uint64(segStat.Length)
	}

	output := struct {
		NumCutpoints    int
		MeanSegLength   float64
		MedianSegLength float64
		MaxSegLength    float64
		MinSegLength    float64
		DupSegCount     int
		DupBytes        int
		UniqueBytes     uint64
		TotalBytes      uint64
	}{
		NumCutpoints:    len(s.Cutpoints),
		MeanSegLength:   mea,
		MedianSegLength: med,
		MaxSegLength:    max,
		MinSegLength:    min,
		DupSegCount:     dupCount,
		DupBytes:        dupBytes,
		UniqueBytes:     lenUnique,
		TotalBytes:      s.BytesParsed,
	}

	marshalled, err := json.MarshalIndent(output, "", "  ")
	if err != nil {
		return errors.Wrapf(err, "Failed to marshal stats into JSON output")
	}
	fmt.Fprintln(out, string(marshalled))
	return nil
}

// PrintSegLengths prints segment lengths to the specified output separated by
// the specified separator
func (s ParseStats) PrintSegLengths(out io.Writer, sep string) error {

	lenStrings := []string{}
	for _, len := range s.SegLengths {
		lenStrings = append(lenStrings, strconv.Itoa(int(len)))
	}

	// Join our string slice.
	result := strings.Join(lenStrings, sep)
	_, err := fmt.Fprint(out, result)
	return err
}

// PrintMostFrequentSegStats prints 'n' "hottest" segments (SegmentStat)
func (s ParseStats) PrintMostFrequentSegStats(out io.Writer, n int) error {
	ss := []SegmentStat{}
	for _, s := range s.SegHashes {
		ss = append(ss, s)
	}

	sort.Sort(sort.Reverse(bySegFreq(ss)))

	for i := 0; i < n; i++ {
		marshalled, err := json.MarshalIndent(ss[i], "", "  ")
		if err != nil {
			return err
		}
		fmt.Fprintln(out, string(marshalled))
	}

	return nil
}

// SegmentStat sorted by segment frequencies
type bySegFreq []SegmentStat

func (a bySegFreq) Len() int           { return len(a) }
func (a bySegFreq) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a bySegFreq) Less(i, j int) bool { return a[i].Freq < a[j].Freq }
<commit_msg>Add method to print seg length histogram<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"hash"
	"io"
	"sort"
	"strconv"
	"strings"
	"sync/atomic"

	"github.com/codahale/hdrhistogram"
	"github.com/montanaflynn/stats"
	"github.com/pkg/errors"
)

// SegmentStat holds stats for a single segment
type SegmentStat struct {
	SeqNum uint64 // First occurrence of this segment in the file
	Length int    // Length of segment
	Freq   int    // How many times this segment occurred in the file
}

// ParseStats holds stats about the parsed file
type ParseStats struct {
	Cutpoints   []uint64               // indices at which we have cutpoints
	SegLengths  []uint64               // lengths of segments (between cutpoints)
	SegHashes   map[string]SegmentStat // map[crypto hash of seg] -> SegmentStat
	BytesParsed uint64                 // number of bytes parsed

	// internal:
	segNum    uint64    // tracks the segment numbers we've issued
	segHasher hash.Hash // used to generate the crypto hash of segments
}

// NewParseStats returns an initialized ParseStats struct
func NewParseStats(hasher hash.Hash) *ParseStats {
	return &ParseStats{
		SegHashes: make(map[string]SegmentStat),
		segNum:    uint64(0),
		segHasher: hasher,
	}
}

// UpdateStats updates the ParseStats with stats for the specified chunk
func (s *ParseStats) UpdateStats(segment []byte) string {

	// Sprint'ing the hash sum causes an allocation that is unnecessary and
	// is completely avoidable.
	//segHash := fmt.Sprintf("%X", s.segHasher.Sum(segment))
	segHash := string(s.segHasher.Sum(segment))
	segStat, there := s.SegHashes[segHash]
	if there {
		segStat.Freq++
	} else {
		//segStat = SegmentStat{SeqNum: s.segNum, Length: len(segment), Freq: 1}
		segStat.Freq = 1
		segStat.Length = len(segment)
		segStat.SeqNum = atomic.AddUint64(&s.segNum, 1)
	}
	s.SegHashes[segHash] = segStat

	// Additional book keeping (TODO: allow this to be disabled)
	s.Cutpoints = append(s.Cutpoints, s.BytesParsed+uint64(len(segment)))
	s.SegLengths = append(s.SegLengths, uint64(len(segment)))
	s.BytesParsed += uint64(len(segment))

	return segHash
}

// Print prints the specified ParseStats on the given output (io.Writer)
//
func (s ParseStats) Print(out io.Writer) error {
	segLens := make([]float64, 0, len(s.SegLengths))
	for _, s := range s.SegLengths {
		segLens = append(segLens, float64(s))
	}

	med, err := stats.Median(segLens)
	if err != nil {
		return errors.Wrapf(err, "Failed compute median")
	}
	max, err := stats.Max(segLens)
	if err != nil {
		return errors.Wrapf(err, "Failed to compute max")
	}
	min, err := stats.Min(segLens)
	if err != nil {
		return errors.Wrapf(err, "Failed to compute min")
	}
	mea, err := stats.Mean(segLens)
	if err != nil {
		return errors.Wrapf(err, "Failed to compute mean")
	}

	mostFreq := 0
	dupCount := 0
	dupBytes := 0
	lenUnique := uint64(0)
	for _, segStat := range s.SegHashes {
		if segStat.Freq > 1 {
			dupCount += (segStat.Freq - 1)
			dupBytes += (segStat.Length * (segStat.Freq - 1))
		}
		if segStat.Freq > mostFreq {
			mostFreq = segStat.Freq
		}
		lenUnique += uint64(segStat.Length)
	}

	output := struct {
		NumCutpoints    int
		MeanSegLength   float64
		MedianSegLength float64
		MaxSegLength    float64
		MinSegLength    float64
		DupSegCount     int
		DupBytes        int
		MaxSegFreq      int
		UniqueBytes     uint64
		TotalBytes      uint64
	}{
		NumCutpoints:    len(s.Cutpoints),
		MeanSegLength:   mea,
		MedianSegLength: med,
		MaxSegLength:    max,
		MinSegLength:    min,
		DupSegCount:     dupCount,
		DupBytes:        dupBytes,
		MaxSegFreq:      mostFreq,
		UniqueBytes:     lenUnique,
		TotalBytes:      s.BytesParsed,
	}

	marshalled, err := json.MarshalIndent(output, "", "  ")
	if err != nil {
		return errors.Wrapf(err, "Failed to marshal stats into JSON output")
	}
	fmt.Fprintln(out, string(marshalled))
	return nil
}

// PrintSegLengths prints segment lengths to the specified output separated by
// the specified separator
func (s ParseStats) PrintSegLengths(out io.Writer, sep string) error {

	lenStrings := []string{}
	for _, len := range s.SegLengths {
		lenStrings = append(lenStrings, strconv.Itoa(int(len)))
	}

	// Join our string slice.
	result := strings.Join(lenStrings, sep)
	_, err := fmt.Fprint(out, result)
	return err
}

// PrintMostFrequentSegStats prints 'n' "hottest" segments (SegmentStat)
func (s ParseStats) PrintMostFrequentSegStats(out io.Writer, n int) error {
	ss := []SegmentStat{}
	for _, s := range s.SegHashes {
		ss = append(ss, s)
	}

	sort.Sort(sort.Reverse(bySegFreq(ss)))

	for i := 0; i < n; i++ {
		marshalled, err := json.MarshalIndent(ss[i], "", "  ")
		if err != nil {
			return err
		}
		fmt.Fprintln(out, string(marshalled))
	}

	return nil
}

// PrintSegLengthHistogram prints histogram (bars in csv) to out
func (s ParseStats) PrintSegLengthHistogram(out io.Writer) error {
	ss := []SegmentStat{}
	for _, s := range s.SegHashes {
		ss = append(ss, s)
	}
	sort.Sort(sort.Reverse(bySegFreq(ss)))

	hist := hdrhistogram.New(int64(ss[0].Length), int64(ss[len(ss)-1].Length), 1)
	for _, s := range ss {
		hist.RecordValue(int64(s.Length))
	}
	for _, bar := range hist.Distribution() {
		fmt.Fprintf(out, "%s\n", bar.String())
	}
	return nil
}

// SegmentStat sorted by segment frequencies
type bySegFreq []SegmentStat

func (a bySegFreq) Len() int           { return len(a) }
func (a bySegFreq) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a bySegFreq) Less(i, j int) bool { return a[i].Freq < a[j].Freq }
<|endoftext|>


<commit_before>// Package store automatically configures a database to store structured information in an sql database
package store

import (
	"database/sql"
	"errors"
	"reflect"
	"strings"
	"sync"

	_ "github.com/mxk/go-sqlite/sqlite3"
)

const (
	add = iota
	get
	update
	remove
	getPage
	count
)

type field struct {
	isStruct bool
	pos      int
	name     string
}

type typeInfo struct {
	primary    int
	fields     []field
	statements []*sql.Stmt
}

type Store struct {
	db    *sql.DB
	types map[string]typeInfo
	mutex sync.Mutex
}

func New(dataSourceName string) (*Store, error) {
	db, err := sql.Open("sqlite3", dataSourceName)
	if err != nil {
		return nil, err
	}
	return &Store{
		db:    db,
		types: make(map[string]typeInfo),
	}, nil
}

func (s *Store) Close() error {
	err := s.db.Close()
	s.db = nil
	return err
}

func (s *Store) Register(i interface{}) error {
	if s.db == nil {
		return ErrDBClosed
	} else if !isPointerStruct(i) {
		return ErrNoPointerStruct
	}
	s.mutex.Lock()
	defer s.mutex.Unlock()
	return s.defineType(i)
}

func (s *Store) defineType(i interface{}) error {
	name := typeName(i)
	if _, ok := s.types[name]; ok {
		return nil
	}

	s.types[name] = typeInfo{}

	v := reflect.ValueOf(i).Elem()
	numFields := v.Type().NumField()
	fields := make([]field, 0, numFields)
	id := 0
	idType := 0

	for n := 0; n < numFields; n++ {
		f := v.Type().Field(n)
		if f.PkgPath != "" { // not exported
			continue
		}
		fieldName := f.Name
		if fn := f.Tag.Get("store"); fn != "" {
			fieldName = fn
		}
		if fieldName == "-" { // Skip field
			continue
		}
		tmp := strings.ToLower(fieldName)
		for _, tf := range fields {
			if strings.ToLower(tf.name) == tmp {
				return ErrDuplicateColumn
			}
		}
		isPointer := f.Type.Kind() == reflect.Ptr
		var iface interface{}
		if isPointer {
			iface = v.Field(n).Interface()
		} else {
			iface = v.Field(n).Addr().Interface()
		}
		isStruct := false
		if isPointerStruct(iface) {
			s.defineType(iface)
			isStruct = true
		} else if !isValidType(iface) {
			continue
		}
		if isValidKeyType(iface) {
			if idType < 3 && f.Tag.Get("key") == "1" {
				idType = 3
				id = len(fields)
			} else if idType < 2 && strings.ToLower(fieldName) == "id" {
				idType = 2
				id = len(fields)
			} else if idType < 1 {
				idType = 1
				id = len(fields)
			}
		}
		fields = append(fields, field{
			isStruct,
			n,
			fieldName,
		})
	}
	if idType == 0 {
		return ErrNoKey
	}
	s.types[name] = typeInfo{
		primary: id,
	}

	// create statements
	var (
		sqlVars, sqlParams, setSQLParams, tableVars string
		doneFirst, doneFirstNonKey                  bool
	)

	for pos, f := range fields {
		if doneFirst {
			tableVars += ", "
		} else {
			doneFirst = true
		}
		if pos != id {
			if doneFirstNonKey {
				sqlVars += ", "
				setSQLParams += ", "
				sqlParams += ", "
			} else {
				doneFirstNonKey = true
			}
		}
		var varType string
		if f.isStruct {
			varType = "INTEGER"
		} else {
			varType = getType(i, f.pos)
		}
		tableVars += "[" + f.name + "] " + varType
		if pos == id {
			tableVars += " PRIMARY KEY AUTOINCREMENT"
		} else {
			sqlVars += "[" + f.name + "]"
			setSQLParams += "[" + f.name + "] = ?"
			sqlParams += "?"
		}
	}

	statements := make([]*sql.Stmt, 6)

	sql := "CREATE TABLE IF NOT EXISTS [" + name + "](" + tableVars + ");"
	_, err := s.db.Exec(sql)
	if err != nil {
		return err
	}

	sql = "INSERT INTO [" + name + "] (" + sqlVars + ") VALUES (" + sqlParams + ");"
	stmt, err := s.db.Prepare(sql)
	if err != nil {
		return err
	}
	statements[add] = stmt

	sql = "SELECT " + sqlVars + " FROM [" + name + "] WHERE [" + fields[id].name + "] = ? LIMIT 1;"
	stmt, err = s.db.Prepare(sql)
	if err != nil {
		return err
	}
	statements[get] = stmt

	sql = "UPDATE [" + name + "] SET " + setSQLParams + " WHERE [" + fields[id].name + "] = ?;"
	stmt, err = s.db.Prepare(sql)
	if err != nil {
		return err
	}
	statements[update] = stmt

	sql = "DELETE FROM [" + name + "] WHERE [" + fields[id].name + "] = ?;"
	stmt, err = s.db.Prepare(sql)
	if err != nil {
		return err
	}
	statements[remove] = stmt

	sql = "SELECT [" + fields[id].name + "] FROM [" + name + "] ORDER BY [" + fields[id].name + "] LIMIT ? OFFSET ?;"
	stmt, err = s.db.Prepare(sql)
	if err != nil {
		return err
	}
	statements[getPage] = stmt

	sql = "SELECT COUNT(1) FROM [" + name + "];"
	stmt, err = s.db.Prepare(sql)
	if err != nil {
		return err
	}
	statements[count] = stmt

	s.types[name] = typeInfo{
		primary:    id,
		fields:     fields,
		statements: statements,
	}
	return nil
}

func (s *Store) Set(is ...interface{}) error {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	var toSet []interface{}
	for _, i := range is {
		t, ok := s.types[typeName(i)]
		if !ok {
			return ErrUnregisteredType
		}
		toSet = toSet[:0]
		err := s.set(i, &t, &toSet)
		if err != nil {
			return err
		}
	}
	return nil
}

func (s *Store) set(i interface{}, t *typeInfo, toSet *[]interface{}) error {
	for _, oi := range *toSet {
		if oi == i {
			return nil
		}
	}
	(*toSet) = append(*toSet, i)
	id := t.GetID(i)
	isUpdate := id != 0
	vars := make([]interface{}, 0, len(t.fields))
	for pos, f := range t.fields {
		if pos == t.primary {
			continue
		}
		if f.isStruct {
			ni := getFieldPointer(i, f.pos)
			nt := s.types[typeName(ni)]
			err := s.set(ni, &nt, toSet)
			if err != nil {
				return err
			}
			vars = append(vars, getField(ni, nt.fields[nt.primary].pos))
		} else {
			vars = append(vars, getField(i, f.pos))
		}
	}
	if isUpdate {
		r, err := t.statements[update].Exec(append(vars, id)...)
		if r.RowsAffected() > 0 || err != nil {
			return err
		}
		// id wasn't found, so insert...
	}
	r, err := t.statements[add].Exec(vars...)
	if err != nil {
		return err
	}
	lid, err := r.LastInsertId()
	if err != nil {
		return err
	}
	t.SetID(i, lid)
	return nil
}

func (s *Store) Get(is ...interface{}) error {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	return s.get(is...)
}
func (s *Store) get(is ...interface{}) error {
	for _, i := range is {
		t, ok := s.types[typeName(i)]
		if !ok {
			return ErrUnregisteredType
		}
		id := t.GetID(i)
		if id == 0 {
			continue
		}
		vars := make([]interface{}, 0, len(t.fields)-1)
		var toGet []interface{}
		for pos, f := range t.fields {
			if pos == t.primary {
				continue
			}
			if f.isStruct {
				ni := getFieldPointer(i, f.pos)
				nt := s.types[typeName(ni)]
				toGet = append(toGet, ni)
				vars = append(vars, getFieldPointer(ni, nt.fields[nt.primary].pos))
			} else {
				vars = append(vars, getFieldPointer(i, f.pos))
			}
		}
		row := t.statements[get].QueryRow(id)
		err := row.Scan(vars...)
		if err == sql.ErrNoRows {
			t.SetID(i, 0)
		} else if err != nil {
			return err
		} else if len(toGet) > 0 {
			if err = s.get(toGet...); err != nil {
				return err
			}
		}
	}
	return nil
}

func (s *Store) GetPage(is []interface{}, offset int) (int, error) {
	if len(is) == 0 {
		return 0, nil
	}
	s.mutex.Lock()
	defer s.mutex.Unlock()
	t := s.types[typeName(is[0])]
	rows, err := t.statements[getPage].Query(len(is), offset)
	if err != nil {
		return 0, err
	}
	defer rows.Close()
	return s.getPage(is, rows)
}

func (s *Store) getPage(is []interface{}, rows *sql.Rows) (int, error) {
	t := s.types[typeName(is[0])]
	n := 0
	for rows.Next() {
		var id int64
		if err := rows.Scan(&id); err != nil {
			return 0, err
		}
		t.SetID(is[n], id)
		n++
	}
	is = is[:n]
	if err := rows.Err(); err == sql.ErrNoRows {
		return 0, nil
	} else if err != nil {
		return 0, err
	} else if err = s.get(is...); err != nil {
		return 0, err
	}
	return n, nil
}

func (s *Store) Remove(is ...interface{}) error {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	for _, i := range is {
		t, ok := s.types[typeName(i)]
		if !ok {
			return ErrUnregisteredType
		}
		_, err := t.statements[remove].Exec(t.GetID(i))
		if err != nil {
			return err
		}
	}
	return nil
}

func (s *Store) Count(i interface{}) (int, error) {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	if !isPointerStruct(i) {
		return 0, ErrNoPointerStruct
	}
	name := typeName(i)
	stmt := s.types[name].statements[count]
	res, err := stmt.Query()
	if err != nil {
		return 0, err
	}
	num := 0
	err = res.Scan(&num)
	return num, err
}

// Errors

var (
	ErrDBClosed         = errors.New("database already closed")
	ErrNoPointerStruct  = errors.New("given variable is not a pointer to a struct")
	ErrNoKey            = errors.New("could not determine key")
	ErrDuplicateColumn  = errors.New("duplicate column name found")
	ErrUnregisteredType = errors.New("type not registered")
	ErrInvalidType      = errors.New("invalid type")
)
<commit_msg>Corrected useage of RowsAffected<commit_after>// Package store automatically configures a database to store structured information in an sql database
package store

import (
	"database/sql"
	"errors"
	"reflect"
	"strings"
	"sync"

	_ "github.com/mxk/go-sqlite/sqlite3"
)

const (
	add = iota
	get
	update
	remove
	getPage
	count
)

type field struct {
	isStruct bool
	pos      int
	name     string
}

type typeInfo struct {
	primary    int
	fields     []field
	statements []*sql.Stmt
}

type Store struct {
	db    *sql.DB
	types map[string]typeInfo
	mutex sync.Mutex
}

func New(dataSourceName string) (*Store, error) {
	db, err := sql.Open("sqlite3", dataSourceName)
	if err != nil {
		return nil, err
	}
	return &Store{
		db:    db,
		types: make(map[string]typeInfo),
	}, nil
}

func (s *Store) Close() error {
	err := s.db.Close()
	s.db = nil
	return err
}

func (s *Store) Register(i interface{}) error {
	if s.db == nil {
		return ErrDBClosed
	} else if !isPointerStruct(i) {
		return ErrNoPointerStruct
	}
	s.mutex.Lock()
	defer s.mutex.Unlock()
	return s.defineType(i)
}

func (s *Store) defineType(i interface{}) error {
	name := typeName(i)
	if _, ok := s.types[name]; ok {
		return nil
	}

	s.types[name] = typeInfo{}

	v := reflect.ValueOf(i).Elem()
	numFields := v.Type().NumField()
	fields := make([]field, 0, numFields)
	id := 0
	idType := 0

	for n := 0; n < numFields; n++ {
		f := v.Type().Field(n)
		if f.PkgPath != "" { // not exported
			continue
		}
		fieldName := f.Name
		if fn := f.Tag.Get("store"); fn != "" {
			fieldName = fn
		}
		if fieldName == "-" { // Skip field
			continue
		}
		tmp := strings.ToLower(fieldName)
		for _, tf := range fields {
			if strings.ToLower(tf.name) == tmp {
				return ErrDuplicateColumn
			}
		}
		isPointer := f.Type.Kind() == reflect.Ptr
		var iface interface{}
		if isPointer {
			iface = v.Field(n).Interface()
		} else {
			iface = v.Field(n).Addr().Interface()
		}
		isStruct := false
		if isPointerStruct(iface) {
			s.defineType(iface)
			isStruct = true
		} else if !isValidType(iface) {
			continue
		}
		if isValidKeyType(iface) {
			if idType < 3 && f.Tag.Get("key") == "1" {
				idType = 3
				id = len(fields)
			} else if idType < 2 && strings.ToLower(fieldName) == "id" {
				idType = 2
				id = len(fields)
			} else if idType < 1 {
				idType = 1
				id = len(fields)
			}
		}
		fields = append(fields, field{
			isStruct,
			n,
			fieldName,
		})
	}
	if idType == 0 {
		return ErrNoKey
	}
	s.types[name] = typeInfo{
		primary: id,
	}

	// create statements
	var (
		sqlVars, sqlParams, setSQLParams, tableVars string
		doneFirst, doneFirstNonKey                  bool
	)

	for pos, f := range fields {
		if doneFirst {
			tableVars += ", "
		} else {
			doneFirst = true
		}
		if pos != id {
			if doneFirstNonKey {
				sqlVars += ", "
				setSQLParams += ", "
				sqlParams += ", "
			} else {
				doneFirstNonKey = true
			}
		}
		var varType string
		if f.isStruct {
			varType = "INTEGER"
		} else {
			varType = getType(i, f.pos)
		}
		tableVars += "[" + f.name + "] " + varType
		if pos == id {
			tableVars += " PRIMARY KEY AUTOINCREMENT"
		} else {
			sqlVars += "[" + f.name + "]"
			setSQLParams += "[" + f.name + "] = ?"
			sqlParams += "?"
		}
	}

	statements := make([]*sql.Stmt, 6)

	sql := "CREATE TABLE IF NOT EXISTS [" + name + "](" + tableVars + ");"
	_, err := s.db.Exec(sql)
	if err != nil {
		return err
	}

	sql = "INSERT INTO [" + name + "] (" + sqlVars + ") VALUES (" + sqlParams + ");"
	stmt, err := s.db.Prepare(sql)
	if err != nil {
		return err
	}
	statements[add] = stmt

	sql = "SELECT " + sqlVars + " FROM [" + name + "] WHERE [" + fields[id].name + "] = ? LIMIT 1;"
	stmt, err = s.db.Prepare(sql)
	if err != nil {
		return err
	}
	statements[get] = stmt

	sql = "UPDATE [" + name + "] SET " + setSQLParams + " WHERE [" + fields[id].name + "] = ?;"
	stmt, err = s.db.Prepare(sql)
	if err != nil {
		return err
	}
	statements[update] = stmt

	sql = "DELETE FROM [" + name + "] WHERE [" + fields[id].name + "] = ?;"
	stmt, err = s.db.Prepare(sql)
	if err != nil {
		return err
	}
	statements[remove] = stmt

	sql = "SELECT [" + fields[id].name + "] FROM [" + name + "] ORDER BY [" + fields[id].name + "] LIMIT ? OFFSET ?;"
	stmt, err = s.db.Prepare(sql)
	if err != nil {
		return err
	}
	statements[getPage] = stmt

	sql = "SELECT COUNT(1) FROM [" + name + "];"
	stmt, err = s.db.Prepare(sql)
	if err != nil {
		return err
	}
	statements[count] = stmt

	s.types[name] = typeInfo{
		primary:    id,
		fields:     fields,
		statements: statements,
	}
	return nil
}

func (s *Store) Set(is ...interface{}) error {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	var toSet []interface{}
	for _, i := range is {
		t, ok := s.types[typeName(i)]
		if !ok {
			return ErrUnregisteredType
		}
		toSet = toSet[:0]
		err := s.set(i, &t, &toSet)
		if err != nil {
			return err
		}
	}
	return nil
}

func (s *Store) set(i interface{}, t *typeInfo, toSet *[]interface{}) error {
	for _, oi := range *toSet {
		if oi == i {
			return nil
		}
	}
	(*toSet) = append(*toSet, i)
	id := t.GetID(i)
	isUpdate := id != 0
	vars := make([]interface{}, 0, len(t.fields))
	for pos, f := range t.fields {
		if pos == t.primary {
			continue
		}
		if f.isStruct {
			ni := getFieldPointer(i, f.pos)
			nt := s.types[typeName(ni)]
			err := s.set(ni, &nt, toSet)
			if err != nil {
				return err
			}
			vars = append(vars, getField(ni, nt.fields[nt.primary].pos))
		} else {
			vars = append(vars, getField(i, f.pos))
		}
	}
	if isUpdate {
		r, err := t.statements[update].Exec(append(vars, id)...)
		if err != nil {
			return err
		}
		if ra, err := r.RowsAffected(); err != nil {
			return err
		} else if ra > 0 {
			return nil
		}
		// id wasn't found, so insert...
	}
	r, err := t.statements[add].Exec(vars...)
	if err != nil {
		return err
	}
	lid, err := r.LastInsertId()
	if err != nil {
		return err
	}
	t.SetID(i, lid)
	return nil
}

func (s *Store) Get(is ...interface{}) error {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	return s.get(is...)
}
func (s *Store) get(is ...interface{}) error {
	for _, i := range is {
		t, ok := s.types[typeName(i)]
		if !ok {
			return ErrUnregisteredType
		}
		id := t.GetID(i)
		if id == 0 {
			continue
		}
		vars := make([]interface{}, 0, len(t.fields)-1)
		var toGet []interface{}
		for pos, f := range t.fields {
			if pos == t.primary {
				continue
			}
			if f.isStruct {
				ni := getFieldPointer(i, f.pos)
				nt := s.types[typeName(ni)]
				toGet = append(toGet, ni)
				vars = append(vars, getFieldPointer(ni, nt.fields[nt.primary].pos))
			} else {
				vars = append(vars, getFieldPointer(i, f.pos))
			}
		}
		row := t.statements[get].QueryRow(id)
		err := row.Scan(vars...)
		if err == sql.ErrNoRows {
			t.SetID(i, 0)
		} else if err != nil {
			return err
		} else if len(toGet) > 0 {
			if err = s.get(toGet...); err != nil {
				return err
			}
		}
	}
	return nil
}

func (s *Store) GetPage(is []interface{}, offset int) (int, error) {
	if len(is) == 0 {
		return 0, nil
	}
	s.mutex.Lock()
	defer s.mutex.Unlock()
	t := s.types[typeName(is[0])]
	rows, err := t.statements[getPage].Query(len(is), offset)
	if err != nil {
		return 0, err
	}
	defer rows.Close()
	return s.getPage(is, rows)
}

func (s *Store) getPage(is []interface{}, rows *sql.Rows) (int, error) {
	t := s.types[typeName(is[0])]
	n := 0
	for rows.Next() {
		var id int64
		if err := rows.Scan(&id); err != nil {
			return 0, err
		}
		t.SetID(is[n], id)
		n++
	}
	is = is[:n]
	if err := rows.Err(); err == sql.ErrNoRows {
		return 0, nil
	} else if err != nil {
		return 0, err
	} else if err = s.get(is...); err != nil {
		return 0, err
	}
	return n, nil
}

func (s *Store) Remove(is ...interface{}) error {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	for _, i := range is {
		t, ok := s.types[typeName(i)]
		if !ok {
			return ErrUnregisteredType
		}
		_, err := t.statements[remove].Exec(t.GetID(i))
		if err != nil {
			return err
		}
	}
	return nil
}

func (s *Store) Count(i interface{}) (int, error) {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	if !isPointerStruct(i) {
		return 0, ErrNoPointerStruct
	}
	name := typeName(i)
	stmt := s.types[name].statements[count]
	res, err := stmt.Query()
	if err != nil {
		return 0, err
	}
	num := 0
	err = res.Scan(&num)
	return num, err
}

// Errors

var (
	ErrDBClosed         = errors.New("database already closed")
	ErrNoPointerStruct  = errors.New("given variable is not a pointer to a struct")
	ErrNoKey            = errors.New("could not determine key")
	ErrDuplicateColumn  = errors.New("duplicate column name found")
	ErrUnregisteredType = errors.New("type not registered")
	ErrInvalidType      = errors.New("invalid type")
)
<|endoftext|>


<commit_before>// Copyright (c) 2016-2017 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package intdataplane

import (
	log "github.com/sirupsen/logrus"

	"github.com/projectcalico/felix/proto"
	"github.com/projectcalico/libcalico-go/lib/set"
)

// endpointStatusCombiner combines the status reports of endpoints from the IPv4 and IPv6
// endpoint managers.  Where conflicts occur, it reports the "worse" status.
type endpointStatusCombiner struct {
	ipVersionToStatuses map[uint8]map[interface{}]string
	dirtyIDs            set.Set
	fromDataplane       chan interface{}
}

func newEndpointStatusCombiner(fromDataplane chan interface{}, ipv6Enabled bool) *endpointStatusCombiner {
	e := &endpointStatusCombiner{
		ipVersionToStatuses: map[uint8]map[interface{}]string{},
		dirtyIDs:            set.New(),
		fromDataplane:       fromDataplane,
	}

	// IPv4 is always enabled.
	e.ipVersionToStatuses[4] = map[interface{}]string{}
	if ipv6Enabled {
		// If IPv6 is enabled, track the IPv6 state too.  We use the presence of this
		// extra map to trigger merging.
		e.ipVersionToStatuses[6] = map[interface{}]string{}
	}
	return e
}

func (e *endpointStatusCombiner) OnEndpointStatusUpdate(
	ipVersion uint8,
	id interface{}, // proto.HostEndpointID or proto.WorkloadEndpointID
	status string,
) {
	log.WithFields(log.Fields{
		"ipVersion": ipVersion,
		"workload":  id,
		"status":    status,
	}).Info("Storing endpoint status update")
	e.dirtyIDs.Add(id)
	if status == "" {
		delete(e.ipVersionToStatuses[ipVersion], id)
	} else {
		e.ipVersionToStatuses[ipVersion][id] = status
	}
}

func (e *endpointStatusCombiner) Apply() {
	e.dirtyIDs.Iter(func(id interface{}) error {
		statusToReport := ""
		logCxt := log.WithField("id", id)
		for ipVer, statuses := range e.ipVersionToStatuses {
			status := statuses[id]
			logCxt := logCxt.WithField("ipVersion", ipVer).WithField("status", status)
			if status == "error" {
				logCxt.Warn("Endpoint is in error, will report error")
				statusToReport = "error"
			} else if status == "down" && statusToReport != "error" {
				logCxt.Info("Endpoint down for at least one IP version")
				statusToReport = "down"
			} else if status == "up" && statusToReport == "" {
				logCxt.Info("Endpoint up for at least one IP version")
				statusToReport = "up"
			}
		}
		if statusToReport == "" {
			logCxt.Info("Reporting endpoint removed.")
			switch id := id.(type) {
			case proto.WorkloadEndpointID:
				e.fromDataplane <- &proto.WorkloadEndpointStatusRemove{
					Id: &id,
				}
			case proto.HostEndpointID:
				e.fromDataplane <- &proto.HostEndpointStatusRemove{
					Id: &id,
				}
			}
		} else {
			logCxt.WithField("status", statusToReport).Info("Reporting combined status.")
			switch id := id.(type) {
			case proto.WorkloadEndpointID:
				e.fromDataplane <- &proto.WorkloadEndpointStatusUpdate{
					Id: &id,
					Status: &proto.EndpointStatus{
						Status: statusToReport,
					},
				}
			case proto.HostEndpointID:
				e.fromDataplane <- &proto.HostEndpointStatusUpdate{
					Id: &id,
					Status: &proto.EndpointStatus{
						Status: statusToReport,
					},
				}
			}
		}
		return set.RemoveItem
	})
}
<commit_msg>Downgrade status combiner log; spammy when interface is removed.<commit_after>// Copyright (c) 2016-2017,2020 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package intdataplane

import (
	log "github.com/sirupsen/logrus"

	"github.com/projectcalico/felix/proto"
	"github.com/projectcalico/libcalico-go/lib/set"
)

// endpointStatusCombiner combines the status reports of endpoints from the IPv4 and IPv6
// endpoint managers.  Where conflicts occur, it reports the "worse" status.
type endpointStatusCombiner struct {
	ipVersionToStatuses map[uint8]map[interface{}]string
	dirtyIDs            set.Set
	fromDataplane       chan interface{}
}

func newEndpointStatusCombiner(fromDataplane chan interface{}, ipv6Enabled bool) *endpointStatusCombiner {
	e := &endpointStatusCombiner{
		ipVersionToStatuses: map[uint8]map[interface{}]string{},
		dirtyIDs:            set.New(),
		fromDataplane:       fromDataplane,
	}

	// IPv4 is always enabled.
	e.ipVersionToStatuses[4] = map[interface{}]string{}
	if ipv6Enabled {
		// If IPv6 is enabled, track the IPv6 state too.  We use the presence of this
		// extra map to trigger merging.
		e.ipVersionToStatuses[6] = map[interface{}]string{}
	}
	return e
}

func (e *endpointStatusCombiner) OnEndpointStatusUpdate(
	ipVersion uint8,
	id interface{}, // proto.HostEndpointID or proto.WorkloadEndpointID
	status string,
) {
	log.WithFields(log.Fields{
		"ipVersion": ipVersion,
		"workload":  id,
		"status":    status,
	}).Info("Storing endpoint status update")
	e.dirtyIDs.Add(id)
	if status == "" {
		delete(e.ipVersionToStatuses[ipVersion], id)
	} else {
		e.ipVersionToStatuses[ipVersion][id] = status
	}
}

func (e *endpointStatusCombiner) Apply() {
	e.dirtyIDs.Iter(func(id interface{}) error {
		statusToReport := ""
		logCxt := log.WithField("id", id)
		for ipVer, statuses := range e.ipVersionToStatuses {
			status := statuses[id]
			logCxt := logCxt.WithField("ipVersion", ipVer).WithField("status", status)
			if status == "error" {
				logCxt.Info("Endpoint is in error, will report error")
				statusToReport = "error"
			} else if status == "down" && statusToReport != "error" {
				logCxt.Info("Endpoint down for at least one IP version")
				statusToReport = "down"
			} else if status == "up" && statusToReport == "" {
				logCxt.Info("Endpoint up for at least one IP version")
				statusToReport = "up"
			}
		}
		if statusToReport == "" {
			logCxt.Info("Reporting endpoint removed.")
			switch id := id.(type) {
			case proto.WorkloadEndpointID:
				e.fromDataplane <- &proto.WorkloadEndpointStatusRemove{
					Id: &id,
				}
			case proto.HostEndpointID:
				e.fromDataplane <- &proto.HostEndpointStatusRemove{
					Id: &id,
				}
			}
		} else {
			logCxt.WithField("status", statusToReport).Info("Reporting combined status.")
			switch id := id.(type) {
			case proto.WorkloadEndpointID:
				e.fromDataplane <- &proto.WorkloadEndpointStatusUpdate{
					Id: &id,
					Status: &proto.EndpointStatus{
						Status: statusToReport,
					},
				}
			case proto.HostEndpointID:
				e.fromDataplane <- &proto.HostEndpointStatusUpdate{
					Id: &id,
					Status: &proto.EndpointStatus{
						Status: statusToReport,
					},
				}
			}
		}
		return set.RemoveItem
	})
}
<|endoftext|>


<commit_before>package keyvalue

import (
	. "github.com/janelia-flyem/go/gocheck"
	"testing"

	"github.com/janelia-flyem/dvid/datastore"
	"github.com/janelia-flyem/dvid/dvid"
	"github.com/janelia-flyem/dvid/server"
)

// Hook up gocheck into the "go test" runner.
func Test(t *testing.T) { TestingT(t) }

type DataSuite struct {
	dir     string
	service *server.Service
	head    dvid.UUID
}

var _ = Suite(&DataSuite{})

// This will setup a new datastore and open it up, keeping the UUID and
// service pointer in the DataSuite.
func (suite *DataSuite) SetUpSuite(c *C) {
	// Make a temporary testing directory that will be auto-deleted after testing.
	suite.dir = c.MkDir()

	// Create a new datastore.
	err := datastore.Init(suite.dir, true, dvid.Config{})
	c.Assert(err, IsNil)

	// Open the datastore
	suite.service, err = server.OpenDatastore(suite.dir)
	c.Assert(err, IsNil)
}

func (suite *DataSuite) TearDownSuite(c *C) {
	suite.service.Shutdown()
}

// Make sure new keyvalue data have different IDs.
func (suite *DataSuite) TestNewDataDifferent(c *C) {
	// Create a new dataset
	root, _, err := suite.service.NewDataset()
	c.Assert(err, IsNil)

	// Add data
	config := dvid.NewConfig()
	config.SetVersioned(true)

	err = suite.service.NewData(root, "keyvalue", "kv1", config)
	c.Assert(err, IsNil)

	dataservice1, err := suite.service.DataService(root, "kv1")
	c.Assert(err, IsNil)

	err = suite.service.NewData(root, "keyvalue", "kv2", config)
	c.Assert(err, IsNil)

	dataservice2, err := suite.service.DataService(root, "kv2")
	c.Assert(err, IsNil)

	data1, ok := dataservice1.(*Data)
	c.Assert(ok, Equals, true)

	data2, ok := dataservice2.(*Data)
	c.Assert(ok, Equals, true)

	c.Assert(data1.DsetID, Equals, data2.DsetID)
	c.Assert(data1.ID, Not(Equals), data2.ID)
}

func (suite *DataSuite) TestRoundTrip(c *C) {
	root, _, err := suite.service.NewDataset()
	c.Assert(err, IsNil)

	config := dvid.NewConfig()
	config.SetVersioned(true)

	err = suite.service.NewData(root, "keyvalue", "kv", config)
	c.Assert(err, IsNil)

	kvservice, err := suite.service.DataService(root, "kv")
	c.Assert(err, IsNil)

	kvdata, ok := kvservice.(*Data)
	if !ok {
		c.Errorf("Can't cast keyservice data service into Data\n")
	}

	keyStr := "testkey"
	value := []byte("I like Japan and this is some unicode: \u65e5\u672c\u8a9e")

	err = kvdata.PutData(root, keyStr, value)
	c.Assert(err, IsNil)

	retrieved, err := kvdata.GetData(root, keyStr)
	c.Assert(err, IsNil)

	c.Assert(retrieved, DeepEquals, value)
}
<commit_msg>Fix keyvalue test<commit_after>package keyvalue

import (
	"testing"
	. "github.com/janelia-flyem/go/gocheck"

	"github.com/janelia-flyem/dvid/datastore"
	"github.com/janelia-flyem/dvid/dvid"
	"github.com/janelia-flyem/dvid/server"
)

// Hook up gocheck into the "go test" runner.
func Test(t *testing.T) { TestingT(t) }

type DataSuite struct {
	dir     string
	service *server.Service
	head    dvid.UUID
}

var _ = Suite(&DataSuite{})

// This will setup a new datastore and open it up, keeping the UUID and
// service pointer in the DataSuite.
func (suite *DataSuite) SetUpSuite(c *C) {
	// Make a temporary testing directory that will be auto-deleted after testing.
	suite.dir = c.MkDir()

	// Create a new datastore.
	err := datastore.Init(suite.dir, true, dvid.Config{})
	c.Assert(err, IsNil)

	// Open the datastore
	suite.service, err = server.OpenDatastore(suite.dir)
	c.Assert(err, IsNil)
}

func (suite *DataSuite) TearDownSuite(c *C) {
	suite.service.Shutdown()
}

// Make sure new keyvalue data have different IDs.
func (suite *DataSuite) TestNewDataDifferent(c *C) {
	// Create a new dataset
	root, _, err := suite.service.NewDataset()
	c.Assert(err, IsNil)

	// Add data
	config := dvid.NewConfig()
	config.SetVersioned(true)

	err = suite.service.NewData(root, "keyvalue", "kv1", config)
	c.Assert(err, IsNil)

	dataservice1, err := suite.service.DataService(root, "kv1")
	c.Assert(err, IsNil)

	err = suite.service.NewData(root, "keyvalue", "kv2", config)
	c.Assert(err, IsNil)

	dataservice2, err := suite.service.DataService(root, "kv2")
	c.Assert(err, IsNil)

	data1, ok := dataservice1.(*Data)
	c.Assert(ok, Equals, true)

	data2, ok := dataservice2.(*Data)
	c.Assert(ok, Equals, true)

	c.Assert(data1.DsetID, Equals, data2.DsetID)
	c.Assert(data1.ID, Not(Equals), data2.ID)
}

func (suite *DataSuite) TestRoundTrip(c *C) {
	root, _, err := suite.service.NewDataset()
	c.Assert(err, IsNil)

	config := dvid.NewConfig()
	config.SetVersioned(true)

	err = suite.service.NewData(root, "keyvalue", "kv", config)
	c.Assert(err, IsNil)

	kvservice, err := suite.service.DataService(root, "kv")
	c.Assert(err, IsNil)

	kvdata, ok := kvservice.(*Data)
	if !ok {
		c.Errorf("Can't cast keyservice data service into Data\n")
	}

	keyStr := "testkey"
	value := []byte("I like Japan and this is some unicode: \u65e5\u672c\u8a9e")

	err = kvdata.PutData(root, keyStr, value)
	c.Assert(err, IsNil)

	retrieved, found, err := kvdata.GetData(root, keyStr)
	c.Assert(err, IsNil)
	c.Assert(found, Equals, true)

	c.Assert(retrieved, DeepEquals, value)
}
<|endoftext|>


<commit_before>// +build cgo,linux

/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cadvisor

import (
	"flag"
	"fmt"
	"net"
	"net/http"
	"os"
	"strconv"
	"time"

	"github.com/golang/glog"
	"github.com/google/cadvisor/cache/memory"
	cadvisormetrics "github.com/google/cadvisor/container"
	"github.com/google/cadvisor/events"
	cadvisorfs "github.com/google/cadvisor/fs"
	cadvisorhttp "github.com/google/cadvisor/http"
	cadvisorapi "github.com/google/cadvisor/info/v1"
	cadvisorapiv2 "github.com/google/cadvisor/info/v2"
	"github.com/google/cadvisor/manager"
	"github.com/google/cadvisor/metrics"
	"github.com/google/cadvisor/utils/sysfs"
	"k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/kubernetes/pkg/kubelet/types"
)

type cadvisorClient struct {
	runtime  string
	rootPath string
	manager.Manager
}

var _ Interface = new(cadvisorClient)

// TODO(vmarmol): Make configurable.
// The amount of time for which to keep stats in memory.
const statsCacheDuration = 2 * time.Minute
const maxHousekeepingInterval = 15 * time.Second
const defaultHousekeepingInterval = 10 * time.Second
const allowDynamicHousekeeping = true

func init() {
	// Override cAdvisor flag defaults.
	flagOverrides := map[string]string{
		// Override the default cAdvisor housekeeping interval.
		"housekeeping_interval": defaultHousekeepingInterval.String(),
		// Disable event storage by default.
		"event_storage_event_limit": "default=0",
		"event_storage_age_limit":   "default=0",
	}
	for name, defaultValue := range flagOverrides {
		if f := flag.Lookup(name); f != nil {
			f.DefValue = defaultValue
			f.Value.Set(defaultValue)
		} else {
			glog.Errorf("Expected cAdvisor flag %q not found", name)
		}
	}
}

func containerLabels(c *cadvisorapi.ContainerInfo) map[string]string {
	set := map[string]string{metrics.LabelID: c.Name}
	if len(c.Aliases) > 0 {
		set[metrics.LabelName] = c.Aliases[0]
	}
	if image := c.Spec.Image; len(image) > 0 {
		set[metrics.LabelImage] = image
	}
	if v, ok := c.Spec.Labels[types.KubernetesPodNameLabel]; ok {
		set["pod_name"] = v
	}
	if v, ok := c.Spec.Labels[types.KubernetesPodNamespaceLabel]; ok {
		set["namespace"] = v
	}
	if v, ok := c.Spec.Labels[types.KubernetesContainerNameLabel]; ok {
		set["container_name"] = v
	}
	return set
}

// New creates a cAdvisor and exports its API on the specified port if port > 0.
func New(address string, port uint, runtime string, rootPath string) (Interface, error) {
	sysFs := sysfs.NewRealSysFs()

	// Create and start the cAdvisor container manager.
	m, err := manager.New(memory.New(statsCacheDuration, nil), sysFs, maxHousekeepingInterval, allowDynamicHousekeeping, cadvisormetrics.MetricSet{cadvisormetrics.NetworkTcpUsageMetrics: struct{}{}, cadvisormetrics.NetworkUdpUsageMetrics: struct{}{}}, http.DefaultClient)
	if err != nil {
		return nil, err
	}

	if _, err := os.Stat(rootPath); err != nil {
		if os.IsNotExist(err) {
			return nil, fmt.Errorf("rootDirectory %q does not exist", rootPath)
		} else {
			return nil, fmt.Errorf("failed to Stat %q: %v", rootPath, err)
		}
	}

	cadvisorClient := &cadvisorClient{
		runtime:  runtime,
		rootPath: rootPath,
		Manager:  m,
	}

	err = cadvisorClient.exportHTTP(address, port)
	if err != nil {
		return nil, err
	}
	return cadvisorClient, nil
}

func (cc *cadvisorClient) Start() error {
	return cc.Manager.Start()
}

func (cc *cadvisorClient) exportHTTP(address string, port uint) error {
	// Register the handlers regardless as this registers the prometheus
	// collector properly.
	mux := http.NewServeMux()
	err := cadvisorhttp.RegisterHandlers(mux, cc, "", "", "", "")
	if err != nil {
		return err
	}

	cadvisorhttp.RegisterPrometheusHandler(mux, cc, "/metrics", containerLabels)

	// Only start the http server if port > 0
	if port > 0 {
		serv := &http.Server{
			Addr:    net.JoinHostPort(address, strconv.Itoa(int(port))),
			Handler: mux,
		}

		// TODO(vmarmol): Remove this when the cAdvisor port is once again free.
		// If export failed, retry in the background until we are able to bind.
		// This allows an existing cAdvisor to be killed before this one registers.
		go func() {
			defer runtime.HandleCrash()

			err := serv.ListenAndServe()
			for err != nil {
				glog.Infof("Failed to register cAdvisor on port %d, retrying. Error: %v", port, err)
				time.Sleep(time.Minute)
				err = serv.ListenAndServe()
			}
		}()
	}

	return nil
}

func (cc *cadvisorClient) ContainerInfo(name string, req *cadvisorapi.ContainerInfoRequest) (*cadvisorapi.ContainerInfo, error) {
	return cc.GetContainerInfo(name, req)
}

func (cc *cadvisorClient) ContainerInfoV2(name string, options cadvisorapiv2.RequestOptions) (map[string]cadvisorapiv2.ContainerInfo, error) {
	return cc.GetContainerInfoV2(name, options)
}

func (cc *cadvisorClient) VersionInfo() (*cadvisorapi.VersionInfo, error) {
	return cc.GetVersionInfo()
}

func (cc *cadvisorClient) SubcontainerInfo(name string, req *cadvisorapi.ContainerInfoRequest) (map[string]*cadvisorapi.ContainerInfo, error) {
	infos, err := cc.SubcontainersInfo(name, req)
	if err != nil && len(infos) == 0 {
		return nil, err
	}

	result := make(map[string]*cadvisorapi.ContainerInfo, len(infos))
	for _, info := range infos {
		result[info.Name] = info
	}
	return result, err
}

func (cc *cadvisorClient) MachineInfo() (*cadvisorapi.MachineInfo, error) {
	return cc.GetMachineInfo()
}

func (cc *cadvisorClient) ImagesFsInfo() (cadvisorapiv2.FsInfo, error) {
	var label string

	switch cc.runtime {
	case "docker":
		label = cadvisorfs.LabelDockerImages
	case "rkt":
		label = cadvisorfs.LabelRktImages
	default:
		return cadvisorapiv2.FsInfo{}, fmt.Errorf("ImagesFsInfo: unknown runtime: %v", cc.runtime)
	}

	return cc.getFsInfo(label)
}

func (cc *cadvisorClient) RootFsInfo() (cadvisorapiv2.FsInfo, error) {
	return cc.GetDirFsInfo(cc.rootPath)
}

func (cc *cadvisorClient) getFsInfo(label string) (cadvisorapiv2.FsInfo, error) {
	res, err := cc.GetFsInfo(label)
	if err != nil {
		return cadvisorapiv2.FsInfo{}, err
	}
	if len(res) == 0 {
		return cadvisorapiv2.FsInfo{}, fmt.Errorf("failed to find information for the filesystem labeled %q", label)
	}
	// TODO(vmarmol): Handle this better when a label has more than one image filesystem.
	if len(res) > 1 {
		glog.Warningf("More than one filesystem labeled %q: %#v. Only using the first one", label, res)
	}

	return res[0], nil
}

func (cc *cadvisorClient) WatchEvents(request *events.Request) (*events.EventChannel, error) {
	return cc.WatchForEvents(request)
}

// HasDedicatedImageFs returns true if the imagefs has a dedicated device.
func (cc *cadvisorClient) HasDedicatedImageFs() (bool, error) {
	imageFsInfo, err := cc.ImagesFsInfo()
	if err != nil {
		return false, err
	}
	rootFsInfo, err := cc.RootFsInfo()
	if err != nil {
		return false, err
	}
	return imageFsInfo.Device != rootFsInfo.Device, nil
}
<commit_msg>Create the directory for cadvisor if needed<commit_after>// +build cgo,linux

/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cadvisor

import (
	"flag"
	"fmt"
	"net"
	"net/http"
	"os"
	"path"
	"strconv"
	"time"

	"github.com/golang/glog"
	"github.com/google/cadvisor/cache/memory"
	cadvisormetrics "github.com/google/cadvisor/container"
	"github.com/google/cadvisor/events"
	cadvisorfs "github.com/google/cadvisor/fs"
	cadvisorhttp "github.com/google/cadvisor/http"
	cadvisorapi "github.com/google/cadvisor/info/v1"
	cadvisorapiv2 "github.com/google/cadvisor/info/v2"
	"github.com/google/cadvisor/manager"
	"github.com/google/cadvisor/metrics"
	"github.com/google/cadvisor/utils/sysfs"
	"k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/kubernetes/pkg/kubelet/types"
)

type cadvisorClient struct {
	runtime  string
	rootPath string
	manager.Manager
}

var _ Interface = new(cadvisorClient)

// TODO(vmarmol): Make configurable.
// The amount of time for which to keep stats in memory.
const statsCacheDuration = 2 * time.Minute
const maxHousekeepingInterval = 15 * time.Second
const defaultHousekeepingInterval = 10 * time.Second
const allowDynamicHousekeeping = true

func init() {
	// Override cAdvisor flag defaults.
	flagOverrides := map[string]string{
		// Override the default cAdvisor housekeeping interval.
		"housekeeping_interval": defaultHousekeepingInterval.String(),
		// Disable event storage by default.
		"event_storage_event_limit": "default=0",
		"event_storage_age_limit":   "default=0",
	}
	for name, defaultValue := range flagOverrides {
		if f := flag.Lookup(name); f != nil {
			f.DefValue = defaultValue
			f.Value.Set(defaultValue)
		} else {
			glog.Errorf("Expected cAdvisor flag %q not found", name)
		}
	}
}

func containerLabels(c *cadvisorapi.ContainerInfo) map[string]string {
	set := map[string]string{metrics.LabelID: c.Name}
	if len(c.Aliases) > 0 {
		set[metrics.LabelName] = c.Aliases[0]
	}
	if image := c.Spec.Image; len(image) > 0 {
		set[metrics.LabelImage] = image
	}
	if v, ok := c.Spec.Labels[types.KubernetesPodNameLabel]; ok {
		set["pod_name"] = v
	}
	if v, ok := c.Spec.Labels[types.KubernetesPodNamespaceLabel]; ok {
		set["namespace"] = v
	}
	if v, ok := c.Spec.Labels[types.KubernetesContainerNameLabel]; ok {
		set["container_name"] = v
	}
	return set
}

// New creates a cAdvisor and exports its API on the specified port if port > 0.
func New(address string, port uint, runtime string, rootPath string) (Interface, error) {
	sysFs := sysfs.NewRealSysFs()

	// Create and start the cAdvisor container manager.
	m, err := manager.New(memory.New(statsCacheDuration, nil), sysFs, maxHousekeepingInterval, allowDynamicHousekeeping, cadvisormetrics.MetricSet{cadvisormetrics.NetworkTcpUsageMetrics: struct{}{}, cadvisormetrics.NetworkUdpUsageMetrics: struct{}{}}, http.DefaultClient)
	if err != nil {
		return nil, err
	}

	if _, err := os.Stat(rootPath); err != nil {
		if os.IsNotExist(err) {
			if err := os.MkdirAll(path.Clean(rootPath), 0750); err != nil {
				return nil, fmt.Errorf("error creating root directory %q: %v", rootPath, err)
			}
		} else {
			return nil, fmt.Errorf("failed to Stat %q: %v", rootPath, err)
		}
	}

	cadvisorClient := &cadvisorClient{
		runtime:  runtime,
		rootPath: rootPath,
		Manager:  m,
	}

	err = cadvisorClient.exportHTTP(address, port)
	if err != nil {
		return nil, err
	}
	return cadvisorClient, nil
}

func (cc *cadvisorClient) Start() error {
	return cc.Manager.Start()
}

func (cc *cadvisorClient) exportHTTP(address string, port uint) error {
	// Register the handlers regardless as this registers the prometheus
	// collector properly.
	mux := http.NewServeMux()
	err := cadvisorhttp.RegisterHandlers(mux, cc, "", "", "", "")
	if err != nil {
		return err
	}

	cadvisorhttp.RegisterPrometheusHandler(mux, cc, "/metrics", containerLabels)

	// Only start the http server if port > 0
	if port > 0 {
		serv := &http.Server{
			Addr:    net.JoinHostPort(address, strconv.Itoa(int(port))),
			Handler: mux,
		}

		// TODO(vmarmol): Remove this when the cAdvisor port is once again free.
		// If export failed, retry in the background until we are able to bind.
		// This allows an existing cAdvisor to be killed before this one registers.
		go func() {
			defer runtime.HandleCrash()

			err := serv.ListenAndServe()
			for err != nil {
				glog.Infof("Failed to register cAdvisor on port %d, retrying. Error: %v", port, err)
				time.Sleep(time.Minute)
				err = serv.ListenAndServe()
			}
		}()
	}

	return nil
}

func (cc *cadvisorClient) ContainerInfo(name string, req *cadvisorapi.ContainerInfoRequest) (*cadvisorapi.ContainerInfo, error) {
	return cc.GetContainerInfo(name, req)
}

func (cc *cadvisorClient) ContainerInfoV2(name string, options cadvisorapiv2.RequestOptions) (map[string]cadvisorapiv2.ContainerInfo, error) {
	return cc.GetContainerInfoV2(name, options)
}

func (cc *cadvisorClient) VersionInfo() (*cadvisorapi.VersionInfo, error) {
	return cc.GetVersionInfo()
}

func (cc *cadvisorClient) SubcontainerInfo(name string, req *cadvisorapi.ContainerInfoRequest) (map[string]*cadvisorapi.ContainerInfo, error) {
	infos, err := cc.SubcontainersInfo(name, req)
	if err != nil && len(infos) == 0 {
		return nil, err
	}

	result := make(map[string]*cadvisorapi.ContainerInfo, len(infos))
	for _, info := range infos {
		result[info.Name] = info
	}
	return result, err
}

func (cc *cadvisorClient) MachineInfo() (*cadvisorapi.MachineInfo, error) {
	return cc.GetMachineInfo()
}

func (cc *cadvisorClient) ImagesFsInfo() (cadvisorapiv2.FsInfo, error) {
	var label string

	switch cc.runtime {
	case "docker":
		label = cadvisorfs.LabelDockerImages
	case "rkt":
		label = cadvisorfs.LabelRktImages
	default:
		return cadvisorapiv2.FsInfo{}, fmt.Errorf("ImagesFsInfo: unknown runtime: %v", cc.runtime)
	}

	return cc.getFsInfo(label)
}

func (cc *cadvisorClient) RootFsInfo() (cadvisorapiv2.FsInfo, error) {
	return cc.GetDirFsInfo(cc.rootPath)
}

func (cc *cadvisorClient) getFsInfo(label string) (cadvisorapiv2.FsInfo, error) {
	res, err := cc.GetFsInfo(label)
	if err != nil {
		return cadvisorapiv2.FsInfo{}, err
	}
	if len(res) == 0 {
		return cadvisorapiv2.FsInfo{}, fmt.Errorf("failed to find information for the filesystem labeled %q", label)
	}
	// TODO(vmarmol): Handle this better when a label has more than one image filesystem.
	if len(res) > 1 {
		glog.Warningf("More than one filesystem labeled %q: %#v. Only using the first one", label, res)
	}

	return res[0], nil
}

func (cc *cadvisorClient) WatchEvents(request *events.Request) (*events.EventChannel, error) {
	return cc.WatchForEvents(request)
}

// HasDedicatedImageFs returns true if the imagefs has a dedicated device.
func (cc *cadvisorClient) HasDedicatedImageFs() (bool, error) {
	imageFsInfo, err := cc.ImagesFsInfo()
	if err != nil {
		return false, err
	}
	rootFsInfo, err := cc.RootFsInfo()
	if err != nil {
		return false, err
	}
	return imageFsInfo.Device != rootFsInfo.Device, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2021 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package instrumentation

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"path/filepath"
	"strconv"
	"time"

	mexporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric"
	"github.com/mitchellh/go-homedir"
	"github.com/rakyll/statik/fs"
	"github.com/sirupsen/logrus"
	"go.opentelemetry.io/otel/api/global"
	"go.opentelemetry.io/otel/api/metric"
	"go.opentelemetry.io/otel/exporters/stdout"
	"go.opentelemetry.io/otel/label"
	"go.opentelemetry.io/otel/sdk/metric/controller/push"
	"google.golang.org/api/option"

	"github.com/GoogleContainerTools/skaffold/cmd/skaffold/app/cmd/statik"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/constants"
	"github.com/GoogleContainerTools/skaffold/proto/v1"
)

var (
	allowedUsers = map[string]struct{}{
		"vsc":      {},
		"intellij": {},
		"gcloud":   {},
	}
)

func ExportMetrics(exitCode int) error {
	if !shouldExportMetrics || meter.Command == "" {
		return nil
	}
	home, err := homedir.Dir()
	if err != nil {
		return fmt.Errorf("retrieving home directory: %w", err)
	}
	meter.ExitCode = exitCode
	meter.Duration = time.Since(meter.StartTime)
	return exportMetrics(context.Background(),
		filepath.Join(home, constants.DefaultSkaffoldDir, constants.DefaultMetricFile),
		meter)
}

func exportMetrics(ctx context.Context, filename string, meter skaffoldMeter) error {
	logrus.Debug("exporting metrics")
	p, err := initExporter()
	if p == nil {
		return err
	}

	b, err := ioutil.ReadFile(filename)
	fileExists := err == nil
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	var meters []skaffoldMeter
	err = json.Unmarshal(b, &meters)
	if err != nil {
		meters = []skaffoldMeter{}
	}
	meters = append(meters, meter)
	if !isOnline {
		b, _ = json.Marshal(meters)
		return ioutil.WriteFile(filename, b, 0666)
	}

	start := time.Now()
	p.Start()
	for _, m := range meters {
		createMetrics(ctx, m)
	}
	p.Stop()
	logrus.Debugf("metrics uploading complete in %s", time.Since(start).String())

	if fileExists {
		return os.Remove(filename)
	}
	return nil
}

func initCloudMonitoringExporterMetrics() (*push.Controller, error) {
	statikFS, err := statik.FS()
	if err != nil {
		return nil, err
	}
	b, err := fs.ReadFile(statikFS, "/secret/keys.json")
	if err != nil {
		// No keys have been set in this version so do not attempt to write metrics
		if os.IsNotExist(err) {
			return devStdOutExporter()
		}
		return nil, err
	}

	var c creds
	err = json.Unmarshal(b, &c)
	if c.ProjectID == "" || err != nil {
		return nil, fmt.Errorf("no project id found in metrics credentials")
	}

	formatter := func(desc *metric.Descriptor) string {
		return fmt.Sprintf("custom.googleapis.com/skaffold/%s", desc.Name())
	}

	global.SetErrorHandler(errHandler{})
	return mexporter.InstallNewPipeline(
		[]mexporter.Option{
			mexporter.WithProjectID(c.ProjectID),
			mexporter.WithMetricDescriptorTypeFormatter(formatter),
			mexporter.WithMonitoringClientOptions(option.WithCredentialsJSON(b)),
			mexporter.WithOnError(func(err error) {
				logrus.Debugf("Error with metrics: %v", err)
			}),
		},
	)
}

func devStdOutExporter() (*push.Controller, error) {
	// export metrics to std out if local env is set.
	if isLocal := os.Getenv("SKAFFOLD_EXPORT_TO_STDOUT"); isLocal != "" {
		return stdout.InstallNewPipeline([]stdout.Option{
			stdout.WithQuantiles([]float64{0.5}),
			stdout.WithPrettyPrint(),
			stdout.WithWriter(os.Stdout),
		}, nil)
	}
	return nil, nil
}

func createMetrics(ctx context.Context, meter skaffoldMeter) {
	// There is a minimum 10 second interval that metrics are allowed to upload to Cloud monitoring
	// A metric is uniquely identified by the metric name and the labels and corresponding values
	// This random number is used as a label to differentiate the metrics per user so if two users
	// run `skaffold build` at the same time they will both have their metrics recorded
	randLabel := label.String("randomizer", strconv.Itoa(rand.Intn(75000)))

	m := global.Meter("skaffold")

	// cloud monitoring only supports string type labels
	labels := []label.KeyValue{
		label.String("version", meter.Version),
		label.String("os", meter.OS),
		label.String("arch", meter.Arch),
		label.String("command", meter.Command),
		label.String("error", meter.ErrorCode.String()),
		label.String("platform_type", meter.PlatformType),
		label.String("config_count", strconv.Itoa(meter.ConfigCount)),
	}
	sharedLabels := []label.KeyValue{
		randLabel,
	}
	if _, ok := allowedUsers[meter.User]; ok {
		sharedLabels = append(sharedLabels, label.String("user", meter.User))
	}
	labels = append(labels, sharedLabels...)

	runCounter := metric.Must(m).NewInt64ValueRecorder("launches", metric.WithDescription("Skaffold Invocations"))
	runCounter.Record(ctx, 1, labels...)

	durationRecorder := metric.Must(m).NewFloat64ValueRecorder("launch/duration",
		metric.WithDescription("durations of skaffold commands in seconds"))
	durationRecorder.Record(ctx, meter.Duration.Seconds(), labels...)
	if meter.Command != "" {
		commandMetrics(ctx, meter, m, sharedLabels...)
		flagMetrics(ctx, meter, m, randLabel)
		if doesBuild.Contains(meter.Command) {
			builderMetrics(ctx, meter, m, sharedLabels...)
		}
		if doesDeploy.Contains(meter.Command) {
			deployerMetrics(ctx, meter, m, sharedLabels...)
		}
	}

	if meter.ErrorCode != 0 {
		errorMetrics(ctx, meter, m, sharedLabels...)
	}
}

func flagMetrics(ctx context.Context, meter skaffoldMeter, m metric.Meter, randLabel label.KeyValue) {
	flagCounter := metric.Must(m).NewInt64ValueRecorder("flags", metric.WithDescription("Tracks usage of enum flags"))
	for k, v := range meter.EnumFlags {
		labels := []label.KeyValue{
			label.String("flag_name", k),
			label.String("flag_value", v),
			label.String("command", meter.Command),
			label.String("error", meter.ErrorCode.String()),
			randLabel,
		}
		flagCounter.Record(ctx, 1, labels...)
	}
}

func commandMetrics(ctx context.Context, meter skaffoldMeter, m metric.Meter, labels ...label.KeyValue) {
	commandCounter := metric.Must(m).NewInt64ValueRecorder(meter.Command,
		metric.WithDescription(fmt.Sprintf("Number of times %s is used", meter.Command)))
	labels = append(labels, label.String("error", meter.ErrorCode.String()))
	commandCounter.Record(ctx, 1, labels...)

	if meter.Command == "dev" || meter.Command == "debug" {
		iterationCounter := metric.Must(m).NewInt64ValueRecorder(fmt.Sprintf("%s/iterations", meter.Command),
			metric.WithDescription(fmt.Sprintf("Number of iterations in a %s session", meter.Command)))

		counts := make(map[string]map[proto.StatusCode]int)

		for _, iteration := range meter.DevIterations {
			if _, ok := counts[iteration.Intent]; !ok {
				counts[iteration.Intent] = make(map[proto.StatusCode]int)
			}
			m := counts[iteration.Intent]
			m[iteration.ErrorCode]++
		}
		for intention, errorCounts := range counts {
			for errorCode, count := range errorCounts {
				iterationCounter.Record(ctx, int64(count),
					append(labels,
						label.String("intent", intention),
						label.String("error", errorCode.String()),
					)...)
			}
		}
	}
}

func deployerMetrics(ctx context.Context, meter skaffoldMeter, m metric.Meter, labels ...label.KeyValue) {
	deployerCounter := metric.Must(m).NewInt64ValueRecorder("deployer", metric.WithDescription("Deployers used"))
	for _, deployer := range meter.Deployers {
		deployerCounter.Record(ctx, 1, append(labels, label.String("deployer", deployer))...)
	}
	if meter.HelmReleasesCount > 0 {
		multiReleasesCounter := metric.Must(m).NewInt64ValueRecorder("helmReleases", metric.WithDescription("Multiple helm releases used"))
		multiReleasesCounter.Record(ctx, 1, append(labels, label.Int("count", meter.HelmReleasesCount))...)
	}
}

func builderMetrics(ctx context.Context, meter skaffoldMeter, m metric.Meter, labels ...label.KeyValue) {
	builderCounter := metric.Must(m).NewInt64ValueRecorder("builders", metric.WithDescription("Builders used"))
	artifactCounter := metric.Must(m).NewInt64ValueRecorder("artifacts", metric.WithDescription("Number of artifacts used"))
	dependenciesCounter := metric.Must(m).NewInt64ValueRecorder("artifact-dependencies", metric.WithDescription("Number of artifacts with dependencies"))
	for builder, count := range meter.Builders {
		bLabel := label.String("builder", builder)
		builderCounter.Record(ctx, 1, append(labels, bLabel)...)
		artifactCounter.Record(ctx, int64(count), append(labels, bLabel)...)
		dependenciesCounter.Record(ctx, int64(meter.BuildDependencies[builder]), append(labels, bLabel)...)
	}
}

func errorMetrics(ctx context.Context, meter skaffoldMeter, m metric.Meter, labels ...label.KeyValue) {
	errCounter := metric.Must(m).NewInt64ValueRecorder("errors", metric.WithDescription("Skaffold errors"))
	errCounter.Record(ctx, 1, append(labels, label.String("error", meter.ErrorCode.String()))...)

	labels = append(labels, label.String("command", meter.Command))

	switch meter.ErrorCode {
	case proto.StatusCode_UNKNOWN_ERROR:
		unknownErrCounter := metric.Must(m).NewInt64ValueRecorder("errors/unknown", metric.WithDescription("Unknown Skaffold Errors"))
		unknownErrCounter.Record(ctx, 1, labels...)
	case proto.StatusCode_TEST_UNKNOWN:
		unknownCounter := metric.Must(m).NewInt64ValueRecorder("test/unknown", metric.WithDescription("Unknown test Skaffold Errors"))
		unknownCounter.Record(ctx, 1, labels...)
	case proto.StatusCode_DEPLOY_UNKNOWN:
		unknownCounter := metric.Must(m).NewInt64ValueRecorder("deploy/unknown", metric.WithDescription("Unknown deploy Skaffold Errors"))
		unknownCounter.Record(ctx, 1, labels...)
	case proto.StatusCode_BUILD_UNKNOWN:
		unknownCounter := metric.Must(m).NewInt64ValueRecorder("build/unknown", metric.WithDescription("Unknown build Skaffold Errors"))
		unknownCounter.Record(ctx, 1, labels...)
	}
}
<commit_msg>use lookup instead of get (#5734)<commit_after>/*
Copyright 2021 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package instrumentation

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"path/filepath"
	"strconv"
	"time"

	mexporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric"
	"github.com/mitchellh/go-homedir"
	"github.com/rakyll/statik/fs"
	"github.com/sirupsen/logrus"
	"go.opentelemetry.io/otel/api/global"
	"go.opentelemetry.io/otel/api/metric"
	"go.opentelemetry.io/otel/exporters/stdout"
	"go.opentelemetry.io/otel/label"
	"go.opentelemetry.io/otel/sdk/metric/controller/push"
	"google.golang.org/api/option"

	"github.com/GoogleContainerTools/skaffold/cmd/skaffold/app/cmd/statik"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/constants"
	"github.com/GoogleContainerTools/skaffold/proto/v1"
)

var (
	allowedUsers = map[string]struct{}{
		"vsc":      {},
		"intellij": {},
		"gcloud":   {},
	}
)

func ExportMetrics(exitCode int) error {
	if !shouldExportMetrics || meter.Command == "" {
		return nil
	}
	home, err := homedir.Dir()
	if err != nil {
		return fmt.Errorf("retrieving home directory: %w", err)
	}
	meter.ExitCode = exitCode
	meter.Duration = time.Since(meter.StartTime)
	return exportMetrics(context.Background(),
		filepath.Join(home, constants.DefaultSkaffoldDir, constants.DefaultMetricFile),
		meter)
}

func exportMetrics(ctx context.Context, filename string, meter skaffoldMeter) error {
	logrus.Debug("exporting metrics")
	p, err := initExporter()
	if p == nil {
		return err
	}

	b, err := ioutil.ReadFile(filename)
	fileExists := err == nil
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	var meters []skaffoldMeter
	err = json.Unmarshal(b, &meters)
	if err != nil {
		meters = []skaffoldMeter{}
	}
	meters = append(meters, meter)
	if !isOnline {
		b, _ = json.Marshal(meters)
		return ioutil.WriteFile(filename, b, 0666)
	}

	start := time.Now()
	p.Start()
	for _, m := range meters {
		createMetrics(ctx, m)
	}
	p.Stop()
	logrus.Debugf("metrics uploading complete in %s", time.Since(start).String())

	if fileExists {
		return os.Remove(filename)
	}
	return nil
}

func initCloudMonitoringExporterMetrics() (*push.Controller, error) {
	statikFS, err := statik.FS()
	if err != nil {
		return nil, err
	}
	b, err := fs.ReadFile(statikFS, "/secret/keys.json")
	if err != nil {
		// No keys have been set in this version so do not attempt to write metrics
		if os.IsNotExist(err) {
			return devStdOutExporter()
		}
		return nil, err
	}

	var c creds
	err = json.Unmarshal(b, &c)
	if c.ProjectID == "" || err != nil {
		return nil, fmt.Errorf("no project id found in metrics credentials")
	}

	formatter := func(desc *metric.Descriptor) string {
		return fmt.Sprintf("custom.googleapis.com/skaffold/%s", desc.Name())
	}

	global.SetErrorHandler(errHandler{})
	return mexporter.InstallNewPipeline(
		[]mexporter.Option{
			mexporter.WithProjectID(c.ProjectID),
			mexporter.WithMetricDescriptorTypeFormatter(formatter),
			mexporter.WithMonitoringClientOptions(option.WithCredentialsJSON(b)),
			mexporter.WithOnError(func(err error) {
				logrus.Debugf("Error with metrics: %v", err)
			}),
		},
	)
}

func devStdOutExporter() (*push.Controller, error) {
	// export metrics to std out if local env is set.
	if _, ok := os.LookupEnv("SKAFFOLD_EXPORT_TO_STDOUT"); ok {
		return stdout.InstallNewPipeline([]stdout.Option{
			stdout.WithQuantiles([]float64{0.5}),
			stdout.WithPrettyPrint(),
			stdout.WithWriter(os.Stdout),
		}, nil)
	}
	return nil, nil
}

func createMetrics(ctx context.Context, meter skaffoldMeter) {
	// There is a minimum 10 second interval that metrics are allowed to upload to Cloud monitoring
	// A metric is uniquely identified by the metric name and the labels and corresponding values
	// This random number is used as a label to differentiate the metrics per user so if two users
	// run `skaffold build` at the same time they will both have their metrics recorded
	randLabel := label.String("randomizer", strconv.Itoa(rand.Intn(75000)))

	m := global.Meter("skaffold")

	// cloud monitoring only supports string type labels
	labels := []label.KeyValue{
		label.String("version", meter.Version),
		label.String("os", meter.OS),
		label.String("arch", meter.Arch),
		label.String("command", meter.Command),
		label.String("error", meter.ErrorCode.String()),
		label.String("platform_type", meter.PlatformType),
		label.String("config_count", strconv.Itoa(meter.ConfigCount)),
	}
	sharedLabels := []label.KeyValue{
		randLabel,
	}
	if _, ok := allowedUsers[meter.User]; ok {
		sharedLabels = append(sharedLabels, label.String("user", meter.User))
	}
	labels = append(labels, sharedLabels...)

	runCounter := metric.Must(m).NewInt64ValueRecorder("launches", metric.WithDescription("Skaffold Invocations"))
	runCounter.Record(ctx, 1, labels...)

	durationRecorder := metric.Must(m).NewFloat64ValueRecorder("launch/duration",
		metric.WithDescription("durations of skaffold commands in seconds"))
	durationRecorder.Record(ctx, meter.Duration.Seconds(), labels...)
	if meter.Command != "" {
		commandMetrics(ctx, meter, m, sharedLabels...)
		flagMetrics(ctx, meter, m, randLabel)
		if doesBuild.Contains(meter.Command) {
			builderMetrics(ctx, meter, m, sharedLabels...)
		}
		if doesDeploy.Contains(meter.Command) {
			deployerMetrics(ctx, meter, m, sharedLabels...)
		}
	}

	if meter.ErrorCode != 0 {
		errorMetrics(ctx, meter, m, sharedLabels...)
	}
}

func flagMetrics(ctx context.Context, meter skaffoldMeter, m metric.Meter, randLabel label.KeyValue) {
	flagCounter := metric.Must(m).NewInt64ValueRecorder("flags", metric.WithDescription("Tracks usage of enum flags"))
	for k, v := range meter.EnumFlags {
		labels := []label.KeyValue{
			label.String("flag_name", k),
			label.String("flag_value", v),
			label.String("command", meter.Command),
			label.String("error", meter.ErrorCode.String()),
			randLabel,
		}
		flagCounter.Record(ctx, 1, labels...)
	}
}

func commandMetrics(ctx context.Context, meter skaffoldMeter, m metric.Meter, labels ...label.KeyValue) {
	commandCounter := metric.Must(m).NewInt64ValueRecorder(meter.Command,
		metric.WithDescription(fmt.Sprintf("Number of times %s is used", meter.Command)))
	labels = append(labels, label.String("error", meter.ErrorCode.String()))
	commandCounter.Record(ctx, 1, labels...)

	if meter.Command == "dev" || meter.Command == "debug" {
		iterationCounter := metric.Must(m).NewInt64ValueRecorder(fmt.Sprintf("%s/iterations", meter.Command),
			metric.WithDescription(fmt.Sprintf("Number of iterations in a %s session", meter.Command)))

		counts := make(map[string]map[proto.StatusCode]int)

		for _, iteration := range meter.DevIterations {
			if _, ok := counts[iteration.Intent]; !ok {
				counts[iteration.Intent] = make(map[proto.StatusCode]int)
			}
			m := counts[iteration.Intent]
			m[iteration.ErrorCode]++
		}
		for intention, errorCounts := range counts {
			for errorCode, count := range errorCounts {
				iterationCounter.Record(ctx, int64(count),
					append(labels,
						label.String("intent", intention),
						label.String("error", errorCode.String()),
					)...)
			}
		}
	}
}

func deployerMetrics(ctx context.Context, meter skaffoldMeter, m metric.Meter, labels ...label.KeyValue) {
	deployerCounter := metric.Must(m).NewInt64ValueRecorder("deployer", metric.WithDescription("Deployers used"))
	for _, deployer := range meter.Deployers {
		deployerCounter.Record(ctx, 1, append(labels, label.String("deployer", deployer))...)
	}
	if meter.HelmReleasesCount > 0 {
		multiReleasesCounter := metric.Must(m).NewInt64ValueRecorder("helmReleases", metric.WithDescription("Multiple helm releases used"))
		multiReleasesCounter.Record(ctx, 1, append(labels, label.Int("count", meter.HelmReleasesCount))...)
	}
}

func builderMetrics(ctx context.Context, meter skaffoldMeter, m metric.Meter, labels ...label.KeyValue) {
	builderCounter := metric.Must(m).NewInt64ValueRecorder("builders", metric.WithDescription("Builders used"))
	artifactCounter := metric.Must(m).NewInt64ValueRecorder("artifacts", metric.WithDescription("Number of artifacts used"))
	dependenciesCounter := metric.Must(m).NewInt64ValueRecorder("artifact-dependencies", metric.WithDescription("Number of artifacts with dependencies"))
	for builder, count := range meter.Builders {
		bLabel := label.String("builder", builder)
		builderCounter.Record(ctx, 1, append(labels, bLabel)...)
		artifactCounter.Record(ctx, int64(count), append(labels, bLabel)...)
		dependenciesCounter.Record(ctx, int64(meter.BuildDependencies[builder]), append(labels, bLabel)...)
	}
}

func errorMetrics(ctx context.Context, meter skaffoldMeter, m metric.Meter, labels ...label.KeyValue) {
	errCounter := metric.Must(m).NewInt64ValueRecorder("errors", metric.WithDescription("Skaffold errors"))
	errCounter.Record(ctx, 1, append(labels, label.String("error", meter.ErrorCode.String()))...)

	labels = append(labels, label.String("command", meter.Command))

	switch meter.ErrorCode {
	case proto.StatusCode_UNKNOWN_ERROR:
		unknownErrCounter := metric.Must(m).NewInt64ValueRecorder("errors/unknown", metric.WithDescription("Unknown Skaffold Errors"))
		unknownErrCounter.Record(ctx, 1, labels...)
	case proto.StatusCode_TEST_UNKNOWN:
		unknownCounter := metric.Must(m).NewInt64ValueRecorder("test/unknown", metric.WithDescription("Unknown test Skaffold Errors"))
		unknownCounter.Record(ctx, 1, labels...)
	case proto.StatusCode_DEPLOY_UNKNOWN:
		unknownCounter := metric.Must(m).NewInt64ValueRecorder("deploy/unknown", metric.WithDescription("Unknown deploy Skaffold Errors"))
		unknownCounter.Record(ctx, 1, labels...)
	case proto.StatusCode_BUILD_UNKNOWN:
		unknownCounter := metric.Must(m).NewInt64ValueRecorder("build/unknown", metric.WithDescription("Unknown build Skaffold Errors"))
		unknownCounter.Record(ctx, 1, labels...)
	}
}
<|endoftext|>