<commit_before>package clicommand

import (
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/buildkite/agent/agent"
	"github.com/buildkite/agent/api"
	"github.com/buildkite/agent/cliconfig"
	"github.com/buildkite/agent/envvar"
	"github.com/buildkite/agent/logger"
	"github.com/buildkite/agent/retry"
	"github.com/buildkite/agent/stdin"
	"github.com/codegangsta/cli"
)

var PipelineUploadHelpDescription = `Usage:

   buildkite-agent pipeline upload <file> [arguments...]

Description:

   Allows you to change the pipeline of a running build by uploading either a
   JSON or Yaml configuration file. If no configuration file is provided,
   we look for the file in the following locations:

   - buildkite.yml
   - buildkite.json
   - .buildkite/pipeline.yml
   - .buildkite/pipeline.json
   - .buildkite/steps.json (deprecated, removed in v2.2)

   You can also pipe build pipelines to the command, allowing you to create scripts
   that generate dynamic pipelines.

Example:

   $ buildkite-agent pipeline upload
   $ buildkite-agent pipeline upload my-custom-steps.json
   $ ./script/dynamic_step_generator | buildkite-agent pipeline upload`

type PipelineUploadConfig struct {
	FilePath         string `cli:"arg:0" label:"upload paths"`
	Replace          bool   `cli:"replace"`
	Job              string `cli:"job" validate:"required"`
	AgentAccessToken string `cli:"agent-access-token" validate:"required"`
	Endpoint         string `cli:"endpoint" validate:"required"`
	NoColor          bool   `cli:"no-color"`
	Debug            bool   `cli:"debug"`
	DebugHTTP        bool   `cli:"debug-http"`
}

var PipelineUploadCommand = cli.Command{
	Name:        "upload",
	Usage:       "Uploads a description of a build pipeline adds it to the currently running build after the current job.",
	Description: PipelineUploadHelpDescription,
	Flags: []cli.Flag{
		cli.BoolFlag{
			Name:   "replace",
			Usage:  "Replace the rest of the existing pipeline with the steps uploaded. Jobs that are already running are not removed.",
			EnvVar: "BUILDKITE_PIPELINE_REPLACE",
		},
		cli.StringFlag{
			Name:   "job",
			Value:  "",
			Usage:  "The job that is making the changes to it's build",
			EnvVar: "BUILDKITE_JOB_ID",
		},
		AgentAccessTokenFlag,
		EndpointFlag,
		NoColorFlag,
		DebugFlag,
		DebugHTTPFlag,
	},
	Action: func(c *cli.Context) {
		// The configuration will be loaded into this struct
		cfg := PipelineUploadConfig{}

		// Load the configuration
		loader := cliconfig.Loader{CLI: c, Config: &cfg}
		if err := loader.Load(); err != nil {
			logger.Fatal("%s", err)
		}

		// Setup the any global configuration options
		HandleGlobalFlags(cfg)

		// Find the pipeline file either from STDIN or the first
		// argument
		var input []byte
		var err error
		var filename string

		if cfg.FilePath != "" {
			logger.Info("Reading pipeline config from \"%s\"", cfg.FilePath)

			filename = filepath.Base(cfg.FilePath)
			input, err = ioutil.ReadFile(cfg.FilePath)
			if err != nil {
				logger.Fatal("Failed to read file: %s", err)
			}
		} else if stdin.IsPipe() {
			logger.Info("Reading pipeline config from STDIN")

			input, err = ioutil.ReadAll(os.Stdin)
			if err != nil {
				logger.Fatal("Failed to read from STDIN: %s", err)
			}
		} else {
			logger.Info("Searching for pipeline config...")

			paths := []string{
				"buildkite.yml",
				"buildkite.json",
				filepath.FromSlash(".buildkite/pipeline.yml"),
				filepath.FromSlash(".buildkite/pipeline.json"),
				filepath.FromSlash(".buildkite/steps.json"),
			}

			// Collect all the files that exist
			exists := []string{}
			for _, path := range paths {
				if _, err := os.Stat(path); err == nil {
					exists = append(exists, path)
				}
			}

			// If more than 1 of the config files exist, throw an
			// error. There can only be one!!
			if len(exists) > 1 {
				logger.Fatal("Found multiple configuration files: %s. Please only have 1 configuration file present.", strings.Join(exists, ", "))
			} else if len(exists) == 0 {
				logger.Fatal("Could not find a default pipeline configuration file. See `buildkite-agent pipeline upload --help` for more information.")
			}

			found := exists[0]

			logger.Info("Found config file \"%s\"", found)

			// Warn about the deprecated steps.json
			if found == filepath.FromSlash(".buildkite/steps.json") {
				logger.Warn("The default steps.json file has been deprecated and will be removed in v2.2. Please rename to .buildkite/pipeline.json and wrap the steps array in a `steps` property: { \"steps\": [ ... ] } }")
			}

			// Read the default file
			filename = path.Base(found)
			input, err = ioutil.ReadFile(found)
			if err != nil {
				logger.Fatal("Failed to read file \"%s\" (%s)", found, err)
			}
		}

		if len(input) == 0 {
			logger.Fatal("Config file is empty")
		}

		var parsed string

		logger.Debug("Parsing pipeline...")

		// Parse the pipeline and prepare it for upload
		parsed, err = envvar.Interpolate(string(input))
		if err != nil {
			logger.Fatal("Pipeline parsing of \"%s\" failed (%s)", filename, err)
		}

		// Create the API client
		client := agent.APIClient{
			Endpoint: cfg.Endpoint,
			Token:    cfg.AgentAccessToken,
		}.Create()

		// Generate a UUID that will identifiy this pipeline change. We
		// do this outside of the retry loop because we want this UUID
		// to be the same for each attempt at updating the pipeline.
		uuid := api.NewUUID()

		// Retry the pipeline upload a few times before giving up
		err = retry.Do(func(s *retry.Stats) error {
			_, err = client.Pipelines.Upload(cfg.Job, &api.Pipeline{UUID: uuid, Data: []byte(parsed), FileName: filename, Replace: cfg.Replace})
			if err != nil {
				logger.Warn("%s (%s)", err, s)
			}

			return err
		}, &retry.Config{Maximum: 5, Interval: 1 * time.Second})
		if err != nil {
			logger.Fatal("Failed to upload and process pipeline: %s", err)
		}

		logger.Info("Successfully uploaded and parsed pipeline config")
	},
}
<commit_msg>Removed support for automatically uploading a `steps.json` file<commit_after>package clicommand

import (
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/buildkite/agent/agent"
	"github.com/buildkite/agent/api"
	"github.com/buildkite/agent/cliconfig"
	"github.com/buildkite/agent/envvar"
	"github.com/buildkite/agent/logger"
	"github.com/buildkite/agent/retry"
	"github.com/buildkite/agent/stdin"
	"github.com/codegangsta/cli"
)

var PipelineUploadHelpDescription = `Usage:

   buildkite-agent pipeline upload <file> [arguments...]

Description:

   Allows you to change the pipeline of a running build by uploading either a
   JSON or Yaml configuration file. If no configuration file is provided,
   we look for the file in the following locations:

   - buildkite.yml
   - buildkite.json
   - .buildkite/pipeline.yml
   - .buildkite/pipeline.json

   You can also pipe build pipelines to the command, allowing you to create scripts
   that generate dynamic pipelines.

Example:

   $ buildkite-agent pipeline upload
   $ buildkite-agent pipeline upload my-custom-pipeline.yml
   $ ./script/dynamic_step_generator | buildkite-agent pipeline upload`

type PipelineUploadConfig struct {
	FilePath         string `cli:"arg:0" label:"upload paths"`
	Replace          bool   `cli:"replace"`
	Job              string `cli:"job" validate:"required"`
	AgentAccessToken string `cli:"agent-access-token" validate:"required"`
	Endpoint         string `cli:"endpoint" validate:"required"`
	NoColor          bool   `cli:"no-color"`
	Debug            bool   `cli:"debug"`
	DebugHTTP        bool   `cli:"debug-http"`
}

var PipelineUploadCommand = cli.Command{
	Name:        "upload",
	Usage:       "Uploads a description of a build pipeline adds it to the currently running build after the current job.",
	Description: PipelineUploadHelpDescription,
	Flags: []cli.Flag{
		cli.BoolFlag{
			Name:   "replace",
			Usage:  "Replace the rest of the existing pipeline with the steps uploaded. Jobs that are already running are not removed.",
			EnvVar: "BUILDKITE_PIPELINE_REPLACE",
		},
		cli.StringFlag{
			Name:   "job",
			Value:  "",
			Usage:  "The job that is making the changes to it's build",
			EnvVar: "BUILDKITE_JOB_ID",
		},
		AgentAccessTokenFlag,
		EndpointFlag,
		NoColorFlag,
		DebugFlag,
		DebugHTTPFlag,
	},
	Action: func(c *cli.Context) {
		// The configuration will be loaded into this struct
		cfg := PipelineUploadConfig{}

		// Load the configuration
		loader := cliconfig.Loader{CLI: c, Config: &cfg}
		if err := loader.Load(); err != nil {
			logger.Fatal("%s", err)
		}

		// Setup the any global configuration options
		HandleGlobalFlags(cfg)

		// Find the pipeline file either from STDIN or the first
		// argument
		var input []byte
		var err error
		var filename string

		if cfg.FilePath != "" {
			logger.Info("Reading pipeline config from \"%s\"", cfg.FilePath)

			filename = filepath.Base(cfg.FilePath)
			input, err = ioutil.ReadFile(cfg.FilePath)
			if err != nil {
				logger.Fatal("Failed to read file: %s", err)
			}
		} else if stdin.IsPipe() {
			logger.Info("Reading pipeline config from STDIN")

			input, err = ioutil.ReadAll(os.Stdin)
			if err != nil {
				logger.Fatal("Failed to read from STDIN: %s", err)
			}
		} else {
			logger.Info("Searching for pipeline config...")

			paths := []string{
				"buildkite.yml",
				"buildkite.json",
				filepath.FromSlash(".buildkite/pipeline.yml"),
				filepath.FromSlash(".buildkite/pipeline.json"),
			}

			// Collect all the files that exist
			exists := []string{}
			for _, path := range paths {
				if _, err := os.Stat(path); err == nil {
					exists = append(exists, path)
				}
			}

			// If more than 1 of the config files exist, throw an
			// error. There can only be one!!
			if len(exists) > 1 {
				logger.Fatal("Found multiple configuration files: %s. Please only have 1 configuration file present.", strings.Join(exists, ", "))
			} else if len(exists) == 0 {
				logger.Fatal("Could not find a default pipeline configuration file. See `buildkite-agent pipeline upload --help` for more information.")
			}

			found := exists[0]

			logger.Info("Found config file \"%s\"", found)

			// Warn about the deprecated steps.json
			if found == filepath.FromSlash(".buildkite/steps.json") {
				logger.Warn("The default steps.json file has been deprecated and will be removed in v2.2. Please rename to .buildkite/pipeline.json and wrap the steps array in a `steps` property: { \"steps\": [ ... ] } }")
			}

			// Read the default file
			filename = path.Base(found)
			input, err = ioutil.ReadFile(found)
			if err != nil {
				logger.Fatal("Failed to read file \"%s\" (%s)", found, err)
			}
		}

		if len(input) == 0 {
			logger.Fatal("Config file is empty")
		}

		var parsed string

		logger.Debug("Parsing pipeline...")

		// Parse the pipeline and prepare it for upload
		parsed, err = envvar.Interpolate(string(input))
		if err != nil {
			logger.Fatal("Pipeline parsing of \"%s\" failed (%s)", filename, err)
		}

		// Create the API client
		client := agent.APIClient{
			Endpoint: cfg.Endpoint,
			Token:    cfg.AgentAccessToken,
		}.Create()

		// Generate a UUID that will identifiy this pipeline change. We
		// do this outside of the retry loop because we want this UUID
		// to be the same for each attempt at updating the pipeline.
		uuid := api.NewUUID()

		// Retry the pipeline upload a few times before giving up
		err = retry.Do(func(s *retry.Stats) error {
			_, err = client.Pipelines.Upload(cfg.Job, &api.Pipeline{UUID: uuid, Data: []byte(parsed), FileName: filename, Replace: cfg.Replace})
			if err != nil {
				logger.Warn("%s (%s)", err, s)
			}

			return err
		}, &retry.Config{Maximum: 5, Interval: 1 * time.Second})
		if err != nil {
			logger.Fatal("Failed to upload and process pipeline: %s", err)
		}

		logger.Info("Successfully uploaded and parsed pipeline config")
	},
}
<|endoftext|>


<commit_before>package data

import (
	"database/sql"
	"fmt"
	"github.com/30x/apid"
	_ "github.com/mattn/go-sqlite3"
	"path"
	"sync"
)

const (
	configDataDriver = "data_driver"
	configDataSource = "data_source"
	configDataPath   = "data_path"

	commonDBID = "_apid_common_"
)

var log apid.LogService
var config apid.ConfigService

var dbMap = make(map[string]*sql.DB)
var dbMapSync sync.RWMutex

func CreateDataService() apid.DataService {
	config = apid.Config()
	log = apid.Log().ForModule("data")

	config.SetDefault(configDataDriver, "sqlite3")
	config.SetDefault(configDataSource, "file:%s?cache=shared&mode=rwc")
	config.SetDefault(configDataPath, "/var/tmp")

	return &dataService{}
}

type dataService struct {
}

func (d *dataService) DB() (*sql.DB, error) {
	return d.DBForID(commonDBID)
}

func (d *dataService) DBForID(id string) (db *sql.DB, err error) {

	dbMapSync.RLock()
	db = dbMap[id]
	dbMapSync.RUnlock()
	if db != nil {
		return
	}

	dbMapSync.Lock()
	defer dbMapSync.Unlock()

	db = dbMap[id]
	if db != nil {
		dbMapSync.Unlock()
		return
	}

	log.Info("LoadDB: ", id)

	dataPath := path.Join(config.GetString(configDataPath), id)
	dataSource := fmt.Sprintf(config.GetString(configDataSource), dataPath)
	db, err = sql.Open(config.GetString(configDataDriver), dataSource)
	if err != nil {
		log.Errorf("error loading db: %s", err)
		return
	}

	log.Errorf("#### DB path %s", dataPath)
	err = db.Ping()
	if err != nil {
		log.Errorf("error pinging db: %s", err)
		return
	}

	sqlString := "PRAGMA journal_mode=WAL;"
	_, err = db.Exec(sqlString)
	if err != nil {
		log.Errorf("error setting journal_mode: %s", err)
		return
	}

	sqlString = "PRAGMA foreign_keys = ON;"
	_, err = db.Exec(sqlString)
	if err != nil {
		log.Errorf("error enabling foreign_keys: %s", err)
		return
	}

	dbMap[id] = db
	return
}
<commit_msg>Add Info on the sqlite DB location<commit_after>package data

import (
	"database/sql"
	"fmt"
	"github.com/30x/apid"
	_ "github.com/mattn/go-sqlite3"
	"path"
	"sync"
)

const (
	configDataDriver = "data_driver"
	configDataSource = "data_source"
	configDataPath   = "data_path"

	commonDBID = "_apid_common_"
)

var log apid.LogService
var config apid.ConfigService

var dbMap = make(map[string]*sql.DB)
var dbMapSync sync.RWMutex

func CreateDataService() apid.DataService {
	config = apid.Config()
	log = apid.Log().ForModule("data")

	config.SetDefault(configDataDriver, "sqlite3")
	config.SetDefault(configDataSource, "file:%s?cache=shared&mode=rwc")
	config.SetDefault(configDataPath, "/var/tmp")

	return &dataService{}
}

type dataService struct {
}

func (d *dataService) DB() (*sql.DB, error) {
	return d.DBForID(commonDBID)
}

func (d *dataService) DBForID(id string) (db *sql.DB, err error) {

	dbMapSync.RLock()
	db = dbMap[id]
	dbMapSync.RUnlock()
	if db != nil {
		return
	}

	dbMapSync.Lock()
	defer dbMapSync.Unlock()

	db = dbMap[id]
	if db != nil {
		dbMapSync.Unlock()
		return
	}

	log.Info("LoadDB: ", id)

	dataPath := path.Join(config.GetString(configDataPath), id)
	dataSource := fmt.Sprintf(config.GetString(configDataSource), dataPath)
	db, err = sql.Open(config.GetString(configDataDriver), dataSource)
	if err != nil {
		log.Errorf("error loading db: %s", err)
		return
	}

	log.Infof("Sqlite DB path used by apid: %s", dataPath)
	err = db.Ping()
	if err != nil {
		log.Errorf("error pinging db: %s", err)
		return
	}

	sqlString := "PRAGMA journal_mode=WAL;"
	_, err = db.Exec(sqlString)
	if err != nil {
		log.Errorf("error setting journal_mode: %s", err)
		return
	}

	sqlString = "PRAGMA foreign_keys = ON;"
	_, err = db.Exec(sqlString)
	if err != nil {
		log.Errorf("error enabling foreign_keys: %s", err)
		return
	}

	dbMap[id] = db
	return
}
<|endoftext|>


<commit_before>package shell

import (
	"bytes"
	"context"
	"flag"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
	"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
	"io"
	"math"
	"sort"
)

func init() {
	Commands = append(Commands, &commandVolumeCheckDisk{})
}

type commandVolumeCheckDisk struct {
	env *CommandEnv
}

func (c *commandVolumeCheckDisk) Name() string {
	return "volume.check.disk"
}

func (c *commandVolumeCheckDisk) Help() string {
	return `check all replicated volumes to find and fix inconsistencies

	How it works:

	find all volumes that are replicated
	  for each volume id, if there are more than 2 replicas, find one pair with the largest 2 in file count.
      for the pair volume A and B
        append entries in A and not in B to B
        append entries in B and not in A to A

`
}

func (c *commandVolumeCheckDisk) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	if err = commandEnv.confirmIsLocked(); err != nil {
		return
	}

	fsckCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
	slowMode := fsckCommand.Bool("slow", false, "slow mode checks all replicas even file counts are the same")
	verbose := fsckCommand.Bool("v", false, "verbose mode")
	applyChanges := fsckCommand.Bool("force", false, "apply the fix")
	nonRepairThreshold := fsckCommand.Float64("nonRepairThreshold", 0.3, "repair when missing keys is not more than this limit")
	if err = fsckCommand.Parse(args); err != nil {
		return nil
	}

	c.env = commandEnv

	// collect topology information
	topologyInfo, _, err := collectTopologyInfo(commandEnv)
	if err != nil {
		return err
	}
	volumeReplicas, _ := collectVolumeReplicaLocations(topologyInfo)

	// pick 1 pairs of volume replica
	fileCount := func(replica *VolumeReplica) uint64 {
		return replica.info.FileCount - replica.info.DeleteCount
	}
	aDB, bDB := needle_map.NewMemDb(), needle_map.NewMemDb()
	defer aDB.Close()
	defer bDB.Close()

	for _, replicas := range volumeReplicas {
		sort.Slice(replicas, func(i, j int) bool {
			return fileCount(replicas[i]) > fileCount(replicas[j])
		})
		for len(replicas) >= 2 {
			a, b := replicas[0], replicas[1]
			if !*slowMode {
				if fileCount(a) == fileCount(b) {
					replicas = replicas[1:]
					continue
				}
			}
			if a.info.ReadOnly || b.info.ReadOnly {
				fmt.Fprintf(writer, "skipping readonly volume %d on %s and %s\n", a.info.Id, a.location.dataNode.Id, b.location.dataNode.Id)
				continue
			}

			// reset index db
			aDB.Close()
			bDB.Close()
			aDB, bDB = needle_map.NewMemDb(), needle_map.NewMemDb()

			// read index db
			if err := c.readIndexDatabase(aDB, a.info.Collection, a.info.Id, a.location.dataNode.Id, *verbose, writer); err != nil {
				return err
			}
			if err := c.readIndexDatabase(bDB, b.info.Collection, b.info.Id, b.location.dataNode.Id, *verbose, writer); err != nil {
				return err
			}

			// find and make up the differnces
			if err := c.doVolumeCheckDisk(aDB, bDB, a, b, *verbose, writer, *applyChanges, *nonRepairThreshold); err != nil {
				return err
			}
			if err := c.doVolumeCheckDisk(bDB, aDB, b, a, *verbose, writer, *applyChanges, *nonRepairThreshold); err != nil {
				return err
			}
			replicas = replicas[1:]
		}
	}

	return nil
}

func (c *commandVolumeCheckDisk) doVolumeCheckDisk(subtrahend, minuend *needle_map.MemDb, source, target *VolumeReplica, verbose bool, writer io.Writer, applyChanges bool, nonRepairThreshold float64) error {

	// find missing keys
	// hash join, can be more efficient
	var missingNeedles []needle_map.NeedleValue
	var counter int
	subtrahend.AscendingVisit(func(value needle_map.NeedleValue) error {
		counter++
		if _, found := minuend.Get(value.Key); !found {
			missingNeedles = append(missingNeedles, value)
		}
		return nil
	})

	fmt.Fprintf(writer, "volume %d %s has %d entries, %s missed %d entries\n", source.info.Id, source.location.dataNode.Id, counter, target.location.dataNode.Id, len(missingNeedles))

	if counter == 0 || len(missingNeedles) == 0 {
		return nil
	}

	missingNeedlesFraction := float64(len(missingNeedles)) / float64(counter)
	if missingNeedlesFraction > nonRepairThreshold {
		return fmt.Errorf(
			"failed to start repair volume %d, percentage of missing keys is greater than the threshold: %.2f > %.2f",
			source.info.Id, missingNeedlesFraction, nonRepairThreshold)
	}

	for _, needleValue := range missingNeedles {

		needleBlob, err := c.readSourceNeedleBlob(source.location.dataNode.Id, source.info.Id, needleValue)
		if err != nil {
			return err
		}

		if !applyChanges {
			continue
		}

		if verbose {
			fmt.Fprintf(writer, "read %d,%x %s => %s \n", source.info.Id, needleValue.Key, source.location.dataNode.Id, target.location.dataNode.Id)
		}

		if err := c.writeNeedleBlobToTarget(target.location.dataNode.Id, source.info.Id, needleValue, needleBlob); err != nil {
			return err
		}

	}

	return nil
}

func (c *commandVolumeCheckDisk) readSourceNeedleBlob(sourceVolumeServer string, volumeId uint32, needleValue needle_map.NeedleValue) (needleBlob []byte, err error) {

	err = operation.WithVolumeServerClient(sourceVolumeServer, c.env.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
		resp, err := client.ReadNeedleBlob(context.Background(), &volume_server_pb.ReadNeedleBlobRequest{
			VolumeId: volumeId,
			NeedleId: uint64(needleValue.Key),
			Offset:   needleValue.Offset.ToActualOffset(),
			Size:     int32(needleValue.Size),
		})
		if err != nil {
			return err
		}
		needleBlob = resp.NeedleBlob
		return nil
	})
	return
}

func (c *commandVolumeCheckDisk) writeNeedleBlobToTarget(targetVolumeServer string, volumeId uint32, needleValue needle_map.NeedleValue, needleBlob []byte) error {

	return operation.WithVolumeServerClient(targetVolumeServer, c.env.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
		_, err := client.WriteNeedleBlob(context.Background(), &volume_server_pb.WriteNeedleBlobRequest{
			VolumeId:   volumeId,
			NeedleId:   uint64(needleValue.Key),
			Size:       int32(needleValue.Size),
			NeedleBlob: needleBlob,
		})
		return err
	})

}

func (c *commandVolumeCheckDisk) readIndexDatabase(db *needle_map.MemDb, collection string, volumeId uint32, volumeServer string, verbose bool, writer io.Writer) error {

	var buf bytes.Buffer
	if err := c.copyVolumeIndexFile(collection, volumeId, volumeServer, &buf, verbose, writer); err != nil {
		return err
	}

	if verbose {
		fmt.Fprintf(writer, "load collection %s volume %d index size %d from %s ...\n", collection, volumeId, buf.Len(), volumeServer)
	}

	return db.LoadFromReaderAt(bytes.NewReader(buf.Bytes()))

}

func (c *commandVolumeCheckDisk) copyVolumeIndexFile(collection string, volumeId uint32, volumeServer string, buf *bytes.Buffer, verbose bool, writer io.Writer) error {

	return operation.WithVolumeServerClient(volumeServer, c.env.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {

		ext := ".idx"

		copyFileClient, err := volumeServerClient.CopyFile(context.Background(), &volume_server_pb.CopyFileRequest{
			VolumeId:                 volumeId,
			Ext:                      ".idx",
			CompactionRevision:       math.MaxUint32,
			StopOffset:               math.MaxInt64,
			Collection:               collection,
			IsEcVolume:               false,
			IgnoreSourceFileNotFound: false,
		})
		if err != nil {
			return fmt.Errorf("failed to start copying volume %d%s: %v", volumeId, ext, err)
		}

		err = writeToBuffer(copyFileClient, buf)
		if err != nil {
			return fmt.Errorf("failed to copy %d%s from %s: %v", volumeId, ext, volumeServer, err)
		}

		return nil

	})
}

func writeToBuffer(client volume_server_pb.VolumeServer_CopyFileClient, buf *bytes.Buffer) error {
	for {
		resp, receiveErr := client.Recv()
		if receiveErr == io.EOF {
			break
		}
		if receiveErr != nil {
			return fmt.Errorf("receiving: %v", receiveErr)
		}
		buf.Write(resp.FileContent)
	}
	return nil
}
<commit_msg>volume.check.disk: break loop for read only volumes<commit_after>package shell

import (
	"bytes"
	"context"
	"flag"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
	"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
	"io"
	"math"
	"sort"
)

func init() {
	Commands = append(Commands, &commandVolumeCheckDisk{})
}

type commandVolumeCheckDisk struct {
	env *CommandEnv
}

func (c *commandVolumeCheckDisk) Name() string {
	return "volume.check.disk"
}

func (c *commandVolumeCheckDisk) Help() string {
	return `check all replicated volumes to find and fix inconsistencies

	How it works:

	find all volumes that are replicated
	  for each volume id, if there are more than 2 replicas, find one pair with the largest 2 in file count.
      for the pair volume A and B
        append entries in A and not in B to B
        append entries in B and not in A to A

`
}

func (c *commandVolumeCheckDisk) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	if err = commandEnv.confirmIsLocked(); err != nil {
		return
	}

	fsckCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
	slowMode := fsckCommand.Bool("slow", false, "slow mode checks all replicas even file counts are the same")
	verbose := fsckCommand.Bool("v", false, "verbose mode")
	applyChanges := fsckCommand.Bool("force", false, "apply the fix")
	nonRepairThreshold := fsckCommand.Float64("nonRepairThreshold", 0.3, "repair when missing keys is not more than this limit")
	if err = fsckCommand.Parse(args); err != nil {
		return nil
	}

	c.env = commandEnv

	// collect topology information
	topologyInfo, _, err := collectTopologyInfo(commandEnv)
	if err != nil {
		return err
	}
	volumeReplicas, _ := collectVolumeReplicaLocations(topologyInfo)

	// pick 1 pairs of volume replica
	fileCount := func(replica *VolumeReplica) uint64 {
		return replica.info.FileCount - replica.info.DeleteCount
	}
	aDB, bDB := needle_map.NewMemDb(), needle_map.NewMemDb()
	defer aDB.Close()
	defer bDB.Close()

	for _, replicas := range volumeReplicas {
		sort.Slice(replicas, func(i, j int) bool {
			return fileCount(replicas[i]) > fileCount(replicas[j])
		})
		for len(replicas) >= 2 {
			a, b := replicas[0], replicas[1]
			if !*slowMode {
				if fileCount(a) == fileCount(b) {
					replicas = replicas[1:]
					continue
				}
			}
			if a.info.ReadOnly || b.info.ReadOnly {
				fmt.Fprintf(writer, "skipping readonly volume %d on %s and %s\n", a.info.Id, a.location.dataNode.Id, b.location.dataNode.Id)
				replicas = replicas[1:]
				continue
			}

			// reset index db
			aDB.Close()
			bDB.Close()
			aDB, bDB = needle_map.NewMemDb(), needle_map.NewMemDb()

			// read index db
			if err := c.readIndexDatabase(aDB, a.info.Collection, a.info.Id, a.location.dataNode.Id, *verbose, writer); err != nil {
				return err
			}
			if err := c.readIndexDatabase(bDB, b.info.Collection, b.info.Id, b.location.dataNode.Id, *verbose, writer); err != nil {
				return err
			}

			// find and make up the differnces
			if err := c.doVolumeCheckDisk(aDB, bDB, a, b, *verbose, writer, *applyChanges, *nonRepairThreshold); err != nil {
				return err
			}
			if err := c.doVolumeCheckDisk(bDB, aDB, b, a, *verbose, writer, *applyChanges, *nonRepairThreshold); err != nil {
				return err
			}
			replicas = replicas[1:]
		}
	}

	return nil
}

func (c *commandVolumeCheckDisk) doVolumeCheckDisk(subtrahend, minuend *needle_map.MemDb, source, target *VolumeReplica, verbose bool, writer io.Writer, applyChanges bool, nonRepairThreshold float64) error {

	// find missing keys
	// hash join, can be more efficient
	var missingNeedles []needle_map.NeedleValue
	var counter int
	subtrahend.AscendingVisit(func(value needle_map.NeedleValue) error {
		counter++
		if _, found := minuend.Get(value.Key); !found {
			missingNeedles = append(missingNeedles, value)
		}
		return nil
	})

	fmt.Fprintf(writer, "volume %d %s has %d entries, %s missed %d entries\n", source.info.Id, source.location.dataNode.Id, counter, target.location.dataNode.Id, len(missingNeedles))

	if counter == 0 || len(missingNeedles) == 0 {
		return nil
	}

	missingNeedlesFraction := float64(len(missingNeedles)) / float64(counter)
	if missingNeedlesFraction > nonRepairThreshold {
		return fmt.Errorf(
			"failed to start repair volume %d, percentage of missing keys is greater than the threshold: %.2f > %.2f",
			source.info.Id, missingNeedlesFraction, nonRepairThreshold)
	}

	for _, needleValue := range missingNeedles {

		needleBlob, err := c.readSourceNeedleBlob(source.location.dataNode.Id, source.info.Id, needleValue)
		if err != nil {
			return err
		}

		if !applyChanges {
			continue
		}

		if verbose {
			fmt.Fprintf(writer, "read %d,%x %s => %s \n", source.info.Id, needleValue.Key, source.location.dataNode.Id, target.location.dataNode.Id)
		}

		if err := c.writeNeedleBlobToTarget(target.location.dataNode.Id, source.info.Id, needleValue, needleBlob); err != nil {
			return err
		}

	}

	return nil
}

func (c *commandVolumeCheckDisk) readSourceNeedleBlob(sourceVolumeServer string, volumeId uint32, needleValue needle_map.NeedleValue) (needleBlob []byte, err error) {

	err = operation.WithVolumeServerClient(sourceVolumeServer, c.env.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
		resp, err := client.ReadNeedleBlob(context.Background(), &volume_server_pb.ReadNeedleBlobRequest{
			VolumeId: volumeId,
			NeedleId: uint64(needleValue.Key),
			Offset:   needleValue.Offset.ToActualOffset(),
			Size:     int32(needleValue.Size),
		})
		if err != nil {
			return err
		}
		needleBlob = resp.NeedleBlob
		return nil
	})
	return
}

func (c *commandVolumeCheckDisk) writeNeedleBlobToTarget(targetVolumeServer string, volumeId uint32, needleValue needle_map.NeedleValue, needleBlob []byte) error {

	return operation.WithVolumeServerClient(targetVolumeServer, c.env.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
		_, err := client.WriteNeedleBlob(context.Background(), &volume_server_pb.WriteNeedleBlobRequest{
			VolumeId:   volumeId,
			NeedleId:   uint64(needleValue.Key),
			Size:       int32(needleValue.Size),
			NeedleBlob: needleBlob,
		})
		return err
	})

}

func (c *commandVolumeCheckDisk) readIndexDatabase(db *needle_map.MemDb, collection string, volumeId uint32, volumeServer string, verbose bool, writer io.Writer) error {

	var buf bytes.Buffer
	if err := c.copyVolumeIndexFile(collection, volumeId, volumeServer, &buf, verbose, writer); err != nil {
		return err
	}

	if verbose {
		fmt.Fprintf(writer, "load collection %s volume %d index size %d from %s ...\n", collection, volumeId, buf.Len(), volumeServer)
	}

	return db.LoadFromReaderAt(bytes.NewReader(buf.Bytes()))

}

func (c *commandVolumeCheckDisk) copyVolumeIndexFile(collection string, volumeId uint32, volumeServer string, buf *bytes.Buffer, verbose bool, writer io.Writer) error {

	return operation.WithVolumeServerClient(volumeServer, c.env.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {

		ext := ".idx"

		copyFileClient, err := volumeServerClient.CopyFile(context.Background(), &volume_server_pb.CopyFileRequest{
			VolumeId:                 volumeId,
			Ext:                      ".idx",
			CompactionRevision:       math.MaxUint32,
			StopOffset:               math.MaxInt64,
			Collection:               collection,
			IsEcVolume:               false,
			IgnoreSourceFileNotFound: false,
		})
		if err != nil {
			return fmt.Errorf("failed to start copying volume %d%s: %v", volumeId, ext, err)
		}

		err = writeToBuffer(copyFileClient, buf)
		if err != nil {
			return fmt.Errorf("failed to copy %d%s from %s: %v", volumeId, ext, volumeServer, err)
		}

		return nil

	})
}

func writeToBuffer(client volume_server_pb.VolumeServer_CopyFileClient, buf *bytes.Buffer) error {
	for {
		resp, receiveErr := client.Recv()
		if receiveErr == io.EOF {
			break
		}
		if receiveErr != nil {
			return fmt.Errorf("receiving: %v", receiveErr)
		}
		buf.Write(resp.FileContent)
	}
	return nil
}
<|endoftext|>


<commit_before>package zeusclient

import (
	"os"
	"os/signal"
	"net"
	"strings"
	"strconv"
	"syscall"

	"github.com/kr/pty"
	"github.com/burke/ttyutils"
	usock "github.com/burke/zeus/unixsocket"
)

const (
	zeusSockName = ".zeus.sock"
)

func Run() {
	master, slave, err := pty.Open()
	if err != nil {
		panic(err)
	}
	defer master.Close()
	defer slave.Close()

	if ttyutils.IsTerminal(os.Stdout.Fd()) {
		oldState, err := ttyutils.MakeTerminalRaw(os.Stdout.Fd())
		if err != nil {
			panic(err)
		}
		defer ttyutils.RestoreTerminalState(os.Stdout.Fd(), oldState)
	}

	ttyutils.MirrorWinsize(os.Stdout, master)

	addr, err := net.ResolveUnixAddr("unixgram", zeusSockName)
	if err != nil {
		panic("Can't resolve server address")
	}

	// TODO: WINCH
	conn, err := net.DialUnix("unix", nil, addr)
	if err != nil {
		panic("Can't connect to Master")
	}

	msg := "Q:console:[]\n"
	conn.Write([]byte(msg))

	usock.SendFdOverUnixSocket(conn, int(slave.Fd()))
	slave.Close()

	msg, _, err = usock.ReadFromUnixSocket(conn)
	if err != nil {
		panic(err)
	}

	parts := strings.Split(msg, "\n")
	pid, err := strconv.Atoi(parts[0])
	if err != nil {
		panic(err)
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.SIGWINCH)
	go func() {
		for _ = range c {
			ttyutils.MirrorWinsize(os.Stdout, master)
			syscall.Kill(pid, syscall.SIGWINCH)
		}
	}()

	println("PID:", pid)
	var exitStatus int = -1
	if len(parts) > 2 {
		exitStatus, err = strconv.Atoi(parts[0])
		if err != nil {
			panic(err)
		}
	}

	eof := make(chan bool)
	go func() {
		for {
			buf := make([]byte,1024)
			n, err := master.Read(buf)
			if err != nil {
				eof <- true
				break
			}
			os.Stdout.Write(buf[:n])
		}
	}()

	go func() {
		for {
			buf := make([]byte,1024)
			n, err := os.Stdin.Read(buf)
			if err != nil {
				eof <- true
				break
			}
			master.Write(buf[:n])
		}
	}()

	<- eof

	if exitStatus == -1 {
		msg, _, err = usock.ReadFromUnixSocket(conn)
		if err != nil {
			panic(err)
		}
		parts := strings.Split(msg, "\n")
		exitStatus, err = strconv.Atoi(parts[0])
		if err != nil {
			panic(err)
		}
	}

	os.Exit(exitStatus)

}

<commit_msg>Really bad direct port of signal handling from ruby<commit_after>package zeusclient

import (
	"os"
	"os/signal"
	"net"
	"regexp"
	"strings"
	"strconv"
	"syscall"

	"github.com/kr/pty"
	"github.com/burke/ttyutils"
	usock "github.com/burke/zeus/unixsocket"
)

const (
	zeusSockName = ".zeus.sock"
	sigIntStr = "\x03"
	sigQuitStr = "\x1C"
)

var signalRegex = regexp.MustCompile(sigIntStr + "|" + sigQuitStr)

func Run() {
	master, slave, err := pty.Open()
	if err != nil {
		panic(err)
	}
	defer master.Close()
	defer slave.Close()

	if ttyutils.IsTerminal(os.Stdout.Fd()) {
		oldState, err := ttyutils.MakeTerminalRaw(os.Stdout.Fd())
		if err != nil {
			panic(err)
		}
		defer ttyutils.RestoreTerminalState(os.Stdout.Fd(), oldState)
	}

	ttyutils.MirrorWinsize(os.Stdout, master)

	addr, err := net.ResolveUnixAddr("unixgram", zeusSockName)
	if err != nil {
		panic("Can't resolve server address")
	}

	// TODO: WINCH
	conn, err := net.DialUnix("unix", nil, addr)
	if err != nil {
		panic("Can't connect to Master")
	}

	msg := "Q:console:[]\n"
	conn.Write([]byte(msg))

	usock.SendFdOverUnixSocket(conn, int(slave.Fd()))
	slave.Close()

	msg, _, err = usock.ReadFromUnixSocket(conn)
	if err != nil {
		panic(err)
	}

	parts := strings.Split(msg, "\n")
	pid, err := strconv.Atoi(parts[0])
	if err != nil {
		panic(err)
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.SIGWINCH)
	go func() {
		for _ = range c {
			ttyutils.MirrorWinsize(os.Stdout, master)
			syscall.Kill(pid, syscall.SIGWINCH)
		}
	}()

	println("PID:", pid)
	var exitStatus int = -1
	if len(parts) > 2 {
		exitStatus, err = strconv.Atoi(parts[0])
		if err != nil {
			panic(err)
		}
	}

	eof := make(chan bool)
	go func() {
		for {
			buf := make([]byte,1024)
			n, err := master.Read(buf)
			if err != nil {
				eof <- true
				break
			}
			os.Stdout.Write(buf[:n])
		}
	}()

	go func() {
		for {
			buf := make([]byte,1024)
			n, err := os.Stdin.Read(buf)
			if err != nil {
				eof <- true
				break
			}
			// TODO: Since we have a byte array, this could actually just check integer values...
			matches := signalRegex.FindAll(buf, 9999)
			for _, match := range matches {
				if m := string(match); m == sigIntStr {
					syscall.Kill(pid, syscall.SIGINT)
				} else if m == sigQuitStr {
					syscall.Kill(pid, syscall.SIGQUIT)
				}
			}
			master.Write(buf[:n])
		}
	}()

	<- eof

	if exitStatus == -1 {
		msg, _, err = usock.ReadFromUnixSocket(conn)
		if err != nil {
			panic(err)
		}
		parts := strings.Split(msg, "\n")
		exitStatus, err = strconv.Atoi(parts[0])
		if err != nil {
			panic(err)
		}
	}

	os.Exit(exitStatus)

}

<|endoftext|>


<commit_before>package main

import (
	"io"
	"net/http"

	"github.com/frozzare/go-config"

	_ "github.com/frozzare/go-config/plugins/yaml"
)

// Example file

func main() {
	// Use file + env as middlewares.
	config.Use(config.NewFromFile("config.yml"))
	config.Use(config.NewEnv())

	// Watch config (replaces first middleware since it's the same file path).
	config.WatchFile("config.yml")

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		v, _ := config.String("name")
		io.WriteString(w, v)
	})

	http.ListenAndServe(":8899", nil)
}
0
<commit_msg>Fix main.go<commit_after>package main

import (
	"io"
	"net/http"

	"github.com/frozzare/go-config"

	_ "github.com/frozzare/go-config/plugins/yaml"
)

// Example file

func main() {
	// Use file + env as middlewares.
	config.Use(config.NewFromFile("config.yml"))
	config.Use(config.NewEnv())

	// Watch config (replaces first middleware since it's the same file path).
	config.WatchFile("config.yml")

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		v, _ := config.String("name")
		io.WriteString(w, v)
	})

	http.ListenAndServe(":8899", nil)
}
<|endoftext|>


<commit_before>// Copyright 2015 SteelSeries ApS.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements a basic LISP interpretor for embedding in a go program for scripting.
// This file tests function evaluation.

package golisp

import (
	"fmt"
	. "gopkg.in/check.v1"
	"unsafe"
)

type SizeSuite struct {
}

var _ = Suite(&SizeSuite{})

func (s *SizeSuite) SetUpSuite(c *C) {

}

func (s *SizeSuite) TestSize(c *C) {
	d := *IntegerWithValue(42)
	fmt.Printf("Data size is %d bytes\n", unsafe.Sizeof(d))
	fmt.Printf("Data alignment is %d\n", unsafe.Alignof(d))
}
<commit_msg>Not required.. it only output alignment & word size.<commit_after><|endoftext|>


<commit_before>package datastore

import (
	"errors"
	"time"

	query "github.com/ipfs/go-datastore/query"
)

/*
Datastore represents storage for any key-value pair.

Datastores are general enough to be backed by all kinds of different storage:
in-memory caches, databases, a remote datastore, flat files on disk, etc.

The general idea is to wrap a more complicated storage facility in a simple,
uniform interface, keeping the freedom of using the right tools for the job.
In particular, a Datastore can aggregate other datastores in interesting ways,
like sharded (to distribute load) or tiered access (caches before databases).

While Datastores should be written general enough to accept all sorts of
values, some implementations will undoubtedly have to be specific (e.g. SQL
databases where fields should be decomposed into columns), particularly to
support queries efficiently. Moreover, certain datastores may enforce certain
types of values (e.g. requiring an io.Reader, a specific struct, etc) or
serialization formats (JSON, Protobufs, etc).

IMPORTANT: No Datastore should ever Panic! This is a cross-module interface,
and thus it should behave predictably and handle exceptional conditions with
proper error reporting. Thus, all Datastore calls may return errors, which
should be checked by callers.
*/
type Datastore interface {
	// Put stores the object `value` named by `key`.
	//
	// The generalized Datastore interface does not impose a value type,
	// allowing various datastore middleware implementations (which do not
	// handle the values directly) to be composed together.
	//
	// Ultimately, the lowest-level datastore will need to do some value checking
	// or risk getting incorrect values. It may also be useful to expose a more
	// type-safe interface to your application, and do the checking up-front.
	Put(key Key, value []byte) error

	// Get retrieves the object `value` named by `key`.
	// Get will return ErrNotFound if the key is not mapped to a value.
	Get(key Key) (value []byte, err error)

	// Has returns whether the `key` is mapped to a `value`.
	// In some contexts, it may be much cheaper only to check for existence of
	// a value, rather than retrieving the value itself. (e.g. HTTP HEAD).
	// The default implementation is found in `GetBackedHas`.
	Has(key Key) (exists bool, err error)

	// GetSize returns the size of the `value` named by `key`.
	// In some contexts, it may be much cheaper to only get the size of the
	// value rather than retrieving the value itself.
	GetSize(key Key) (size int, err error)

	// Delete removes the value for given `key`.
	Delete(key Key) error

	// Query searches the datastore and returns a query result. This function
	// may return before the query actually runs. To wait for the query:
	//
	//   result, _ := ds.Query(q)
	//
	//   // use the channel interface; result may come in at different times
	//   for entry := range result.Next() { ... }
	//
	//   // or wait for the query to be completely done
	//   entries, _ := result.Rest()
	//   for entry := range entries { ... }
	//
	Query(q query.Query) (query.Results, error)
}

type Batching interface {
	Datastore

	Batch() (Batch, error)
}

var ErrBatchUnsupported = errors.New("this datastore does not support batching")

// ThreadSafeDatastore is an interface that all threadsafe datastore should
// implement to leverage type safety checks.
type ThreadSafeDatastore interface {
	Datastore

	IsThreadSafe()
}

// CheckedDatastore is an interface that should be implemented by datastores
// which may need checking on-disk data integrity.
type CheckedDatastore interface {
	Datastore

	Check() error
}

// CheckedDatastore is an interface that should be implemented by datastores
// which want to provide a mechanism to check data integrity and/or
// error correction.
type ScrubbedDatastore interface {
	Datastore

	Scrub() error
}

// GCDatastore is an interface that should be implemented by datastores which
// don't free disk space by just removing data from them.
type GCDatastore interface {
	Datastore

	CollectGarbage() error
}

// PersistentDatastore is an interface that should be implemented by datastores
// which can report disk usage.
type PersistentDatastore interface {
	Datastore

	// DiskUsage returns the space used by a datastore, in bytes.
	DiskUsage() (uint64, error)
}

// DiskUsage checks if a Datastore is a
// PersistentDatastore and returns its DiskUsage(),
// otherwise returns 0.
func DiskUsage(d Datastore) (uint64, error) {
	persDs, ok := d.(PersistentDatastore)
	if !ok {
		return 0, nil
	}
	return persDs.DiskUsage()
}

// TTLDatastore is an interface that should be implemented by datastores that
// support expiring entries.
type TTLDatastore interface {
	Datastore

	PutWithTTL(key Key, value []byte, ttl time.Duration) error
	SetTTL(key Key, ttl time.Duration) error
	GetExpiration(key Key) (time.Time, error)
}

// Txn extends the Datastore type. Txns allow users to batch queries and
// mutations to the Datastore into atomic groups, or transactions. Actions
// performed on a transaction will not take hold until a successful call to
// Commit has been made. Likewise, transactions can be aborted by calling
// Discard before a successful Commit has been made.
type Txn interface {
	Datastore

	// Commit finalizes a transaction, attempting to commit it to the Datastore.
	// May return an error if the transaction has gone stale. The presence of an
	// error is an indication that the data was not committed to the Datastore.
	Commit() error
	// Discard throws away changes recorded in a transaction without committing
	// them to the underlying Datastore. Any calls made to Discard after Commit
	// has been successfully called will have no effect on the transaction and
	// state of the Datastore, making it safe to defer.
	Discard()
}

// TxnDatastore is an interface that should be implemented by datastores that
// support transactions.
type TxnDatastore interface {
	Datastore

	NewTransaction(readOnly bool) (Txn, error)
}

// Errors

// ErrNotFound is returned by Get, Has, and Delete when a datastore does not
// map the given key to a value.
var ErrNotFound = errors.New("datastore: key not found")

// ErrInvalidType is returned by Put when a given value is incopatible with
// the type the datastore supports. This means a conversion (or serialization)
// is needed beforehand.
var ErrInvalidType = errors.New("datastore: invalid type error")

// GetBackedHas provides a default Datastore.Has implementation.
// It exists so Datastore.Has implementations can use it, like so:
//
// func (*d SomeDatastore) Has(key Key) (exists bool, err error) {
//   return GetBackedHas(d, key)
// }
func GetBackedHas(ds Datastore, key Key) (bool, error) {
	_, err := ds.Get(key)
	switch err {
	case nil:
		return true, nil
	case ErrNotFound:
		return false, nil
	default:
		return false, err
	}
}

// GetBackedSize provides a default Datastore.GetSize implementation.
// It exists so Datastore.GetSize implementations can use it, like so:
//
// func (*d SomeDatastore) GetSize(key Key) (size int, err error) {
//   return GetBackedSize(d, key)
// }
func GetBackedSize(ds Datastore, key Key) (int, error) {
	value, err := ds.Get(key)
	if err == nil {
		return len(value), nil
	}
	return -1, err
}

type Batch interface {
	Put(key Key, val []byte) error

	Delete(key Key) error

	Commit() error
}
<commit_msg>Describe behavior of Batching datastores<commit_after>package datastore

import (
	"errors"
	"time"

	query "github.com/ipfs/go-datastore/query"
)

/*
Datastore represents storage for any key-value pair.

Datastores are general enough to be backed by all kinds of different storage:
in-memory caches, databases, a remote datastore, flat files on disk, etc.

The general idea is to wrap a more complicated storage facility in a simple,
uniform interface, keeping the freedom of using the right tools for the job.
In particular, a Datastore can aggregate other datastores in interesting ways,
like sharded (to distribute load) or tiered access (caches before databases).

While Datastores should be written general enough to accept all sorts of
values, some implementations will undoubtedly have to be specific (e.g. SQL
databases where fields should be decomposed into columns), particularly to
support queries efficiently. Moreover, certain datastores may enforce certain
types of values (e.g. requiring an io.Reader, a specific struct, etc) or
serialization formats (JSON, Protobufs, etc).

IMPORTANT: No Datastore should ever Panic! This is a cross-module interface,
and thus it should behave predictably and handle exceptional conditions with
proper error reporting. Thus, all Datastore calls may return errors, which
should be checked by callers.
*/
type Datastore interface {
	// Put stores the object `value` named by `key`.
	//
	// The generalized Datastore interface does not impose a value type,
	// allowing various datastore middleware implementations (which do not
	// handle the values directly) to be composed together.
	//
	// Ultimately, the lowest-level datastore will need to do some value checking
	// or risk getting incorrect values. It may also be useful to expose a more
	// type-safe interface to your application, and do the checking up-front.
	Put(key Key, value []byte) error

	// Get retrieves the object `value` named by `key`.
	// Get will return ErrNotFound if the key is not mapped to a value.
	Get(key Key) (value []byte, err error)

	// Has returns whether the `key` is mapped to a `value`.
	// In some contexts, it may be much cheaper only to check for existence of
	// a value, rather than retrieving the value itself. (e.g. HTTP HEAD).
	// The default implementation is found in `GetBackedHas`.
	Has(key Key) (exists bool, err error)

	// GetSize returns the size of the `value` named by `key`.
	// In some contexts, it may be much cheaper to only get the size of the
	// value rather than retrieving the value itself.
	GetSize(key Key) (size int, err error)

	// Delete removes the value for given `key`.
	Delete(key Key) error

	// Query searches the datastore and returns a query result. This function
	// may return before the query actually runs. To wait for the query:
	//
	//   result, _ := ds.Query(q)
	//
	//   // use the channel interface; result may come in at different times
	//   for entry := range result.Next() { ... }
	//
	//   // or wait for the query to be completely done
	//   entries, _ := result.Rest()
	//   for entry := range entries { ... }
	//
	Query(q query.Query) (query.Results, error)
}

// Batching datastores support deferred, grouped updates to the database.
// `Batch`es do NOT have transactional semantics: updates to the underlying
// datastore are not guaranteed to occur in the same iota of time. Similarly,
// batched updates will not be flushed to the underlying datastore until
// `Commit` has been called. `Txn`s from a `TxnDatastore` have all the
// capabilities of a `Batch`, but the reverse is NOT true.
type Batching interface {
	Datastore

	Batch() (Batch, error)
}

var ErrBatchUnsupported = errors.New("this datastore does not support batching")

// ThreadSafeDatastore is an interface that all threadsafe datastore should
// implement to leverage type safety checks.
type ThreadSafeDatastore interface {
	Datastore

	IsThreadSafe()
}

// CheckedDatastore is an interface that should be implemented by datastores
// which may need checking on-disk data integrity.
type CheckedDatastore interface {
	Datastore

	Check() error
}

// CheckedDatastore is an interface that should be implemented by datastores
// which want to provide a mechanism to check data integrity and/or
// error correction.
type ScrubbedDatastore interface {
	Datastore

	Scrub() error
}

// GCDatastore is an interface that should be implemented by datastores which
// don't free disk space by just removing data from them.
type GCDatastore interface {
	Datastore

	CollectGarbage() error
}

// PersistentDatastore is an interface that should be implemented by datastores
// which can report disk usage.
type PersistentDatastore interface {
	Datastore

	// DiskUsage returns the space used by a datastore, in bytes.
	DiskUsage() (uint64, error)
}

// DiskUsage checks if a Datastore is a
// PersistentDatastore and returns its DiskUsage(),
// otherwise returns 0.
func DiskUsage(d Datastore) (uint64, error) {
	persDs, ok := d.(PersistentDatastore)
	if !ok {
		return 0, nil
	}
	return persDs.DiskUsage()
}

// TTLDatastore is an interface that should be implemented by datastores that
// support expiring entries.
type TTLDatastore interface {
	Datastore

	PutWithTTL(key Key, value []byte, ttl time.Duration) error
	SetTTL(key Key, ttl time.Duration) error
	GetExpiration(key Key) (time.Time, error)
}

// Txn extends the Datastore type. Txns allow users to batch queries and
// mutations to the Datastore into atomic groups, or transactions. Actions
// performed on a transaction will not take hold until a successful call to
// Commit has been made. Likewise, transactions can be aborted by calling
// Discard before a successful Commit has been made.
type Txn interface {
	Datastore

	// Commit finalizes a transaction, attempting to commit it to the Datastore.
	// May return an error if the transaction has gone stale. The presence of an
	// error is an indication that the data was not committed to the Datastore.
	Commit() error
	// Discard throws away changes recorded in a transaction without committing
	// them to the underlying Datastore. Any calls made to Discard after Commit
	// has been successfully called will have no effect on the transaction and
	// state of the Datastore, making it safe to defer.
	Discard()
}

// TxnDatastore is an interface that should be implemented by datastores that
// support transactions.
type TxnDatastore interface {
	Datastore

	NewTransaction(readOnly bool) (Txn, error)
}

// Errors

// ErrNotFound is returned by Get, Has, and Delete when a datastore does not
// map the given key to a value.
var ErrNotFound = errors.New("datastore: key not found")

// ErrInvalidType is returned by Put when a given value is incopatible with
// the type the datastore supports. This means a conversion (or serialization)
// is needed beforehand.
var ErrInvalidType = errors.New("datastore: invalid type error")

// GetBackedHas provides a default Datastore.Has implementation.
// It exists so Datastore.Has implementations can use it, like so:
//
// func (*d SomeDatastore) Has(key Key) (exists bool, err error) {
//   return GetBackedHas(d, key)
// }
func GetBackedHas(ds Datastore, key Key) (bool, error) {
	_, err := ds.Get(key)
	switch err {
	case nil:
		return true, nil
	case ErrNotFound:
		return false, nil
	default:
		return false, err
	}
}

// GetBackedSize provides a default Datastore.GetSize implementation.
// It exists so Datastore.GetSize implementations can use it, like so:
//
// func (*d SomeDatastore) GetSize(key Key) (size int, err error) {
//   return GetBackedSize(d, key)
// }
func GetBackedSize(ds Datastore, key Key) (int, error) {
	value, err := ds.Get(key)
	if err == nil {
		return len(value), nil
	}
	return -1, err
}

type Batch interface {
	Put(key Key, val []byte) error

	Delete(key Key) error

	Commit() error
}
<|endoftext|>


<commit_before>package decayment

import (
	"sync"
	"time"
)

// States engine
type States struct {
	sync.Mutex
	Counts     map[interface{}]int64
	Seens      map[interface{}]time.Time
	tickerChan chan struct{}
}

// New state
func New() *States {
	s := States{
		Counts:     make(map[interface{}]int64),
		Seens:      make(map[interface{}]time.Time),
		tickerChan: make(chan struct{}),
	}
	return &s
}

// Incr increments parameter key by one setting seen to now
func (s *States) Incr(key interface{}) error {
	return s.IncrTime(key, time.Now())
}

// IncrTime increments parameter key by one setting seen to parameter t
func (s *States) IncrTime(key interface{}, t time.Time) error {
	s.Lock()
	defer s.Unlock()
	s.Counts[key]++
	s.Seens[key] = t
	return nil
}

// Decr decrements all keys if seen below threshold*time.Second
func (s *States) Decr(threshold int) error {
	s.Lock()
	defer s.Unlock()
	for intIP, seen := range s.Seens {
		if time.Since(seen) >= time.Duration(threshold)*time.Second {
			s.Counts[intIP]--
			if s.Counts[intIP] <= 0 {
				delete(s.Counts, intIP)
				delete(s.Seens, intIP)
			}
		}
	}
	return nil
}

// Start starts the decrement loop with interval*time.Second and threshold*time.Second
func (s *States) Start(interval int, threshold int) {
	ticker := time.NewTicker(time.Duration(interval) * time.Second)
	go func() {
		for {
			select {
			case <-ticker.C:
				s.Decr(threshold)
			case <-s.tickerChan:
				ticker.Stop()
				return
			}
		}
	}()
}

// Stop stops the decrement loop
func (s *States) Stop() {
	close(s.tickerChan)
}
<commit_msg>package comment<commit_after>/*
Copyright 2016 Lukas Rist. All rights reserved.
Use of this source code is governed by a MIT-style
license that can be found in the LICENSE file.
*/

// Package decayment decays counters after not being updated during threshold
package decayment

import (
	"sync"
	"time"
)

// States to be decayed
type States struct {
	sync.Mutex
	// Countes per key
	Counts map[interface{}]int64
	// Seens keeps last seen per key
	Seens      map[interface{}]time.Time
	tickerChan chan struct{}
}

// New state instance
func New() *States {
	s := States{
		Counts:     make(map[interface{}]int64),
		Seens:      make(map[interface{}]time.Time),
		tickerChan: make(chan struct{}),
	}
	return &s
}

// Incr increments parameter key by one setting seen to now
func (s *States) Incr(key interface{}) error {
	return s.IncrTime(key, time.Now())
}

// IncrTime increments parameter key by one setting seen to parameter t
func (s *States) IncrTime(key interface{}, t time.Time) error {
	s.Lock()
	defer s.Unlock()
	s.Counts[key]++
	s.Seens[key] = t
	return nil
}

// Decr decrements all keys if seen below threshold*time.Second
func (s *States) Decr(threshold int) error {
	s.Lock()
	defer s.Unlock()
	for intIP, seen := range s.Seens {
		if time.Since(seen) >= time.Duration(threshold)*time.Second {
			s.Counts[intIP]--
			if s.Counts[intIP] <= 0 {
				delete(s.Counts, intIP)
				delete(s.Seens, intIP)
			}
		}
	}
	return nil
}

// Start starts the decrement loop with interval*time.Second and threshold*time.Second
func (s *States) Start(interval int, threshold int) {
	ticker := time.NewTicker(time.Duration(interval) * time.Second)
	go func() {
		for {
			select {
			case <-ticker.C:
				s.Decr(threshold)
			case <-s.tickerChan:
				ticker.Stop()
				return
			}
		}
	}()
}

// Stop stops the decrement loop
func (s *States) Stop() {
	close(s.tickerChan)
}
<|endoftext|>


<commit_before>package decor

import (
	"fmt"
	"math"
	"time"

	"github.com/VividCortex/ewma"
)

// TimeNormalizer interface. Implementors could be passed into
// MovingAverageETA, in order to affect i.e. normalize its output.
type TimeNormalizer interface {
	Normalize(time.Duration) time.Duration
}

// TimeNormalizerFunc is function type adapter to convert function
// into TimeNormalizer.
type TimeNormalizerFunc func(time.Duration) time.Duration

func (f TimeNormalizerFunc) Normalize(src time.Duration) time.Duration {
	return f(src)
}

// EwmaETA exponential-weighted-moving-average based ETA decorator.
// Note that it's necessary to supply bar.Incr* methods with incremental
// work duration as second argument, in order for this decorator to
// work correctly. This decorator is a wrapper of MovingAverageETA.
func EwmaETA(style TimeStyle, age float64, wcc ...WC) Decorator {
	var average MovingAverage
	if age == 0 {
		average = ewma.NewMovingAverage()
	} else {
		average = ewma.NewMovingAverage(age)
	}
	return MovingAverageETA(style, average, nil, wcc...)
}

// MovingAverageETA decorator relies on MovingAverage implementation to calculate its average.
//
//	`style` one of [ET_STYLE_GO|ET_STYLE_HHMMSS|ET_STYLE_HHMM|ET_STYLE_MMSS]
//
//	`average` implementation of MovingAverage interface
//
//	`normalizer` available implementations are [FixedIntervalTimeNormalizer|MaxTolerateTimeNormalizer]
//
//	`wcc` optional WC config
func MovingAverageETA(style TimeStyle, average MovingAverage, normalizer TimeNormalizer, wcc ...WC) Decorator {
	var wc WC
	for _, widthConf := range wcc {
		wc = widthConf
	}
	wc.Init()
	d := &movingAverageETA{
		WC:         wc,
		average:    average,
		normalizer: normalizer,
		producer:   chooseTimeProducer(style),
	}
	return d
}

type movingAverageETA struct {
	WC
	average     ewma.MovingAverage
	normalizer  TimeNormalizer
	producer    func(time.Duration) string
	completeMsg *string
}

func (d *movingAverageETA) Decor(st *Statistics) string {
	if st.Completed && d.completeMsg != nil {
		return d.FormatMsg(*d.completeMsg)
	}

	v := math.Round(d.average.Value())
	remaining := time.Duration((st.Total - st.Current) * int64(v))
	if d.normalizer != nil {
		remaining = d.normalizer.Normalize(remaining)
	}
	return d.FormatMsg(d.producer(remaining))
}

func (d *movingAverageETA) NextAmount(n int64, wdd ...time.Duration) {
	var workDuration time.Duration
	for _, wd := range wdd {
		workDuration = wd
	}
	durPerItem := float64(workDuration) / float64(n)
	if math.IsInf(durPerItem, 0) || math.IsNaN(durPerItem) {
		return
	}
	d.average.Add(durPerItem)
}

func (d *movingAverageETA) OnCompleteMessage(msg string) {
	d.completeMsg = &msg
}

// AverageETA decorator. It's wrapper of NewAverageETA.
//
//	`style` one of [ET_STYLE_GO|ET_STYLE_HHMMSS|ET_STYLE_HHMM|ET_STYLE_MMSS]
//
//	`wcc` optional WC config
func AverageETA(style TimeStyle, wcc ...WC) Decorator {
	return NewAverageETA(style, time.Now(), wcc...)
}

// NewAverageETA decorator with user provided start time.
//
//	`style` one of [ET_STYLE_GO|ET_STYLE_HHMMSS|ET_STYLE_HHMM|ET_STYLE_MMSS]
//
//	`startTime` start time
//
//	`wcc` optional WC config
func NewAverageETA(style TimeStyle, startTime time.Time, wcc ...WC) Decorator {
	var wc WC
	for _, widthConf := range wcc {
		wc = widthConf
	}
	wc.Init()
	d := &averageETA{
		WC:        wc,
		startTime: startTime,
		producer:  chooseTimeProducer(style),
	}
	return d
}

type averageETA struct {
	WC
	startTime   time.Time
	producer    func(time.Duration) string
	completeMsg *string
}

func (d *averageETA) Decor(st *Statistics) string {
	if st.Completed && d.completeMsg != nil {
		return d.FormatMsg(*d.completeMsg)
	}

	timeElapsed := time.Since(d.startTime)
	v := float64(timeElapsed) / float64(st.Current)
	if math.IsInf(v, 0) || math.IsNaN(v) {
		v = 0
	}
	remaining := time.Duration((st.Total - st.Current) * int64(math.Round(v)))
	return d.FormatMsg(d.producer(remaining))
}

func (d *averageETA) OnCompleteMessage(msg string) {
	d.completeMsg = &msg
}

func (d *averageETA) AverageAdjust(startTime time.Time) {
	d.startTime = startTime
}

// MaxTolerateTimeNormalizer returns implementation of TimeNormalizer.
func MaxTolerateTimeNormalizer(maxTolerate time.Duration) TimeNormalizer {
	var normalized time.Duration
	var lastCall time.Time
	return TimeNormalizerFunc(func(remaining time.Duration) time.Duration {
		if diff := normalized - remaining; diff <= 0 || diff > maxTolerate || remaining < maxTolerate/2 {
			normalized = remaining
			lastCall = time.Now()
			return remaining
		}
		normalized -= time.Since(lastCall)
		lastCall = time.Now()
		return normalized
	})
}

// FixedIntervalTimeNormalizer returns implementation of TimeNormalizer.
func FixedIntervalTimeNormalizer(updInterval int) TimeNormalizer {
	var normalized time.Duration
	var lastCall time.Time
	var count int
	return TimeNormalizerFunc(func(remaining time.Duration) time.Duration {
		if count == 0 || remaining <= time.Duration(15*time.Second) {
			count = updInterval
			normalized = remaining
			lastCall = time.Now()
			return remaining
		}
		count--
		normalized -= time.Since(lastCall)
		lastCall = time.Now()
		if normalized > 0 {
			return normalized
		}
		return remaining
	})
}

func chooseTimeProducer(style TimeStyle) func(time.Duration) string {
	switch style {
	case ET_STYLE_HHMMSS:
		return func(remaining time.Duration) string {
			hours := int64(remaining/time.Hour) % 60
			minutes := int64(remaining/time.Minute) % 60
			seconds := int64(remaining/time.Second) % 60
			return fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds)
		}
	case ET_STYLE_HHMM:
		return func(remaining time.Duration) string {
			hours := int64(remaining/time.Hour) % 60
			minutes := int64(remaining/time.Minute) % 60
			return fmt.Sprintf("%02d:%02d", hours, minutes)
		}
	case ET_STYLE_MMSS:
		return func(remaining time.Duration) string {
			hours := int64(remaining/time.Hour) % 60
			minutes := int64(remaining/time.Minute) % 60
			seconds := int64(remaining/time.Second) % 60
			if hours > 0 {
				return fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds)
			}
			return fmt.Sprintf("%02d:%02d", minutes, seconds)
		}
	default:
		return func(remaining time.Duration) string {
			// strip off nanoseconds
			return ((remaining / time.Second) * time.Second).String()
		}
	}
}
<commit_msg>TimeNormalizer option for average eta<commit_after>package decor

import (
	"fmt"
	"math"
	"time"

	"github.com/VividCortex/ewma"
)

// TimeNormalizer interface. Implementors could be passed into
// MovingAverageETA, in order to affect i.e. normalize its output.
type TimeNormalizer interface {
	Normalize(time.Duration) time.Duration
}

// TimeNormalizerFunc is function type adapter to convert function
// into TimeNormalizer.
type TimeNormalizerFunc func(time.Duration) time.Duration

func (f TimeNormalizerFunc) Normalize(src time.Duration) time.Duration {
	return f(src)
}

// EwmaETA exponential-weighted-moving-average based ETA decorator.
// Note that it's necessary to supply bar.Incr* methods with incremental
// work duration as second argument, in order for this decorator to
// work correctly. This decorator is a wrapper of MovingAverageETA.
func EwmaETA(style TimeStyle, age float64, wcc ...WC) Decorator {
	var average MovingAverage
	if age == 0 {
		average = ewma.NewMovingAverage()
	} else {
		average = ewma.NewMovingAverage(age)
	}
	return MovingAverageETA(style, average, nil, wcc...)
}

// MovingAverageETA decorator relies on MovingAverage implementation to calculate its average.
//
//	`style` one of [ET_STYLE_GO|ET_STYLE_HHMMSS|ET_STYLE_HHMM|ET_STYLE_MMSS]
//
//	`average` implementation of MovingAverage interface
//
//	`normalizer` available implementations are [FixedIntervalTimeNormalizer|MaxTolerateTimeNormalizer]
//
//	`wcc` optional WC config
func MovingAverageETA(style TimeStyle, average MovingAverage, normalizer TimeNormalizer, wcc ...WC) Decorator {
	var wc WC
	for _, widthConf := range wcc {
		wc = widthConf
	}
	wc.Init()
	d := &movingAverageETA{
		WC:         wc,
		average:    average,
		normalizer: normalizer,
		producer:   chooseTimeProducer(style),
	}
	return d
}

type movingAverageETA struct {
	WC
	average     ewma.MovingAverage
	normalizer  TimeNormalizer
	producer    func(time.Duration) string
	completeMsg *string
}

func (d *movingAverageETA) Decor(st *Statistics) string {
	if st.Completed && d.completeMsg != nil {
		return d.FormatMsg(*d.completeMsg)
	}

	v := math.Round(d.average.Value())
	remaining := time.Duration((st.Total - st.Current) * int64(v))
	if d.normalizer != nil {
		remaining = d.normalizer.Normalize(remaining)
	}
	return d.FormatMsg(d.producer(remaining))
}

func (d *movingAverageETA) NextAmount(n int64, wdd ...time.Duration) {
	var workDuration time.Duration
	for _, wd := range wdd {
		workDuration = wd
	}
	durPerItem := float64(workDuration) / float64(n)
	if math.IsInf(durPerItem, 0) || math.IsNaN(durPerItem) {
		return
	}
	d.average.Add(durPerItem)
}

func (d *movingAverageETA) OnCompleteMessage(msg string) {
	d.completeMsg = &msg
}

// AverageETA decorator. It's wrapper of NewAverageETA.
//
//	`style` one of [ET_STYLE_GO|ET_STYLE_HHMMSS|ET_STYLE_HHMM|ET_STYLE_MMSS]
//
//	`wcc` optional WC config
func AverageETA(style TimeStyle, wcc ...WC) Decorator {
	return NewAverageETA(style, time.Now(), nil, wcc...)
}

// NewAverageETA decorator with user provided start time.
//
//	`style` one of [ET_STYLE_GO|ET_STYLE_HHMMSS|ET_STYLE_HHMM|ET_STYLE_MMSS]
//
//	`startTime` start time
//
//	`normalizer` available implementations are [FixedIntervalTimeNormalizer|MaxTolerateTimeNormalizer]
//
//	`wcc` optional WC config
func NewAverageETA(style TimeStyle, startTime time.Time, normalizer TimeNormalizer, wcc ...WC) Decorator {
	var wc WC
	for _, widthConf := range wcc {
		wc = widthConf
	}
	wc.Init()
	d := &averageETA{
		WC:         wc,
		startTime:  startTime,
		normalizer: normalizer,
		producer:   chooseTimeProducer(style),
	}
	return d
}

type averageETA struct {
	WC
	startTime   time.Time
	normalizer  TimeNormalizer
	producer    func(time.Duration) string
	completeMsg *string
}

func (d *averageETA) Decor(st *Statistics) string {
	if st.Completed && d.completeMsg != nil {
		return d.FormatMsg(*d.completeMsg)
	}

	var remaining time.Duration
	if st.Current != 0 {
		durPerItem := float64(time.Since(d.startTime)) / float64(st.Current)
		durPerItem = math.Round(durPerItem)
		remaining = time.Duration((st.Total - st.Current) * int64(durPerItem))
		if d.normalizer != nil {
			remaining = d.normalizer.Normalize(remaining)
		}
	}
	return d.FormatMsg(d.producer(remaining))
}

func (d *averageETA) OnCompleteMessage(msg string) {
	d.completeMsg = &msg
}

func (d *averageETA) AverageAdjust(startTime time.Time) {
	d.startTime = startTime
}

// MaxTolerateTimeNormalizer returns implementation of TimeNormalizer.
func MaxTolerateTimeNormalizer(maxTolerate time.Duration) TimeNormalizer {
	var normalized time.Duration
	var lastCall time.Time
	return TimeNormalizerFunc(func(remaining time.Duration) time.Duration {
		if diff := normalized - remaining; diff <= 0 || diff > maxTolerate || remaining < maxTolerate/2 {
			normalized = remaining
			lastCall = time.Now()
			return remaining
		}
		normalized -= time.Since(lastCall)
		lastCall = time.Now()
		return normalized
	})
}

// FixedIntervalTimeNormalizer returns implementation of TimeNormalizer.
func FixedIntervalTimeNormalizer(updInterval int) TimeNormalizer {
	var normalized time.Duration
	var lastCall time.Time
	var count int
	return TimeNormalizerFunc(func(remaining time.Duration) time.Duration {
		if count == 0 || remaining <= time.Duration(15*time.Second) {
			count = updInterval
			normalized = remaining
			lastCall = time.Now()
			return remaining
		}
		count--
		normalized -= time.Since(lastCall)
		lastCall = time.Now()
		if normalized > 0 {
			return normalized
		}
		return remaining
	})
}

func chooseTimeProducer(style TimeStyle) func(time.Duration) string {
	switch style {
	case ET_STYLE_HHMMSS:
		return func(remaining time.Duration) string {
			hours := int64(remaining/time.Hour) % 60
			minutes := int64(remaining/time.Minute) % 60
			seconds := int64(remaining/time.Second) % 60
			return fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds)
		}
	case ET_STYLE_HHMM:
		return func(remaining time.Duration) string {
			hours := int64(remaining/time.Hour) % 60
			minutes := int64(remaining/time.Minute) % 60
			return fmt.Sprintf("%02d:%02d", hours, minutes)
		}
	case ET_STYLE_MMSS:
		return func(remaining time.Duration) string {
			hours := int64(remaining/time.Hour) % 60
			minutes := int64(remaining/time.Minute) % 60
			seconds := int64(remaining/time.Second) % 60
			if hours > 0 {
				return fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds)
			}
			return fmt.Sprintf("%02d:%02d", minutes, seconds)
		}
	default:
		return func(remaining time.Duration) string {
			// strip off nanoseconds
			return ((remaining / time.Second) * time.Second).String()
		}
	}
}
<|endoftext|>


<commit_before>package prometheus

import (
	"fmt"
	"github.com/spf13/afero"
	"github.com/stretchr/testify/suite"
	"os"
	"testing"
)

type ConfigTestSuite struct {
	suite.Suite
}

func (s *ConfigTestSuite) SetupTest() {
}

func TestConfigUnitTestSuite(t *testing.T) {
	s := new(ConfigTestSuite)
	logPrintlnOrig := logPrintf
	defer func() { logPrintf = logPrintlnOrig }()
	logPrintf = func(format string, v ...interface{}) {}
	suite.Run(t, s)
}

// GetRemoteConfig

func (s *ConfigTestSuite) Test_GetRemoteConfig_ReturnsEmptyString_WhenEnvVarsAreNotSet() {
	actual := GetRemoteConfig()

	s.Empty(actual)
}

func (s *ConfigTestSuite) Test_GetRemoteConfig_ReturnsRemoteWriteUrl() {
	defer func() { os.Unsetenv("REMOTE_WRITE_URL") }()
	os.Setenv("REMOTE_WRITE_URL", "http://acme.com/write")
	expected := `
remote_write:
  url: http://acme.com/write`
	actual := GetRemoteConfig()

	s.Equal(expected, actual)
}

func (s *ConfigTestSuite) Test_GetRemoteConfig_ReturnsRemoteReadUrl() {
	defer func() { os.Unsetenv("REMOTE_READ_URL") }()
	os.Setenv("REMOTE_READ_URL", "http://acme.com/read")
	expected := `
remote_read:
  url: http://acme.com/read`
	actual := GetRemoteConfig()

	s.Equal(expected, actual)
}

// GetGlobalConfig

func (s *ConfigTestSuite) Test_GlobalConfig_ReturnsConfigWithData() {
	scrapeIntervalOrig := os.Getenv("GLOBAL_SCRAPE_INTERVAL")
	defer func() { os.Setenv("GLOBAL_SCRAPE_INTERVAL", scrapeIntervalOrig) }()
	os.Setenv("GLOBAL_SCRAPE_INTERVAL", "123s")
	expected := `
global:
  scrape_interval: 123s`

	actual := GetGlobalConfig()
	s.Equal(expected, actual)
}

func (s *ConfigTestSuite) Test_GlobalConfig_AllowsNestedEntries() {
	scrapeIntervalOrig := os.Getenv("GLOBAL_SCRAPE_INTERVAL")
	defer func() {
		os.Setenv("GLOBAL_SCRAPE_INTERVAL", scrapeIntervalOrig)
		os.Unsetenv("GLOBAL_EXTERNAL_LABELS-CLUSTER")
		os.Unsetenv("GLOBAL_EXTERNAL_LABELS-TYPE")
	}()
	os.Setenv("GLOBAL_SCRAPE_INTERVAL", "123s")
	os.Setenv("GLOBAL_EXTERNAL_LABELS-CLUSTER", "swarm")
	os.Setenv("GLOBAL_EXTERNAL_LABELS-TYPE", "production")
	expected := `
global:
  scrape_interval: 123s
  external_labels:
    cluster: swarm
    type: production`
	actual := ""

	// Because of ordering, the config is not always the same so we're repeating a failure for a few times.
	for i := 0; i < 5; i++ {
		actual = GetGlobalConfig()
		if actual == expected {
			return
		}
	}
	s.Equal(expected, actual)
}

// GetScrapeConfig

func (s *ConfigTestSuite) Test_GetScrapeConfig_ReturnsConfigWithData() {
	expected := `
scrape_configs:
  - job_name: "service-1"
    metrics_path: /metrics
    dns_sd_configs:
      - names: ["tasks.service-1"]
        type: A
        port: 1234
  - job_name: "service-2"
    metrics_path: /metrics
    dns_sd_configs:
      - names: ["tasks.service-2"]
        type: A
        port: 5678
  - job_name: "service-3"
    metrics_path: /something
    static_configs:
      - targets:
        - service-3:4321
`
	scrapes := map[string]Scrape{
		"service-1": {ServiceName: "service-1", ScrapePort: 1234},
		"service-2": {ServiceName: "service-2", ScrapePort: 5678},
		"service-3": {ServiceName: "service-3", ScrapePort: 4321, ScrapeType: "static_configs", MetricsPath: "/something"},
	}

	actual := GetScrapeConfig(scrapes)

	s.Equal(expected, actual)
}

func (s *ConfigTestSuite) Test_GetScrapeConfig_ReturnsConfigWithDataAndSecrets() {
	fsOrig := FS
	defer func() { FS = fsOrig }()
	FS = afero.NewMemMapFs()
    job2 := `  - job_name: "service-2"
    metrics_path: /metrics
    dns_sd_configs:
      - names: ["tasks.service-2"]
        type: A
        port: 5678`
    job3 := `  - job_name: "service-3"
    metrics_path: /metrics
    dns_sd_configs:
      - names: ["tasks.service-3"]
        port: 9999
`
	expected := fmt.Sprintf(`
scrape_configs:
  - job_name: "service-1"
    metrics_path: /metrics
    dns_sd_configs:
      - names: ["tasks.service-1"]
        type: A
        port: 1234
%s
%s`,
		job2,
		job3,
	)
	scrapes := map[string]Scrape{
		"service-1": {ServiceName: "service-1", ScrapePort: 1234},
	}
	afero.WriteFile(FS, "/run/secrets/scrape_job2", []byte(job2), 0644)
	afero.WriteFile(FS, "/run/secrets/scrape_job3", []byte(job3), 0644)

	actual := GetScrapeConfig(scrapes)

	s.Equal(expected, actual)
}

func (s *ConfigTestSuite) Test_GetScrapeConfig_ReturnsOnlySecretsWithScrapePrefix() {
	fsOrig := FS
	defer func() { FS = fsOrig }()
	FS = afero.NewMemMapFs()
	job := `  - job_name: "my-service"
    dns_sd_configs:
      - names: ["tasks.my-service"]
        type: A
        port: 5678`
	expected := fmt.Sprintf(`
scrape_configs:
%s
`,
		job,
	)
	scrapes := map[string]Scrape{}
	afero.WriteFile(FS, "/run/secrets/scrape_job", []byte(job), 0644)
	afero.WriteFile(FS, "/run/secrets/job_without_scrape_prefix", []byte("something silly"), 0644)

	actual := GetScrapeConfig(scrapes)

	s.Equal(expected, actual)
}

func (s *ConfigTestSuite) Test_GetScrapeConfig_ReturnsConfigsFromCustomDirectory() {
	fsOrig := FS
	defer func() { FS = fsOrig }()
	FS = afero.NewMemMapFs()
	defer func() { os.Unsetenv("CONFIGS_DIR") }()
	os.Setenv("CONFIGS_DIR", "/tmp")
	job := `  - job_name: "my-service"
    dns_sd_configs:
      - names: ["tasks.my-service"]
        type: A
        port: 5678`
	expected := fmt.Sprintf(`
scrape_configs:
%s
`,
		job,
	)
	scrapes := map[string]Scrape{}
	afero.WriteFile(FS, "/tmp/scrape_job", []byte(job), 0644)

	actual := GetScrapeConfig(scrapes)

	s.Equal(expected, actual)
}

func (s *ConfigTestSuite) Test_GetScrapeConfig_ReturnsEmptyString_WhenNoData() {
	actual := GetScrapeConfig(map[string]Scrape{})

	s.Empty(actual)
}

// WriteConfig

func (s *ConfigTestSuite) Test_WriteConfig_WritesConfig() {
	fsOrig := FS
	defer func() {
		FS = fsOrig
		os.Unsetenv("REMOTE_WRITE_URL")
		os.Unsetenv("REMOTE_READ_URL")
	}()
	os.Setenv("REMOTE_WRITE_URL", "http://acme.com/write")
	os.Setenv("REMOTE_READ_URL", "http://acme.com/read")
	FS = afero.NewMemMapFs()
	scrapes := map[string]Scrape{
		"service-1": {ServiceName: "service-1", ScrapePort: 1234},
		"service-2": {ServiceName: "service-2", ScrapePort: 5678},
	}
	gc := GetGlobalConfig()
	sc := GetScrapeConfig(scrapes)
	rc := GetRemoteConfig()
	expected := fmt.Sprintf(`%s
%s
%s`,
		gc,
		sc,
		rc,
	)
	println("000")
	println(rc)
	println("111")

	WriteConfig(scrapes, map[string]Alert{})

	actual, _ := afero.ReadFile(FS, "/etc/prometheus/prometheus.yml")
	s.Equal(expected, string(actual))
	//	s.Fail(string(actual))
}

func (s *ConfigTestSuite) Test_WriteConfig_WritesAlerts() {
	fsOrig := FS
	defer func() { FS = fsOrig }()
	FS = afero.NewMemMapFs()
	alerts := map[string]Alert{}
	alerts["myalert"] = Alert{
		ServiceName:        "my-service",
		AlertName:          "alert-name",
		AlertNameFormatted: "myservicealertname",
		AlertIf:            "a>b",
	}
	gc := GetGlobalConfig()
	expectedConfig := fmt.Sprintf(`%s

rule_files:
  - 'alert.rules'
`,
		gc,
	)
	expectedAlerts := GetAlertConfig(alerts)

	WriteConfig(map[string]Scrape{}, alerts)

	actualConfig, _ := afero.ReadFile(FS, "/etc/prometheus/prometheus.yml")
	s.Equal(expectedConfig, string(actualConfig))
	actualAlerts, _ := afero.ReadFile(FS, "/etc/prometheus/alert.rules")
	s.Equal(expectedAlerts, string(actualAlerts))
}
<commit_msg>fixed tests to match changes introduced to recognise static targets<commit_after>package prometheus

import (
	"fmt"
	"github.com/spf13/afero"
	"github.com/stretchr/testify/suite"
	"os"
	"testing"
)

type ConfigTestSuite struct {
	suite.Suite
}

func (s *ConfigTestSuite) SetupTest() {
}

func TestConfigUnitTestSuite(t *testing.T) {
	s := new(ConfigTestSuite)
	logPrintlnOrig := logPrintf
	defer func() { logPrintf = logPrintlnOrig }()
	logPrintf = func(format string, v ...interface{}) {}
	suite.Run(t, s)
}

// GetRemoteConfig

func (s *ConfigTestSuite) Test_GetRemoteConfig_ReturnsEmptyString_WhenEnvVarsAreNotSet() {
	actual := GetRemoteConfig()

	s.Empty(actual)
}

func (s *ConfigTestSuite) Test_GetRemoteConfig_ReturnsRemoteWriteUrl() {
	defer func() { os.Unsetenv("REMOTE_WRITE_URL") }()
	os.Setenv("REMOTE_WRITE_URL", "http://acme.com/write")
	expected := `
remote_write:
  url: http://acme.com/write`
	actual := GetRemoteConfig()

	s.Equal(expected, actual)
}

func (s *ConfigTestSuite) Test_GetRemoteConfig_ReturnsRemoteReadUrl() {
	defer func() { os.Unsetenv("REMOTE_READ_URL") }()
	os.Setenv("REMOTE_READ_URL", "http://acme.com/read")
	expected := `
remote_read:
  url: http://acme.com/read`
	actual := GetRemoteConfig()

	s.Equal(expected, actual)
}

// GetGlobalConfig

func (s *ConfigTestSuite) Test_GlobalConfig_ReturnsConfigWithData() {
	scrapeIntervalOrig := os.Getenv("GLOBAL_SCRAPE_INTERVAL")
	defer func() { os.Setenv("GLOBAL_SCRAPE_INTERVAL", scrapeIntervalOrig) }()
	os.Setenv("GLOBAL_SCRAPE_INTERVAL", "123s")
	expected := `
global:
  scrape_interval: 123s`

	actual := GetGlobalConfig()
	s.Equal(expected, actual)
}

func (s *ConfigTestSuite) Test_GlobalConfig_AllowsNestedEntries() {
	scrapeIntervalOrig := os.Getenv("GLOBAL_SCRAPE_INTERVAL")
	defer func() {
		os.Setenv("GLOBAL_SCRAPE_INTERVAL", scrapeIntervalOrig)
		os.Unsetenv("GLOBAL_EXTERNAL_LABELS-CLUSTER")
		os.Unsetenv("GLOBAL_EXTERNAL_LABELS-TYPE")
	}()
	os.Setenv("GLOBAL_SCRAPE_INTERVAL", "123s")
	os.Setenv("GLOBAL_EXTERNAL_LABELS-CLUSTER", "swarm")
	os.Setenv("GLOBAL_EXTERNAL_LABELS-TYPE", "production")
	expected := `
global:
  scrape_interval: 123s
  external_labels:
    cluster: swarm
    type: production`
	actual := ""

	// Because of ordering, the config is not always the same so we're repeating a failure for a few times.
	for i := 0; i < 5; i++ {
		actual = GetGlobalConfig()
		if actual == expected {
			return
		}
	}
	s.Equal(expected, actual)
}

// GetScrapeConfig

func (s *ConfigTestSuite) Test_GetScrapeConfig_ReturnsConfigWithData() {
	expected := `
scrape_configs:
  - job_name: "service-1"
    metrics_path: /metrics
    dns_sd_configs:
      - names: ["tasks.service-1"]
        type: A
        port: 1234
  - job_name: "service-2"
    metrics_path: /metrics
    dns_sd_configs:
      - names: ["tasks.service-2"]
        type: A
        port: 5678
  - job_name: "service-3"
    metrics_path: /something
    static_configs:
      - targets:
        - ["service-3:4321"]
`
	scrapes := map[string]Scrape{
		"service-1": {ServiceName: "service-1", ScrapePort: 1234},
		"service-2": {ServiceName: "service-2", ScrapePort: 5678},
		"service-3": {ServiceName: "service-3", ScrapePort: 4321, ScrapeType: "static_configs", MetricsPath: "/something"},
	}

	actual := GetScrapeConfig(scrapes)

	s.Equal(expected, actual)
}

func (s *ConfigTestSuite) Test_GetScrapeConfig_ReturnsConfigWithDataAndSecrets() {
	fsOrig := FS
	defer func() { FS = fsOrig }()
	FS = afero.NewMemMapFs()
    job2 := `  - job_name: "service-2"
    metrics_path: /metrics
    dns_sd_configs:
      - names: ["tasks.service-2"]
        type: A
        port: 5678`
    job3 := `  - job_name: "service-3"
    metrics_path: /metrics
    dns_sd_configs:
      - names: ["tasks.service-3"]
        port: 9999
`
	expected := fmt.Sprintf(`
scrape_configs:
  - job_name: "service-1"
    metrics_path: /metrics
    dns_sd_configs:
      - names: ["tasks.service-1"]
        type: A
        port: 1234
%s
%s`,
		job2,
		job3,
	)
	scrapes := map[string]Scrape{
		"service-1": {ServiceName: "service-1", ScrapePort: 1234},
	}
	afero.WriteFile(FS, "/run/secrets/scrape_job2", []byte(job2), 0644)
	afero.WriteFile(FS, "/run/secrets/scrape_job3", []byte(job3), 0644)

	actual := GetScrapeConfig(scrapes)

	s.Equal(expected, actual)
}

func (s *ConfigTestSuite) Test_GetScrapeConfig_ReturnsOnlySecretsWithScrapePrefix() {
	fsOrig := FS
	defer func() { FS = fsOrig }()
	FS = afero.NewMemMapFs()
	job := `  - job_name: "my-service"
    dns_sd_configs:
      - names: ["tasks.my-service"]
        type: A
        port: 5678`
	expected := fmt.Sprintf(`
scrape_configs:
%s
`,
		job,
	)
	scrapes := map[string]Scrape{}
	afero.WriteFile(FS, "/run/secrets/scrape_job", []byte(job), 0644)
	afero.WriteFile(FS, "/run/secrets/job_without_scrape_prefix", []byte("something silly"), 0644)

	actual := GetScrapeConfig(scrapes)

	s.Equal(expected, actual)
}

func (s *ConfigTestSuite) Test_GetScrapeConfig_ReturnsConfigsFromCustomDirectory() {
	fsOrig := FS
	defer func() { FS = fsOrig }()
	FS = afero.NewMemMapFs()
	defer func() { os.Unsetenv("CONFIGS_DIR") }()
	os.Setenv("CONFIGS_DIR", "/tmp")
	job := `  - job_name: "my-service"
    dns_sd_configs:
      - names: ["tasks.my-service"]
        type: A
        port: 5678`
	expected := fmt.Sprintf(`
scrape_configs:
%s
`,
		job,
	)
	scrapes := map[string]Scrape{}
	afero.WriteFile(FS, "/tmp/scrape_job", []byte(job), 0644)

	actual := GetScrapeConfig(scrapes)

	s.Equal(expected, actual)
}

func (s *ConfigTestSuite) Test_GetScrapeConfig_ReturnsEmptyString_WhenNoData() {
	actual := GetScrapeConfig(map[string]Scrape{})

	s.Empty(actual)
}

// WriteConfig

func (s *ConfigTestSuite) Test_WriteConfig_WritesConfig() {
	fsOrig := FS
	defer func() {
		FS = fsOrig
		os.Unsetenv("REMOTE_WRITE_URL")
		os.Unsetenv("REMOTE_READ_URL")
	}()
	os.Setenv("REMOTE_WRITE_URL", "http://acme.com/write")
	os.Setenv("REMOTE_READ_URL", "http://acme.com/read")
	FS = afero.NewMemMapFs()
	scrapes := map[string]Scrape{
		"service-1": {ServiceName: "service-1", ScrapePort: 1234},
		"service-2": {ServiceName: "service-2", ScrapePort: 5678},
	}
	gc := GetGlobalConfig()
	sc := GetScrapeConfig(scrapes)
	rc := GetRemoteConfig()
	expected := fmt.Sprintf(`%s
%s
%s`,
		gc,
		sc,
		rc,
	)
	println("000")
	println(rc)
	println("111")

	WriteConfig(scrapes, map[string]Alert{})

	actual, _ := afero.ReadFile(FS, "/etc/prometheus/prometheus.yml")
	s.Equal(expected, string(actual))
	//	s.Fail(string(actual))
}

func (s *ConfigTestSuite) Test_WriteConfig_WritesAlerts() {
	fsOrig := FS
	defer func() { FS = fsOrig }()
	FS = afero.NewMemMapFs()
	alerts := map[string]Alert{}
	alerts["myalert"] = Alert{
		ServiceName:        "my-service",
		AlertName:          "alert-name",
		AlertNameFormatted: "myservicealertname",
		AlertIf:            "a>b",
	}
	gc := GetGlobalConfig()
	expectedConfig := fmt.Sprintf(`%s

rule_files:
  - 'alert.rules'
`,
		gc,
	)
	expectedAlerts := GetAlertConfig(alerts)

	WriteConfig(map[string]Scrape{}, alerts)

	actualConfig, _ := afero.ReadFile(FS, "/etc/prometheus/prometheus.yml")
	s.Equal(expectedConfig, string(actualConfig))
	actualAlerts, _ := afero.ReadFile(FS, "/etc/prometheus/alert.rules")
	s.Equal(expectedAlerts, string(actualAlerts))
}
<|endoftext|>


<commit_before>package protocol

type FetchPartition struct {
	Partition   int32
	FetchOffset int64
	MaxBytes    int32
}

type FetchTopic struct {
	Topic      string
	Partitions []*FetchPartition
}

type FetchRequest struct {
	ReplicaID   int32
	MaxWaitTime int32
	MinBytes    int32
	MaxBytes    int32
	Topics      []*FetchTopic
}

func (r *FetchRequest) Encode(e PacketEncoder) error {
	e.PutInt32(r.ReplicaID)
	e.PutInt32(r.MaxWaitTime)
	e.PutInt32(r.MinBytes)
	e.PutInt32(r.MaxBytes)
	e.PutArrayLength(len(r.Topics))
	for _, t := range r.Topics {
		e.PutString(t.Topic)
		e.PutArrayLength(len(t.Partitions))
		for _, p := range t.Partitions {
			e.PutInt32(p.Partition)
			e.PutInt64(p.FetchOffset)
			e.PutInt32(p.MaxBytes)
		}
	}
	return nil
}

func (r *FetchRequest) Decode(d PacketDecoder) error {
	var err error
	r.ReplicaID, err = d.Int32()
	if err != nil {
		return err
	}
	r.MaxWaitTime, err = d.Int32()
	if err != nil {
		return err
	}
	r.MinBytes, err = d.Int32()
	if err != nil {
		return err
	}
	r.MaxBytes, err = d.Int32()
	if err != nil {
		return err
	}
	tlen, err := d.ArrayLength()
	if err != nil {
		return err
	}
	topics := make([]*FetchTopic, tlen)
	for i := range topics {
		t := &FetchTopic{}
		t.Topic, err = d.String()
		if err != nil {
			return err
		}
		plen, err := d.ArrayLength()
		if err != nil {
			return err
		}
		ps := make([]*FetchPartition, plen)
		for j := range ps {
			p := &FetchPartition{}
			p.Partition, err = d.Int32()
			if err != nil {
				return err
			}
			p.FetchOffset, err = d.Int64()
			if err != nil {
				return err
			}
			p.MaxBytes, err = d.Int32()
			if err != nil {
				return err
			}
			ps[j] = p
		}
		t.Partitions = ps
		topics[i] = t
	}
	r.Topics = topics
	return nil
}

func (r *FetchRequest) Key() int16 {
	return 1
}

func (r *FetchRequest) Version() int16 {
	return 3
}
<commit_msg>support version 1 fetch request<commit_after>package protocol

type FetchPartition struct {
	Partition   int32
	FetchOffset int64
	MaxBytes    int32
}

type FetchTopic struct {
	Topic      string
	Partitions []*FetchPartition
}

type FetchRequest struct {
	ReplicaID   int32
	MaxWaitTime int32
	MinBytes    int32
	// MaxBytes    int32
	Topics []*FetchTopic
}

func (r *FetchRequest) Encode(e PacketEncoder) error {
	e.PutInt32(-1) // replica ID is -1 for clients
	e.PutInt32(r.MaxWaitTime)
	e.PutInt32(r.MinBytes)
	// e.PutInt32(r.MaxBytes)
	e.PutArrayLength(len(r.Topics))
	for _, t := range r.Topics {
		e.PutString(t.Topic)
		e.PutArrayLength(len(t.Partitions))
		for _, p := range t.Partitions {
			e.PutInt32(p.Partition)
			e.PutInt64(p.FetchOffset)
			e.PutInt32(p.MaxBytes)
		}
	}
	return nil
}

func (r *FetchRequest) Decode(d PacketDecoder) error {
	var err error
	r.ReplicaID, err = d.Int32()
	if err != nil {
		return err
	}
	r.MaxWaitTime, err = d.Int32()
	if err != nil {
		return err
	}
	r.MinBytes, err = d.Int32()
	if err != nil {
		return err
	}
	// r.MaxBytes, err = d.Int32()
	// if err != nil {
	// 	return err
	// }
	tlen, err := d.ArrayLength()
	if err != nil {
		return err
	}
	topics := make([]*FetchTopic, tlen)
	for i := range topics {
		t := &FetchTopic{}
		t.Topic, err = d.String()
		if err != nil {
			return err
		}
		plen, err := d.ArrayLength()
		if err != nil {
			return err
		}
		ps := make([]*FetchPartition, plen)
		for j := range ps {
			p := &FetchPartition{}
			p.Partition, err = d.Int32()
			if err != nil {
				return err
			}
			p.FetchOffset, err = d.Int64()
			if err != nil {
				return err
			}
			p.MaxBytes, err = d.Int32()
			if err != nil {
				return err
			}
			ps[j] = p
		}
		t.Partitions = ps
		topics[i] = t
	}
	r.Topics = topics
	return nil
}

func (r *FetchRequest) Key() int16 {
	return 1
}

func (r *FetchRequest) Version() int16 {
	return 1
}
<|endoftext|>


<commit_before>package version

// VERSION ...
const VERSION = "1.1.60"
<commit_msg>v1.1.61 [skip ci]<commit_after>package version

// VERSION ...
const VERSION = "1.1.61"
<|endoftext|>


<commit_before>// package version provide the version info for the node, and also provide
// support for version compatibility check and update notification.
//
// The version format should follow Semantic Versioning (https://semver.org/):
// MAJOR.MINOR.PATCH
//  1. MAJOR version when you make incompatible API changes,
//  2. MINOR version when you add functionality in a backwards-compatible manner, and
// 	3. PATCH version when you make backwards-compatible bug fixes.
//
// A pre-release version MAY be denoted by appending a hyphen and a series of
// dot separated identifiers immediately following the patch version.
// Examples:
// 1.0.0-alpha, 1.0.0-alpha.1, 1.0.0-0.3.7, 1.0.0-x.7.z.92.
// Precedence:
// 1. Pre-release versions have a lower precedence than the associated normal version!
//    Numeric identifiers always have lower precedence than non-numeric identifiers.
// 2. A larger set of pre-release fields has a higher precedence than a smaller set,
//    if all of the preceding identifiers are equal.
// 3. Example:
//    1.0.0-alpha < 1.0.0-alpha.1 < 1.0.0-alpha.beta < 1.0.0-beta < 1.0.0-beta.2 < 1.0.0-beta.11 < 1.0.0-rc.1 < 1.0.0.
//
// Build metadata MAY be denoted by appending a plus sign and a series of dot
// separated identifiers immediately following the patch or pre-release version.
// Build metadata SHOULD be ignored when determining version precedence. Thus
// two versions that differ only in the build metadata, have the same precedence.

package version

import (
	"sync"

	gover "github.com/hashicorp/go-version"
	log "github.com/sirupsen/logrus"
	"gopkg.in/fatih/set.v0"
)

const (
	// If needing to edit the iota, please ensure the following:
	// noUpdate = 0
	// hasUpdate = 1
	// hasMUpdate = 2
	noUpdate uint16 = iota
	hasUpdate
	hasMUpdate
	logModule = "version"
)

var (
	// The full version string
	Version = "2.0.4"
	// GitCommit is set with --ldflags "-X main.gitCommit=$(git rev-parse HEAD)"
	GitCommit string
	Status    *UpdateStatus
)

func init() {
	if GitCommit != "" {
		Version += "+" + GitCommit[:8]
	}

	Status = &UpdateStatus{
		maxVerSeen:    Version,
		notified:      false,
		seedSet:       set.New(),
		versionStatus: noUpdate,
	}
}

type UpdateStatus struct {
	sync.RWMutex
	maxVerSeen    string
	notified      bool
	seedSet       *set.Set
	versionStatus uint16
}

func (s *UpdateStatus) AddSeed(seedAddr string) {
	s.Lock()
	defer s.Unlock()
	s.seedSet.Add(seedAddr)
}

// CheckUpdate checks whether there is a newer version to update.
// If there is, it set the "Status" variable to a proper value.
// 	params:
// 		localVerStr: the version of the node itself
// 		remoteVerStr: the version received from a seed node.
// 		remoteAddr: the version received from a seed node.
// current rule:
// 		1. small update: seed version is higher than the node itself
// 		2. significant update: seed mojor version is higher than the node itself
func (s *UpdateStatus) CheckUpdate(localVerStr string, remoteVerStr string, remoteAddr string) error {
	s.Lock()
	defer s.Unlock()

	if !s.seedSet.Has(remoteAddr) {
		return nil
	}

	localVersion, err := gover.NewVersion(localVerStr)
	if err != nil {
		return err
	}
	remoteVersion, err := gover.NewVersion(remoteVerStr)
	if err != nil {
		return err
	}
	if remoteVersion.GreaterThan(localVersion) {
		if s.versionStatus == noUpdate {
			s.versionStatus = hasUpdate
		}

		maxVersion, err := gover.NewVersion(s.maxVerSeen)
		if err != nil {
			return err
		}

		if remoteVersion.GreaterThan(maxVersion) {
			s.maxVerSeen = remoteVerStr
		}
	}
	if remoteVersion.Segments()[0] > localVersion.Segments()[0] {
		s.versionStatus = hasMUpdate
	}
	if s.versionStatus != noUpdate {
		log.WithFields(log.Fields{
			"module":          logModule,
			"Current version": localVerStr,
			"Newer version":   remoteVerStr,
			"seed":            remoteAddr,
		}).Warn("Please update your bytomd via https://github.com/Bytom/bytom/releases/ or http://bytom.io/wallet/")
		s.notified = true
	}
	return nil
}

func (s *UpdateStatus) MaxVerSeen() string {
	s.RLock()
	defer s.RUnlock()
	return s.maxVerSeen
}

func (s *UpdateStatus) VersionStatus() uint16 {
	s.RLock()
	defer s.RUnlock()
	return s.versionStatus
}

// CompatibleWith checks whether the remote peer version is compatible with the
// node itself.
// RULES:
// | local |           remote           |
// |   -   |             -              |
// | 1.0.3 | same major&moinor version. |
// | 1.0.4 |     same major version.    |
func CompatibleWith(remoteVerStr string) (bool, error) {
	localVersion, err := gover.NewVersion(Version)
	if err != nil {
		return false, err
	}
	remoteVersion, err := gover.NewVersion(remoteVerStr)
	if err != nil {
		return false, err
	}
	return (localVersion.Segments()[0] == remoteVersion.Segments()[0]), nil
}
<commit_msg>chore(version): update version to 2.0.5 (#2102)<commit_after>// package version provide the version info for the node, and also provide
// support for version compatibility check and update notification.
//
// The version format should follow Semantic Versioning (https://semver.org/):
// MAJOR.MINOR.PATCH
//  1. MAJOR version when you make incompatible API changes,
//  2. MINOR version when you add functionality in a backwards-compatible manner, and
// 	3. PATCH version when you make backwards-compatible bug fixes.
//
// A pre-release version MAY be denoted by appending a hyphen and a series of
// dot separated identifiers immediately following the patch version.
// Examples:
// 1.0.0-alpha, 1.0.0-alpha.1, 1.0.0-0.3.7, 1.0.0-x.7.z.92.
// Precedence:
// 1. Pre-release versions have a lower precedence than the associated normal version!
//    Numeric identifiers always have lower precedence than non-numeric identifiers.
// 2. A larger set of pre-release fields has a higher precedence than a smaller set,
//    if all of the preceding identifiers are equal.
// 3. Example:
//    1.0.0-alpha < 1.0.0-alpha.1 < 1.0.0-alpha.beta < 1.0.0-beta < 1.0.0-beta.2 < 1.0.0-beta.11 < 1.0.0-rc.1 < 1.0.0.
//
// Build metadata MAY be denoted by appending a plus sign and a series of dot
// separated identifiers immediately following the patch or pre-release version.
// Build metadata SHOULD be ignored when determining version precedence. Thus
// two versions that differ only in the build metadata, have the same precedence.

package version

import (
	"sync"

	gover "github.com/hashicorp/go-version"
	log "github.com/sirupsen/logrus"
	"gopkg.in/fatih/set.v0"
)

const (
	// If needing to edit the iota, please ensure the following:
	// noUpdate = 0
	// hasUpdate = 1
	// hasMUpdate = 2
	noUpdate uint16 = iota
	hasUpdate
	hasMUpdate
	logModule = "version"
)

var (
	// The full version string
	Version = "2.0.5"
	// GitCommit is set with --ldflags "-X main.gitCommit=$(git rev-parse HEAD)"
	GitCommit string
	Status    *UpdateStatus
)

func init() {
	if GitCommit != "" {
		Version += "+" + GitCommit[:8]
	}

	Status = &UpdateStatus{
		maxVerSeen:    Version,
		notified:      false,
		seedSet:       set.New(),
		versionStatus: noUpdate,
	}
}

type UpdateStatus struct {
	sync.RWMutex
	maxVerSeen    string
	notified      bool
	seedSet       *set.Set
	versionStatus uint16
}

func (s *UpdateStatus) AddSeed(seedAddr string) {
	s.Lock()
	defer s.Unlock()
	s.seedSet.Add(seedAddr)
}

// CheckUpdate checks whether there is a newer version to update.
// If there is, it set the "Status" variable to a proper value.
// 	params:
// 		localVerStr: the version of the node itself
// 		remoteVerStr: the version received from a seed node.
// 		remoteAddr: the version received from a seed node.
// current rule:
// 		1. small update: seed version is higher than the node itself
// 		2. significant update: seed mojor version is higher than the node itself
func (s *UpdateStatus) CheckUpdate(localVerStr string, remoteVerStr string, remoteAddr string) error {
	s.Lock()
	defer s.Unlock()

	if !s.seedSet.Has(remoteAddr) {
		return nil
	}

	localVersion, err := gover.NewVersion(localVerStr)
	if err != nil {
		return err
	}
	remoteVersion, err := gover.NewVersion(remoteVerStr)
	if err != nil {
		return err
	}
	if remoteVersion.GreaterThan(localVersion) {
		if s.versionStatus == noUpdate {
			s.versionStatus = hasUpdate
		}

		maxVersion, err := gover.NewVersion(s.maxVerSeen)
		if err != nil {
			return err
		}

		if remoteVersion.GreaterThan(maxVersion) {
			s.maxVerSeen = remoteVerStr
		}
	}
	if remoteVersion.Segments()[0] > localVersion.Segments()[0] {
		s.versionStatus = hasMUpdate
	}
	if s.versionStatus != noUpdate {
		log.WithFields(log.Fields{
			"module":          logModule,
			"Current version": localVerStr,
			"Newer version":   remoteVerStr,
			"seed":            remoteAddr,
		}).Warn("Please update your bytomd via https://github.com/Bytom/bytom/releases/ or http://bytom.io/wallet/")
		s.notified = true
	}
	return nil
}

func (s *UpdateStatus) MaxVerSeen() string {
	s.RLock()
	defer s.RUnlock()
	return s.maxVerSeen
}

func (s *UpdateStatus) VersionStatus() uint16 {
	s.RLock()
	defer s.RUnlock()
	return s.versionStatus
}

// CompatibleWith checks whether the remote peer version is compatible with the
// node itself.
// RULES:
// | local |           remote           |
// |   -   |             -              |
// | 1.0.3 | same major&moinor version. |
// | 1.0.4 |     same major version.    |
func CompatibleWith(remoteVerStr string) (bool, error) {
	localVersion, err := gover.NewVersion(Version)
	if err != nil {
		return false, err
	}
	remoteVersion, err := gover.NewVersion(remoteVerStr)
	if err != nil {
		return false, err
	}
	return (localVersion.Segments()[0] == remoteVersion.Segments()[0]), nil
}
<|endoftext|>


<commit_before>package version

const (
	Version = "0.4.0"
)
<commit_msg>Reset version to 0.3.2.<commit_after>package version

const (
	Version = "0.3.2"
)
<|endoftext|>