<commit_before>package pipelineserver

import (
	"fmt"
	"sync"

	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"

	"golang.org/x/net/context"

	"go.pachyderm.com/pachyderm/src/pfs"
	"go.pachyderm.com/pachyderm/src/pps"
	"go.pedge.io/protolog"
)

type pipelineController struct {
	pfsAPIClient      pfs.APIClient
	jobAPIClient      pps.JobAPIClient
	pipelineAPIClient pps.PipelineAPIClient
	pipelineInfo      *pps.PipelineInfo

	ctx       context.Context
	cancel    context.CancelFunc
	waitGroup *sync.WaitGroup
}

func newPipelineController(
	pfsAPIClient pfs.APIClient,
	jobAPIClient pps.JobAPIClient,
	pipelineAPIClient pps.PipelineAPIClient,
	pipelineInfo *pps.PipelineInfo,
) *pipelineController {
	ctx, cancel := context.WithCancel(context.Background())
	return &pipelineController{
		pfsAPIClient,
		jobAPIClient,
		pipelineAPIClient,
		pipelineInfo,
		ctx,
		cancel,
		&sync.WaitGroup{},
	}
}

func (p *pipelineController) Start() error {
	// TODO(pedge): do not get all jobs each time, need a limit call on persist, more
	// generally, need all persist calls to have a limit
	jobInfos, err := p.jobAPIClient.ListJob(context.Background(), &pps.ListJobRequest{Pipeline: p.pipelineInfo.Pipeline})
	if err != nil {
		return err
	}
	lastCommit := &pfs.Commit{
		Repo: p.pipelineInfo.Input,
		// TODO(pedge): use initial commit id when moved to pfs package
		Id: "scratch",
	}
	if len(jobInfos.JobInfo) > 0 {
		lastCommit = jobInfos.JobInfo[0].Input
	}
	p.waitGroup.Add(1)
	go func() {
		defer p.waitGroup.Done()
		if err := p.run(lastCommit); ignoreCanceledError(err) != nil {
			// TODO(pedge): what to do with error?
			protolog.Errorln(err.Error())
		}
	}()
	return nil
}

func (p *pipelineController) Cancel() error {
	p.cancel()
	// does not block until run is complete, but run will be in the process of cancelling
	<-p.ctx.Done()
	// wait until run completes
	p.waitGroup.Wait()
	return ignoreCanceledError(p.ctx.Err())
}

func (p *pipelineController) run(lastCommit *pfs.Commit) error {
	for {
		// http://blog.golang.org/context
		commitErrorPairC := make(chan commitErrorPair, 1)
		go func() { commitErrorPairC <- p.runInner(p.ctx, lastCommit) }()
		select {
		case <-p.ctx.Done():
			_ = <-commitErrorPairC
			return ignoreCanceledError(p.ctx.Err())
		case commitErrorPair := <-commitErrorPairC:
			if ignoreCanceledError(commitErrorPair.Err) != nil {
				return commitErrorPair.Err
			}
			lastCommit = commitErrorPair.Commit
		}
	}
	return nil
}

type commitErrorPair struct {
	Commit *pfs.Commit
	Err    error
}

func (p *pipelineController) runInner(ctx context.Context, lastCommit *pfs.Commit) commitErrorPair {
	commitInfos, err := p.pfsAPIClient.ListCommit(
		ctx,
		&pfs.ListCommitRequest{
			Repo:       lastCommit.Repo,
			CommitType: pfs.CommitType_COMMIT_TYPE_READ,
			From:       lastCommit,
			Block:      true,
		},
	)
	if err != nil {
		return commitErrorPair{Err: err}
	}
	if len(commitInfos.CommitInfo) == 0 {
		return commitErrorPair{Err: fmt.Errorf("pachyderm.pps.pipelineserver: we expected at least one *pfs.CommitInfo returned from blocking call, but no *pfs.CommitInfo structs were returned for %v", lastCommit)}
	}
	// going in reverse order, oldest to newest
	for _, commitInfo := range commitInfos.CommitInfo {
		if err := p.createJobForCommitInfo(commitInfo); err != nil {
			return commitErrorPair{Err: err}
		}
	}
	return commitErrorPair{Commit: commitInfos.CommitInfo[len(commitInfos.CommitInfo)-1].Commit}
}

// TODO(pedge): implement
func (p *pipelineController) createJobForCommitInfo(commitInfo *pfs.CommitInfo) error {
	return nil
}

func ignoreCanceledError(err error) error {
	if err != context.Canceled && grpc.Code(err) != codes.Canceled {
		return err
	}
	return nil
}
<commit_msg>implement pps pipeline controller create job function<commit_after>package pipelineserver

import (
	"fmt"
	"sync"

	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"

	"golang.org/x/net/context"

	"go.pachyderm.com/pachyderm/src/pfs"
	"go.pachyderm.com/pachyderm/src/pps"
	"go.pedge.io/protolog"
)

type pipelineController struct {
	pfsAPIClient      pfs.APIClient
	jobAPIClient      pps.JobAPIClient
	pipelineAPIClient pps.PipelineAPIClient
	pipelineInfo      *pps.PipelineInfo

	ctx       context.Context
	cancel    context.CancelFunc
	waitGroup *sync.WaitGroup
}

func newPipelineController(
	pfsAPIClient pfs.APIClient,
	jobAPIClient pps.JobAPIClient,
	pipelineAPIClient pps.PipelineAPIClient,
	pipelineInfo *pps.PipelineInfo,
) *pipelineController {
	ctx, cancel := context.WithCancel(context.Background())
	return &pipelineController{
		pfsAPIClient,
		jobAPIClient,
		pipelineAPIClient,
		pipelineInfo,
		ctx,
		cancel,
		&sync.WaitGroup{},
	}
}

func (p *pipelineController) Start() error {
	// TODO(pedge): do not get all jobs each time, need a limit call on persist, more
	// generally, need all persist calls to have a limit
	jobInfos, err := p.jobAPIClient.ListJob(context.Background(), &pps.ListJobRequest{Pipeline: p.pipelineInfo.Pipeline})
	if err != nil {
		return err
	}
	lastCommit := &pfs.Commit{
		Repo: p.pipelineInfo.Input,
		// TODO(pedge): use initial commit id when moved to pfs package
		Id: "scratch",
	}
	if len(jobInfos.JobInfo) > 0 {
		lastCommit = jobInfos.JobInfo[0].Input
	}
	p.waitGroup.Add(1)
	go func() {
		defer p.waitGroup.Done()
		if err := p.run(lastCommit); ignoreCanceledError(err) != nil {
			// TODO(pedge): what to do with error?
			protolog.Errorln(err.Error())
		}
	}()
	return nil
}

func (p *pipelineController) Cancel() error {
	p.cancel()
	// does not block until run is complete, but run will be in the process of cancelling
	<-p.ctx.Done()
	// wait until run completes
	p.waitGroup.Wait()
	return ignoreCanceledError(p.ctx.Err())
}

func (p *pipelineController) run(lastCommit *pfs.Commit) error {
	for {
		// http://blog.golang.org/context
		commitErrorPairC := make(chan commitErrorPair, 1)
		go func() { commitErrorPairC <- p.runInner(p.ctx, lastCommit) }()
		select {
		case <-p.ctx.Done():
			_ = <-commitErrorPairC
			return ignoreCanceledError(p.ctx.Err())
		case commitErrorPair := <-commitErrorPairC:
			if ignoreCanceledError(commitErrorPair.Err) != nil {
				return commitErrorPair.Err
			}
			lastCommit = commitErrorPair.Commit
		}
	}
	return nil
}

type commitErrorPair struct {
	Commit *pfs.Commit
	Err    error
}

func (p *pipelineController) runInner(ctx context.Context, lastCommit *pfs.Commit) commitErrorPair {
	commitInfos, err := p.pfsAPIClient.ListCommit(
		ctx,
		&pfs.ListCommitRequest{
			Repo:       lastCommit.Repo,
			CommitType: pfs.CommitType_COMMIT_TYPE_READ,
			From:       lastCommit,
			Block:      true,
		},
	)
	if err != nil {
		return commitErrorPair{Err: err}
	}
	if len(commitInfos.CommitInfo) == 0 {
		return commitErrorPair{Err: fmt.Errorf("pachyderm.pps.pipelineserver: we expected at least one *pfs.CommitInfo returned from blocking call, but no *pfs.CommitInfo structs were returned for %v", lastCommit)}
	}
	// going in reverse order, oldest to newest
	for _, commitInfo := range commitInfos.CommitInfo {
		if err := p.createJobForCommitInfo(ctx, commitInfo); err != nil {
			return commitErrorPair{Err: err}
		}
	}
	return commitErrorPair{Commit: commitInfos.CommitInfo[len(commitInfos.CommitInfo)-1].Commit}
}

func (p *pipelineController) createJobForCommitInfo(ctx context.Context, commitInfo *pfs.CommitInfo) error {
	parentOutputCommit, err := p.getParentOutputCommit(commitInfo)
	if err != nil {
		return err
	}
	_, err = p.jobAPIClient.CreateJob(
		ctx,
		&pps.CreateJobRequest{
			Spec: &pps.CreateJobRequest_Pipeline{
				Pipeline: p.pipelineInfo.Pipeline,
			},
			Input:        commitInfo.Commit,
			OutputParent: parentOutputCommit,
		},
	)
	return err
}

func (p *pipelineController) getParentOutputCommit(commitInfo *pfs.CommitInfo) (*pfs.Commit, error) {
	return nil, nil
}

func ignoreCanceledError(err error) error {
	if err != context.Canceled && grpc.Code(err) != codes.Canceled {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package accessory

import (
	"github.com/brutella/hc/model"
	"github.com/brutella/hc/model/service"
)

type thermostat struct {
	*Accessory

	thermostat *service.Thermostat
}

// NewThermostat returns a thermostat which implements model.Thermostat.
func NewThermostat(info model.Info, temp, min, max, steps float64) *thermostat {
	accessory := New(info)
	t := service.NewThermostat(info.Name, temp, min, max, steps)

	accessory.AddService(t.Service)

	return &thermostat{accessory, t}
}

func (t *thermostat) Temperature() float64 {
	return t.thermostat.Temp.Temperature()
}

func (t *thermostat) SetTemperature(value float64) {
	t.thermostat.Temp.SetTemperature(value)
}

func (t *thermostat) Unit() model.TempUnit {
	return t.thermostat.Unit.Unit()
}

func (t *thermostat) SetTargetTemperature(value float64) {
	t.thermostat.TargetTemp.SetTemperature(value)
}

func (t *thermostat) TargetTemperature() float64 {
	return t.thermostat.TargetTemp.Temperature()
}

func (t *thermostat) SetMode(value model.HeatCoolModeType) {
	if value != model.HeatCoolModeAuto {
		t.thermostat.Mode.SetHeatingCoolingMode(value)
	}
}

func (t *thermostat) Mode() model.HeatCoolModeType {
	return t.thermostat.Mode.HeatingCoolingMode()
}

func (t *thermostat) SetTargetMode(value model.HeatCoolModeType) {
	t.thermostat.TargetMode.SetHeatingCoolingMode(value)
}

func (t *thermostat) TargetMode() model.HeatCoolModeType {
	return t.thermostat.TargetMode.HeatingCoolingMode()
}
<commit_msg>Add callback func for thermostat target.<commit_after>package accessory

import (
	"github.com/brutella/hc/model"
	"github.com/brutella/hc/model/characteristic"
	"github.com/brutella/hc/model/service"
	"net"
)

type thermostat struct {
	*Accessory

	thermostat         *service.Thermostat
	OnTargetTempChange func(float64)
}

// NewThermostat returns a thermostat which implements model.Thermostat.
func NewThermostat(info model.Info, temp, min, max, steps float64) *thermostat {
	accessory := New(info)
	t := service.NewThermostat(info.Name, temp, min, max, steps)

	accessory.AddService(t.Service)

	ts := thermostat{accessory, t, nil}

	t.TargetTemp.OnConnChange(func(conn net.Conn, c *characteristic.Characteristic, new, old interface{}) {
		if ts.OnTargetTempChange != nil {
			ts.OnTargetTempChange(t.TargetTemp.Temperature())
		}
	})

	return &ts
}

func (t *thermostat) Temperature() float64 {
	return t.thermostat.Temp.Temperature()
}

func (t *thermostat) SetTemperature(value float64) {
	t.thermostat.Temp.SetTemperature(value)
}

func (t *thermostat) Unit() model.TempUnit {
	return t.thermostat.Unit.Unit()
}

func (t *thermostat) SetTargetTemperature(value float64) {
	t.thermostat.TargetTemp.SetTemperature(value)
}

func (t *thermostat) TargetTemperature() float64 {
	return t.thermostat.TargetTemp.Temperature()
}

func (t *thermostat) SetMode(value model.HeatCoolModeType) {
	if value != model.HeatCoolModeAuto {
		t.thermostat.Mode.SetHeatingCoolingMode(value)
	}
}

func (t *thermostat) Mode() model.HeatCoolModeType {
	return t.thermostat.Mode.HeatingCoolingMode()
}

func (t *thermostat) SetTargetMode(value model.HeatCoolModeType) {
	t.thermostat.TargetMode.SetHeatingCoolingMode(value)
}

func (t *thermostat) TargetMode() model.HeatCoolModeType {
	return t.thermostat.TargetMode.HeatingCoolingMode()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net"
	"os"
	"strings"

	etcdErr "github.com/coreos/etcd/error"
	"github.com/coreos/go-etcd/etcd"
)

func isKeyNotFound(err error) bool {
	e, ok := err.(*etcd.EtcdError)
	return ok && e.ErrorCode == etcdErr.EcodeKeyNotFound
}

func getNodeIP() (net.IP, error) {

	if nodeIP == "" {
		addrs, err := net.InterfaceAddrs()
		if err != nil {
			return nil, fmt.Errorf("failed to get interface addresses: %s", err)
		}

		for _, a := range addrs {
			ip, _, err := net.ParseCIDR(a.String())
			if err != nil {
				// log error?
				continue
			}
			if ip.IsGlobalUnicast() {
				nodeIP = ip.String()
				break
			}
		}
	}

	if nodeIP == "" {
		return nil, fmt.Errorf("failed to get address")
	}

	ip := net.ParseIP(nodeIP)
	if ip == nil {
		return nil, fmt.Errorf("failed to parse address: %s", nodeIP)
	}

	// XXX: we currently only correctly handle v4
	if ip.To4() == nil {
		return nil, fmt.Errorf("not an ipv4 address: %s", nodeIP)
	}
	return ip, nil

}

func getNodeName() (string, error) {
	if nodeName == "" {
		var err error
		nodeName, err = os.Hostname()
		if err != nil {
			return "", fmt.Errorf("failed to get hostname: %s", err)
		}
	}

	return strings.ToLower(nodeName), nil

}
<commit_msg>need to check here for v4 as well<commit_after>package main

import (
	"fmt"
	"net"
	"os"
	"strings"

	etcdErr "github.com/coreos/etcd/error"
	"github.com/coreos/go-etcd/etcd"
)

func isKeyNotFound(err error) bool {
	e, ok := err.(*etcd.EtcdError)
	return ok && e.ErrorCode == etcdErr.EcodeKeyNotFound
}

func getNodeIP() (net.IP, error) {

	if nodeIP == "" {
		addrs, err := net.InterfaceAddrs()
		if err != nil {
			return nil, fmt.Errorf("failed to get interface addresses: %s", err)
		}

		for _, a := range addrs {
			ip, _, err := net.ParseCIDR(a.String())
			if err != nil {
				// log error?
				continue
			}
			if ip.To4() == nil {
				continue
			}
			if ip.IsGlobalUnicast() {
				nodeIP = ip.String()
				break
			}
		}
	}

	if nodeIP == "" {
		return nil, fmt.Errorf("failed to get address")
	}

	ip := net.ParseIP(nodeIP)
	if ip == nil {
		return nil, fmt.Errorf("failed to parse address: %s", nodeIP)
	}

	// XXX: we currently only correctly handle v4
	if ip.To4() == nil {
		return nil, fmt.Errorf("not an ipv4 address: %s", nodeIP)
	}
	return ip, nil

}

func getNodeName() (string, error) {
	if nodeName == "" {
		var err error
		nodeName, err = os.Hostname()
		if err != nil {
			return "", fmt.Errorf("failed to get hostname: %s", err)
		}
	}

	return strings.ToLower(nodeName), nil

}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
)

func makeCmd(args []string) (*exec.Cmd, error) {
	var cmd *exec.Cmd
	switch len(args) {
	case 0:
		return nil, errors.New("empty command")
	case 1:
		cmd = exec.Command(args[0])
	default:
		cmd = exec.Command(args[0], args[1:]...)
	}

	return cmd, nil
}

func runCommand(args []string) error {
	cmd, err := makeCmd(args)
	if err != nil {
		return err
	}

	checkVerboseEnabled(cmd)
	return cmd.Run()
}

func fileExists(path string) bool {
	_, err := os.Stat(path)
	if err != nil {
		return false
	}
	return true
}

func saveCurrentDir() string {
	prevDir, _ := filepath.Abs(".")
	return prevDir
}

func restoreCurrentDir(prevDir string) {
	os.Chdir(prevDir)
}

func clearWorkDir(workDir string) error {
	err := os.RemoveAll(workDir)
	if err != nil {
		// workaround for a restriction of os.RemoveAll()
		// os.RemoveAll() call fd.Readdirnames(100).
		// So os.RemoveAll() does not always remove all entries.
		// Some 3rd-party module (e.g. lua-nginx-module) tumbles this restriction.
		if fileExists(workDir) {
			err = os.RemoveAll(workDir)
		}
	}
	return err
}

func fileGetContents(path string) (string, error) {
	conf := ""
	if len(path) > 0 {
		confb, err := ioutil.ReadFile(path)
		if err != nil {
			return "", fmt.Errorf("confPath(%s) does not exist.", path)
		}
		conf = string(confb)
	}
	return conf, nil
}

func printConfigureOptions() error {
	cmd := exec.Command("objs/nginx", "-V")
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func printFirstMsg() {
	fmt.Printf(`nginx-build: %s
Compiler: %s %s
`,
		NGINX_BUILD_VERSION,
		runtime.Compiler,
		runtime.Version())
}

func printLastMsg(workDir, srcDir string, openResty, configureOnly bool) {
	log.Println("Complete building nginx!")

	if !openResty {
		if !configureOnly {
			fmt.Println()
			err := printConfigureOptions()
			if err != nil {
				fmt.Println(err.Error())
			}
		}
	}
	fmt.Println()

	lastMsgFormat := `Enter the following command for install nginx.

   $ cd %s/%s%s
   $ sudo make install
`
	if configureOnly {
		log.Printf(lastMsgFormat, workDir, srcDir, "\n   $ make")
	} else {
		log.Printf(lastMsgFormat, workDir, srcDir, "")
	}
}

func printFatalMsg(err error, path string) {
	if VerboseEnabled {
		log.Fatal(err)
	}

	f, err2 := os.Open(path)
	if err2 != nil {
		log.Printf("error-log: %s is not found\n", path)
		log.Fatal(err)
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		os.Stderr.Write(scanner.Bytes())
		os.Stderr.Write([]byte("\n"))
	}

	log.Fatal(err)
}
<commit_msg>removed unused function.<commit_after>package main

import (
	"bufio"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
)

func makeCmd(args []string) (*exec.Cmd, error) {
	var cmd *exec.Cmd
	switch len(args) {
	case 0:
		return nil, errors.New("empty command")
	case 1:
		cmd = exec.Command(args[0])
	default:
		cmd = exec.Command(args[0], args[1:]...)
	}

	return cmd, nil
}

func runCommand(args []string) error {
	cmd, err := makeCmd(args)
	if err != nil {
		return err
	}

	checkVerboseEnabled(cmd)
	return cmd.Run()
}

func fileExists(path string) bool {
	_, err := os.Stat(path)
	if err != nil {
		return false
	}
	return true
}

func saveCurrentDir() string {
	prevDir, _ := filepath.Abs(".")
	return prevDir
}

func clearWorkDir(workDir string) error {
	err := os.RemoveAll(workDir)
	if err != nil {
		// workaround for a restriction of os.RemoveAll()
		// os.RemoveAll() call fd.Readdirnames(100).
		// So os.RemoveAll() does not always remove all entries.
		// Some 3rd-party module (e.g. lua-nginx-module) tumbles this restriction.
		if fileExists(workDir) {
			err = os.RemoveAll(workDir)
		}
	}
	return err
}

func fileGetContents(path string) (string, error) {
	conf := ""
	if len(path) > 0 {
		confb, err := ioutil.ReadFile(path)
		if err != nil {
			return "", fmt.Errorf("confPath(%s) does not exist.", path)
		}
		conf = string(confb)
	}
	return conf, nil
}

func printConfigureOptions() error {
	cmd := exec.Command("objs/nginx", "-V")
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func printFirstMsg() {
	fmt.Printf(`nginx-build: %s
Compiler: %s %s
`,
		NGINX_BUILD_VERSION,
		runtime.Compiler,
		runtime.Version())
}

func printLastMsg(workDir, srcDir string, openResty, configureOnly bool) {
	log.Println("Complete building nginx!")

	if !openResty {
		if !configureOnly {
			fmt.Println()
			err := printConfigureOptions()
			if err != nil {
				fmt.Println(err.Error())
			}
		}
	}
	fmt.Println()

	lastMsgFormat := `Enter the following command for install nginx.

   $ cd %s/%s%s
   $ sudo make install
`
	if configureOnly {
		log.Printf(lastMsgFormat, workDir, srcDir, "\n   $ make")
	} else {
		log.Printf(lastMsgFormat, workDir, srcDir, "")
	}
}

func printFatalMsg(err error, path string) {
	if VerboseEnabled {
		log.Fatal(err)
	}

	f, err2 := os.Open(path)
	if err2 != nil {
		log.Printf("error-log: %s is not found\n", path)
		log.Fatal(err)
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		os.Stderr.Write(scanner.Bytes())
		os.Stderr.Write([]byte("\n"))
	}

	log.Fatal(err)
}
<|endoftext|>


<commit_before>package stats

import (
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"
	"time"

	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

var _env_monitor_shard bool
var _env_stop_chunk_number int64 = 40
var _env_shards []string
var _env_mongos string
var _shard_connections []Shard
var _shard_num int64 = 3        //FIXME need findout
var _shard_chunk_number []int64 // count how many chunk per shard
//var _shard_session *mgo.Session
var _shard_mongos_session *mgo.Session

var _shard_monitor_channel *time.Ticker

// struct for Shard
type Shard struct {
	Name    string `bson:"name"`
	Url     string `bson:"url"`
	Session *mgo.Session
}

type ShardChunk struct {
	Id    string `bson:"_id"`
	Count int64  `bson:"count"`
}

// helper
func getShardName(i int64) string {
	return fmt.Sprintf("shard%04d", i)
}

func getShardNumber(name string) int64 {
	i, err := strconv.ParseInt(string(name[5:]), 10, 64)
	if err != nil {
		log.Fatalln("Failed to parse shard name ", name)
	}
	return i
}

// goal here is to monitor shard cluster
func monitorShardCluster() {
	_shard_monitor_channel = time.NewTicker(time.Second * time.Duration(10)) // monitor every 10 seconds

	var _t_total_chunk int64
	var _t_chunk_size struct {
		Size int64 `bson:"value"`
	}

	_chunks := []ShardChunk{}

	for {
		_t_total_chunk = 0

		// count chunk for every shard
		// this is the aggregation pipeline:
		//     db.chunks.aggregate([{$project: {shard: 1, _id: 0}}, {$group: {_id: "$shard", count: {$sum: 1}}}])}}})
		//         { "_id" : "shard0002", "count" : 2 }
		//         { "_id" : "shard0001", "count" : 1 }
		//         { "_id" : "shard0000", "count" : 1 }
		_shard_mongos_session.DB("config").C("chunks").Pipe(
			[]bson.M{
				{"$project": bson.M{"shard": 1, "_id": 0}},
				{"$group": bson.M{"_id": "$shard", "count": bson.M{"$sum": 1}}},
			}).All(&_chunks)

		for i := 0; i < len(_chunks); i++ {
			_t_total_chunk += ShardChunk(_chunks[i]).Count
			_shard_chunk_number[getShardNumber(_chunks[i].Id)] = _chunks[i].Count
		}

		// to get chunk size
		// > db.settings.find()
		//      { "_id" : "chunksize", "value" : 64 })
		_shard_mongos_session.DB("config").C("settings").Find(bson.M{"_id": "chunksize"}).One(&_t_chunk_size)

		log.Printf("%v\t%d\t%d\n", _shard_chunk_number, _t_total_chunk, _t_chunk_size.Size)
		<-_shard_monitor_channel.C
	}
}

// to initilize connection to the shard and some initial variables
func initShardCluster() {
	var err error

	// create connection pool
	_shard_connections = make([]Shard, len(_env_shards), len(_env_shards))
	for i, url := range _env_shards {
		_shard_connections[i].Name = ""
		_shard_connections[i].Url = url
		_shard_connections[i].Session, err = mgo.Dial(url)

		if err != nil {
			log.Fatalln("Cannot open mongo connection to ", url)
		}
	}

	// create connection to mongos
	_shard_mongos_session, err = mgo.Dial(_env_mongos)

	if err != nil {
		log.Fatalln("Cannot open mongos connection to ", _env_mongos)
	}

	// make sure sharding and collection is setup properly

	// 1. drop db
	_shard_mongos_session.DB("htest1").DropDatabase()

	// 2. insert one doc into collection
	_shard_mongos_session.DB("htest1").C("htest1").Insert(bson.M{"a": 100})

	// 3. enable sharding
	err = _shard_mongos_session.DB("admin").Run(bson.D{{"enableSharding", "htest1"}}, nil)
	if err != nil {
		log.Fatalln("Failed to shard DB with error: ", err)
	}

	err = _shard_mongos_session.DB("htest1").C("htest1").EnsureIndex(mgo.Index{Key: []string{"$hashed:_id"}})
	if err != nil {
		log.Fatalln("Failed to create hashed index for collection with error: ", err)
	}

	err = _shard_mongos_session.DB("admin").Run(bson.D{{"shardCollection", "htest1.htest1"}, {"key", bson.M{"_id": "hashed"}}}, nil)
	if err != nil {
		log.Fatalln("Failed to shard collection with error: ", err)
	}

}

func init() {
	// hack for test
	//  os.Setenv("HT_MONITOR_SHARD", "true")
	// .Setenv("HT_MONGOS_URL", "54.68.41.49:27017")

	s := os.Getenv("HT_MONITOR_SHARD")
	if s != "" {
		_env_monitor_shard = true
	}

	if !_env_monitor_shard {
		// stop init if not monitoring
		return
	}

	s = os.Getenv("HT_MONGOS_URL")
	if s != "" {
		_env_mongos = s
	}

	s = os.Getenv("HT_SHARDS")
	if s != "" {
		_env_shards = strings.Fields(s)
		_env_monitor_shard = true

		if len(_env_shards) == 0 {
			log.Fatalln("HT_SHARDS, if set, must have at least one mongod URL")
		}
	}

	// a few special cases for test shard/auto-split
	_shard_chunk_number = make([]int64, 3, 3)

	initShardCluster()
	go monitorShardCluster()
}
<commit_msg>change shard display format<commit_after>package stats

import (
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"
	"time"

	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

var _env_monitor_shard bool
var _env_stop_chunk_number int64 = 40
var _env_shards []string
var _env_mongos string
var _shard_connections []Shard
var _shard_num int64 = 3        //FIXME need findout
var _shard_chunk_number []int64 // count how many chunk per shard
//var _shard_session *mgo.Session
var _shard_mongos_session *mgo.Session

var _shard_monitor_channel *time.Ticker

// struct for Shard
type Shard struct {
	Name    string `bson:"name"`
	Url     string `bson:"url"`
	Session *mgo.Session
}

type ShardChunk struct {
	Id    string `bson:"_id"`
	Count int64  `bson:"count"`
}

// helper
func getShardName(i int64) string {
	return fmt.Sprintf("shard%04d", i)
}

func getShardNumber(name string) int64 {
	i, err := strconv.ParseInt(string(name[5:]), 10, 64)
	if err != nil {
		log.Fatalln("Failed to parse shard name ", name)
	}
	return i
}

// goal here is to monitor shard cluster
func monitorShardCluster() {
	_shard_monitor_channel = time.NewTicker(time.Second * time.Duration(10)) // monitor every 10 seconds

	var _t_total_chunk int64
	var _t_chunk_size struct {
		Size int64 `bson:"value"`
	}

	_chunks := []ShardChunk{}

	for {
		_t_total_chunk = 0

		// count chunk for every shard
		// this is the aggregation pipeline:
		//     db.chunks.aggregate([{$project: {shard: 1, _id: 0}}, {$group: {_id: "$shard", count: {$sum: 1}}}])}}})
		//         { "_id" : "shard0002", "count" : 2 }
		//         { "_id" : "shard0001", "count" : 1 }
		//         { "_id" : "shard0000", "count" : 1 }
		_shard_mongos_session.DB("config").C("chunks").Pipe(
			[]bson.M{
				{"$project": bson.M{"shard": 1, "_id": 0}},
				{"$group": bson.M{"_id": "$shard", "count": bson.M{"$sum": 1}}},
			}).All(&_chunks)

		for i := 0; i < len(_chunks); i++ {
			_t_total_chunk += ShardChunk(_chunks[i]).Count
			_shard_chunk_number[getShardNumber(_chunks[i].Id)] = _chunks[i].Count
		}

		// to get chunk size
		// > db.settings.find()
		//      { "_id" : "chunksize", "value" : 64 })
		_shard_mongos_session.DB("config").C("settings").Find(bson.M{"_id": "chunksize"}).One(&_t_chunk_size)

		log.Printf("\t%d\t%d\t%d\t|\t%d\t%d\n", _shard_chunk_number[0], _shard_chunk_number[1], _shard_chunk_number[2], _t_total_chunk, _t_chunk_size.Size)
		<-_shard_monitor_channel.C
	}
}

// to initilize connection to the shard and some initial variables
func initShardCluster() {
	var err error

	// create connection pool
	_shard_connections = make([]Shard, len(_env_shards), len(_env_shards))
	for i, url := range _env_shards {
		_shard_connections[i].Name = ""
		_shard_connections[i].Url = url
		_shard_connections[i].Session, err = mgo.Dial(url)

		if err != nil {
			log.Fatalln("Cannot open mongo connection to ", url)
		}
	}

	// create connection to mongos
	_shard_mongos_session, err = mgo.Dial(_env_mongos)

	if err != nil {
		log.Fatalln("Cannot open mongos connection to ", _env_mongos)
	}

	// make sure sharding and collection is setup properly

	// 1. drop db
	_shard_mongos_session.DB("htest1").DropDatabase()

	// 2. insert one doc into collection
	_shard_mongos_session.DB("htest1").C("htest1").Insert(bson.M{"a": 100})

	// 3. enable sharding
	err = _shard_mongos_session.DB("admin").Run(bson.D{{"enableSharding", "htest1"}}, nil)
	if err != nil {
		log.Fatalln("Failed to shard DB with error: ", err)
	}

	err = _shard_mongos_session.DB("htest1").C("htest1").EnsureIndex(mgo.Index{Key: []string{"$hashed:_id"}})
	if err != nil {
		log.Fatalln("Failed to create hashed index for collection with error: ", err)
	}

	err = _shard_mongos_session.DB("admin").Run(bson.D{{"shardCollection", "htest1.htest1"}, {"key", bson.M{"_id": "hashed"}}}, nil)
	if err != nil {
		log.Fatalln("Failed to shard collection with error: ", err)
	}

}

func init() {
	// hack for test
	//  os.Setenv("HT_MONITOR_SHARD", "true")
	// .Setenv("HT_MONGOS_URL", "54.68.41.49:27017")

	s := os.Getenv("HT_MONITOR_SHARD")
	if s != "" {
		_env_monitor_shard = true
	}

	if !_env_monitor_shard {
		// stop init if not monitoring
		return
	}

	s = os.Getenv("HT_MONGOS_URL")
	if s != "" {
		_env_mongos = s
	}

	s = os.Getenv("HT_SHARDS")
	if s != "" {
		_env_shards = strings.Fields(s)
		_env_monitor_shard = true

		if len(_env_shards) == 0 {
			log.Fatalln("HT_SHARDS, if set, must have at least one mongod URL")
		}
	}

	// a few special cases for test shard/auto-split
	_shard_chunk_number = make([]int64, 3, 3)

	initShardCluster()
	go monitorShardCluster()
}
<|endoftext|>


<commit_before>// Copyright 2016, Cossack Labs Limited
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package postgresql

import (
	"bytes"
	"encoding/binary"
	. "github.com/cossacklabs/acra/utils"
	"io"
	"log"
	"strconv"

	"fmt"
	"github.com/cossacklabs/acra/decryptor/base"
	"github.com/cossacklabs/acra/keystore"
	"github.com/cossacklabs/acra/zone"
	"github.com/cossacklabs/themis/gothemis/cell"
	"github.com/cossacklabs/themis/gothemis/keys"
	"github.com/cossacklabs/themis/gothemis/message"
)

// TAG_BEGIN in hex format
// 133              32   251
//[92, 50, 48, 53], 32, [92, 51, 55, 51]
var ESCAPE_TAG_BEGIN = []byte{92, 50, 48, 53, 32, 92, 51, 55, 51}

func encodeToOctal(from, to []byte) int {
	output_length := 0
	for _, c := range from {
		if IsPrintableEscapeChar(c) {
			if c == SLASH_CHAR {
				to = append(to[:output_length], []byte{SLASH_CHAR, SLASH_CHAR}...)
				output_length += 2
			} else {
				to = append(to[:output_length], c)
				output_length++
			}
		} else {
			to = append(to[:output_length], SLASH_CHAR)
			output_length++
			octal := strconv.FormatInt(int64(c), 8)
			switch len(octal) {
			case 3:
				to = append(to[:output_length], []byte(octal)...)
			case 2:
				to = append(to[:output_length], '0', octal[0], octal[1])

			case 1:
				to = append(to[:output_length], '0', '0', octal[0])
			}
			output_length += 3
		}
	}
	return output_length
}

func EncodeToOctal(from []byte)[]byte {
	output_length := 0
	for _, c := range from {
		if IsPrintableEscapeChar(c) {
			if c == SLASH_CHAR {
				output_length += 2
			} else {
				output_length++
			}
		} else {
			output_length += 4
		}
	}
	buffer := make([]byte, output_length)
	encodeToOctal(from, buffer)
	return buffer
}

type PgEscapeDecryptor struct {
	current_index    uint8
	output_size      int
	is_with_zone     bool
	poison_key       []byte
	callback_storage *base.PoisonCallbackStorage
	// max size can be 4 characters for octal representation per byte
	oct_key_block_buffer     [base.KEY_BLOCK_LENGTH * 4]byte
	decoded_key_block_buffer []byte
	//uint64
	length_buf [8]byte
	// 4 oct symbols (\000) ber byte
	oct_length_buf [8 * 4]byte
	oct_char_buf   [3]byte
	key_store      keystore.KeyStore
	zone_matcher   *zone.ZoneIdMatcher
}

func NewPgEscapeDecryptor() *PgEscapeDecryptor {
	return &PgEscapeDecryptor{
		current_index:            0,
		is_with_zone:             false,
		output_size:              0,
		decoded_key_block_buffer: make([]byte, base.KEY_BLOCK_LENGTH),
	}
}

func (decryptor *PgEscapeDecryptor) SetWithZone(b bool) {
	decryptor.is_with_zone = b
}

func (decryptor *PgEscapeDecryptor) SetPoisonKey(key []byte) {
	decryptor.poison_key = key
}

func (decryptor *PgEscapeDecryptor) GetPoisonKey() []byte {
	return decryptor.poison_key
}

func (decryptor *PgEscapeDecryptor) MatchBeginTag(char byte) bool {
	if char == ESCAPE_TAG_BEGIN[decryptor.current_index] {
		decryptor.current_index++
		decryptor.output_size++
		return true
	} else {
		return false
	}

}
func (decryptor *PgEscapeDecryptor) IsMatched() bool {
	return int(decryptor.current_index) == len(ESCAPE_TAG_BEGIN)
}
func (decryptor *PgEscapeDecryptor) Reset() {
	decryptor.current_index = 0
	decryptor.output_size = 0
}
func (decryptor *PgEscapeDecryptor) GetMatched() []byte {
	return ESCAPE_TAG_BEGIN[:decryptor.current_index]
}

func (decryptor *PgEscapeDecryptor) readOctalData(data, oct_data []byte, reader io.Reader) (int, int, error) {
	data_index := 0
	oct_data_index := 0
	var char_buf [1]byte
	for {
		n, err := reader.Read(char_buf[:])
		if err != nil {
			return data_index, oct_data_index, err
		}
		if n != 1 {
			log.Println("Debug: readOctalData read 0 bytes")
			return data_index, oct_data_index, base.FAKE_ACRA_STRUCT
		}
		oct_data[oct_data_index] = char_buf[0]
		oct_data_index++
		if !IsPrintableEscapeChar(char_buf[0]) {
			return data_index, oct_data_index, base.FAKE_ACRA_STRUCT
		}

		// if slash than next char must be slash too
		if char_buf[0] == SLASH_CHAR {
			// read next char
			_, err := reader.Read(char_buf[:])
			if err != nil {
				return data_index, oct_data_index, err
			}
			oct_data[oct_data_index] = char_buf[0]
			oct_data_index++
			if char_buf[0] == SLASH_CHAR {
				// just write slash char
				data[data_index] = char_buf[0]
				data_index++
			} else {
				decryptor.oct_char_buf[0] = char_buf[0]
				// read next 3 oct bytes
				n, err := io.ReadFull(reader, decryptor.oct_char_buf[1:])
				if err != nil {
					return data_index, oct_data_index, err
				}
				if n != len(decryptor.oct_char_buf)-1 {
					if n != 0 {
						copy(oct_data[oct_data_index:oct_data_index+n], decryptor.oct_char_buf[1:1+n])
						oct_data_index += n
					}
					log.Printf("Warning: expected 2 octal symbols, but read %v\n", n)
					return data_index, oct_data_index, base.FAKE_ACRA_STRUCT
				}
				// parse 3 octal symbols
				num, err := strconv.ParseInt(string(decryptor.oct_char_buf[:]), 8, 9)
				if err != nil {
					return data_index, oct_data_index, base.FAKE_ACRA_STRUCT
				}
				data[data_index] = byte(num)
				data_index++

				copy(oct_data[oct_data_index:oct_data_index+len(decryptor.oct_char_buf)-1], decryptor.oct_char_buf[1:])
				oct_data_index += 2
			}
		} else {
			// just write to data
			data[data_index] = char_buf[0]
			data_index++
		}
		if data_index == cap(data) {
			return data_index, oct_data_index, nil
		}
	}
}
func (decryptor *PgEscapeDecryptor) ReadSymmetricKey(private_key *keys.PrivateKey, reader io.Reader) ([]byte, []byte, error) {
	data_length, oct_data_length, err := decryptor.readOctalData(decryptor.decoded_key_block_buffer, decryptor.oct_key_block_buffer[:], reader)
	if err != nil {
		return nil, decryptor.oct_key_block_buffer[:oct_data_length], err
	}
	if len(decryptor.decoded_key_block_buffer) != base.KEY_BLOCK_LENGTH || data_length != base.KEY_BLOCK_LENGTH {
		return nil, decryptor.oct_key_block_buffer[:oct_data_length], base.FAKE_ACRA_STRUCT
	}
	smessage := message.New(private_key, &keys.PublicKey{Value: decryptor.decoded_key_block_buffer[:base.PUBLIC_KEY_LENGTH]})
	symmetric_key, err := smessage.Unwrap(decryptor.decoded_key_block_buffer[base.PUBLIC_KEY_LENGTH:])
	if err != nil {
		log.Printf("Warning: %v\n", ErrorMessage("can't unwrap symmetric key", err))
		return nil, decryptor.oct_key_block_buffer[:oct_data_length], base.FAKE_ACRA_STRUCT
	}
	decryptor.output_size += oct_data_length
	return symmetric_key, decryptor.oct_key_block_buffer[:oct_data_length], nil
}

func (decryptor *PgEscapeDecryptor) readDataLength(reader io.Reader) (uint64, []byte, error) {
	var length uint64

	len_count, oct_len_count, err := decryptor.readOctalData(decryptor.length_buf[:], decryptor.oct_length_buf[:], reader)
	if err != nil {
		log.Printf("Warning: %v\n", ErrorMessage("can't read data length", err))
		return 0, decryptor.oct_length_buf[:oct_len_count], err
	}
	if len_count != len(decryptor.length_buf) {
		log.Printf("Warning: incorrect length count, %v!=%v\n", len_count, len(decryptor.length_buf))
		return 0, decryptor.oct_length_buf[:oct_len_count], base.FAKE_ACRA_STRUCT
	}
	decryptor.output_size += oct_len_count
	binary.Read(bytes.NewBuffer(decryptor.length_buf[:]), binary.LittleEndian, &length)
	return length, decryptor.oct_length_buf[:oct_len_count], nil
}
func (decryptor *PgEscapeDecryptor) readScellData(length uint64, reader io.Reader) ([]byte, []byte, error) {
	hex_buf := make([]byte, int(length)*4)
	buf := make([]byte, int(length))
	n, oct_n, err := decryptor.readOctalData(buf, hex_buf, reader)
	if err != nil {
		log.Printf("Warning: %v\n", ErrorMessage(fmt.Sprintf("can't read scell data with passed length=%v", length), err))
		return nil, hex_buf[:oct_n], err
	}
	if n != int(length) {
		log.Printf("Warning: read incorrect length, %v!=%v\n", n, length)
		return nil, hex_buf[:oct_n], base.FAKE_ACRA_STRUCT
	}
	decryptor.output_size += oct_n
	return buf, hex_buf[:oct_n], nil
}

func (decryptor *PgEscapeDecryptor) getFullDataLength() int {
	return decryptor.output_size
}

func (decryptor *PgEscapeDecryptor) ReadData(symmetric_key, zone_id []byte, reader io.Reader) ([]byte, error) {
	length, hex_length_buf, err := decryptor.readDataLength(reader)
	if err != nil {
		return hex_length_buf, err
	}
	data, oct_data, err := decryptor.readScellData(length, reader)
	if err != nil {
		return append(hex_length_buf, oct_data...), err
	}

	scell := cell.New(symmetric_key, cell.CELL_MODE_SEAL)
	decrypted, err := scell.Unprotect(data, nil, zone_id)
	// fill zero symmetric_key
	FillSlice(byte(0), symmetric_key[:])
	if err != nil {
		return append(hex_length_buf, oct_data...), base.FAKE_ACRA_STRUCT
	}
	return EncodeToOctal(decrypted), nil
}

func (decryptor *PgEscapeDecryptor) SetKeyStore(store keystore.KeyStore) {
	decryptor.key_store = store
}

func (decryptor *PgEscapeDecryptor) GetPrivateKey() (*keys.PrivateKey, error) {
	return decryptor.key_store.GetZonePrivateKey(decryptor.GetMatchedZoneId())
}

func (decryptor *PgEscapeDecryptor) SetZoneMatcher(zone_matcher *zone.ZoneIdMatcher) {
	decryptor.zone_matcher = zone_matcher
}

func (decryptor *PgEscapeDecryptor) MatchZone(c byte) bool {
	return decryptor.zone_matcher.Match(c)
}

func (decryptor *PgEscapeDecryptor) IsWithZone() bool {
	return decryptor.is_with_zone
}

func (decryptor *PgEscapeDecryptor) IsMatchedZone() bool {
	return decryptor.zone_matcher.IsMatched()
}

func (decryptor *PgEscapeDecryptor) ResetZoneMatch() {
	decryptor.zone_matcher.Reset()
}

func (decryptor *PgEscapeDecryptor) GetMatchedZoneId() []byte {
	if decryptor.IsWithZone() {
		return decryptor.zone_matcher.GetZoneId()
	} else {
		return nil
	}
}

func (decryptor *PgEscapeDecryptor) SetPoisonCallbackStorage(storage *base.PoisonCallbackStorage) {
	decryptor.callback_storage = storage
}

func (decryptor *PgEscapeDecryptor) GetPoisonCallbackStorage() *base.PoisonCallbackStorage {
	return decryptor.callback_storage
}
<commit_msg>optimize encodeToOctal<commit_after>// Copyright 2016, Cossack Labs Limited
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package postgresql

import (
	"bytes"
	"encoding/binary"
	. "github.com/cossacklabs/acra/utils"
	"io"
	"log"
	"strconv"

	"fmt"
	"github.com/cossacklabs/acra/decryptor/base"
	"github.com/cossacklabs/acra/keystore"
	"github.com/cossacklabs/acra/zone"
	"github.com/cossacklabs/themis/gothemis/cell"
	"github.com/cossacklabs/themis/gothemis/keys"
	"github.com/cossacklabs/themis/gothemis/message"
)

// TAG_BEGIN in hex format
// 133              32   251
//[92, 50, 48, 53], 32, [92, 51, 55, 51]
var ESCAPE_TAG_BEGIN = []byte{92, 50, 48, 53, 32, 92, 51, 55, 51}

func encodeToOctal(from, to []byte) {
	to = to[:0]
	for _, c := range from {
		if IsPrintableEscapeChar(c) {
			if c == SLASH_CHAR {
				to = append(to, []byte{SLASH_CHAR, SLASH_CHAR}...)
			} else {
				to = append(to, c)
			}
		} else {
			to = append(to, SLASH_CHAR)
			octal := strconv.FormatInt(int64(c), 8)
			switch len(octal) {
			case 3:
				to = append(to, []byte(octal)...)
			case 2:
				to = append(to, '0', octal[0], octal[1])

			case 1:
				to = append(to, '0', '0', octal[0])
			}
		}
	}
}

func EncodeToOctal(from []byte)[]byte {
	// count output size
	output_length := 0
	for _, c := range from {
		if IsPrintableEscapeChar(c) {
			if c == SLASH_CHAR {
				output_length += 2
			} else {
				output_length++
			}
		} else {
			output_length += 4
		}
	}
	buffer := make([]byte, output_length)
	encodeToOctal(from, buffer)
	return buffer
}

type PgEscapeDecryptor struct {
	current_index    uint8
	output_size      int
	is_with_zone     bool
	poison_key       []byte
	callback_storage *base.PoisonCallbackStorage
	// max size can be 4 characters for octal representation per byte
	oct_key_block_buffer     [base.KEY_BLOCK_LENGTH * 4]byte
	decoded_key_block_buffer []byte
	//uint64
	length_buf [8]byte
	// 4 oct symbols (\000) ber byte
	oct_length_buf [8 * 4]byte
	oct_char_buf   [3]byte
	key_store      keystore.KeyStore
	zone_matcher   *zone.ZoneIdMatcher
}

func NewPgEscapeDecryptor() *PgEscapeDecryptor {
	return &PgEscapeDecryptor{
		current_index:            0,
		is_with_zone:             false,
		output_size:              0,
		decoded_key_block_buffer: make([]byte, base.KEY_BLOCK_LENGTH),
	}
}

func (decryptor *PgEscapeDecryptor) SetWithZone(b bool) {
	decryptor.is_with_zone = b
}

func (decryptor *PgEscapeDecryptor) SetPoisonKey(key []byte) {
	decryptor.poison_key = key
}

func (decryptor *PgEscapeDecryptor) GetPoisonKey() []byte {
	return decryptor.poison_key
}

func (decryptor *PgEscapeDecryptor) MatchBeginTag(char byte) bool {
	if char == ESCAPE_TAG_BEGIN[decryptor.current_index] {
		decryptor.current_index++
		decryptor.output_size++
		return true
	} else {
		return false
	}

}
func (decryptor *PgEscapeDecryptor) IsMatched() bool {
	return int(decryptor.current_index) == len(ESCAPE_TAG_BEGIN)
}
func (decryptor *PgEscapeDecryptor) Reset() {
	decryptor.current_index = 0
	decryptor.output_size = 0
}
func (decryptor *PgEscapeDecryptor) GetMatched() []byte {
	return ESCAPE_TAG_BEGIN[:decryptor.current_index]
}

func (decryptor *PgEscapeDecryptor) readOctalData(data, oct_data []byte, reader io.Reader) (int, int, error) {
	data_index := 0
	oct_data_index := 0
	var char_buf [1]byte
	for {
		n, err := reader.Read(char_buf[:])
		if err != nil {
			return data_index, oct_data_index, err
		}
		if n != 1 {
			log.Println("Debug: readOctalData read 0 bytes")
			return data_index, oct_data_index, base.FAKE_ACRA_STRUCT
		}
		oct_data[oct_data_index] = char_buf[0]
		oct_data_index++
		if !IsPrintableEscapeChar(char_buf[0]) {
			return data_index, oct_data_index, base.FAKE_ACRA_STRUCT
		}

		// if slash than next char must be slash too
		if char_buf[0] == SLASH_CHAR {
			// read next char
			_, err := reader.Read(char_buf[:])
			if err != nil {
				return data_index, oct_data_index, err
			}
			oct_data[oct_data_index] = char_buf[0]
			oct_data_index++
			if char_buf[0] == SLASH_CHAR {
				// just write slash char
				data[data_index] = char_buf[0]
				data_index++
			} else {
				decryptor.oct_char_buf[0] = char_buf[0]
				// read next 3 oct bytes
				n, err := io.ReadFull(reader, decryptor.oct_char_buf[1:])
				if err != nil {
					return data_index, oct_data_index, err
				}
				if n != len(decryptor.oct_char_buf)-1 {
					if n != 0 {
						copy(oct_data[oct_data_index:oct_data_index+n], decryptor.oct_char_buf[1:1+n])
						oct_data_index += n
					}
					log.Printf("Warning: expected 2 octal symbols, but read %v\n", n)
					return data_index, oct_data_index, base.FAKE_ACRA_STRUCT
				}
				// parse 3 octal symbols
				num, err := strconv.ParseInt(string(decryptor.oct_char_buf[:]), 8, 9)
				if err != nil {
					return data_index, oct_data_index, base.FAKE_ACRA_STRUCT
				}
				data[data_index] = byte(num)
				data_index++

				copy(oct_data[oct_data_index:oct_data_index+len(decryptor.oct_char_buf)-1], decryptor.oct_char_buf[1:])
				oct_data_index += 2
			}
		} else {
			// just write to data
			data[data_index] = char_buf[0]
			data_index++
		}
		if data_index == cap(data) {
			return data_index, oct_data_index, nil
		}
	}
}
func (decryptor *PgEscapeDecryptor) ReadSymmetricKey(private_key *keys.PrivateKey, reader io.Reader) ([]byte, []byte, error) {
	data_length, oct_data_length, err := decryptor.readOctalData(decryptor.decoded_key_block_buffer, decryptor.oct_key_block_buffer[:], reader)
	if err != nil {
		return nil, decryptor.oct_key_block_buffer[:oct_data_length], err
	}
	if len(decryptor.decoded_key_block_buffer) != base.KEY_BLOCK_LENGTH || data_length != base.KEY_BLOCK_LENGTH {
		return nil, decryptor.oct_key_block_buffer[:oct_data_length], base.FAKE_ACRA_STRUCT
	}
	smessage := message.New(private_key, &keys.PublicKey{Value: decryptor.decoded_key_block_buffer[:base.PUBLIC_KEY_LENGTH]})
	symmetric_key, err := smessage.Unwrap(decryptor.decoded_key_block_buffer[base.PUBLIC_KEY_LENGTH:])
	if err != nil {
		log.Printf("Warning: %v\n", ErrorMessage("can't unwrap symmetric key", err))
		return nil, decryptor.oct_key_block_buffer[:oct_data_length], base.FAKE_ACRA_STRUCT
	}
	decryptor.output_size += oct_data_length
	return symmetric_key, decryptor.oct_key_block_buffer[:oct_data_length], nil
}

func (decryptor *PgEscapeDecryptor) readDataLength(reader io.Reader) (uint64, []byte, error) {
	var length uint64

	len_count, oct_len_count, err := decryptor.readOctalData(decryptor.length_buf[:], decryptor.oct_length_buf[:], reader)
	if err != nil {
		log.Printf("Warning: %v\n", ErrorMessage("can't read data length", err))
		return 0, decryptor.oct_length_buf[:oct_len_count], err
	}
	if len_count != len(decryptor.length_buf) {
		log.Printf("Warning: incorrect length count, %v!=%v\n", len_count, len(decryptor.length_buf))
		return 0, decryptor.oct_length_buf[:oct_len_count], base.FAKE_ACRA_STRUCT
	}
	decryptor.output_size += oct_len_count
	binary.Read(bytes.NewBuffer(decryptor.length_buf[:]), binary.LittleEndian, &length)
	return length, decryptor.oct_length_buf[:oct_len_count], nil
}
func (decryptor *PgEscapeDecryptor) readScellData(length uint64, reader io.Reader) ([]byte, []byte, error) {
	hex_buf := make([]byte, int(length)*4)
	buf := make([]byte, int(length))
	n, oct_n, err := decryptor.readOctalData(buf, hex_buf, reader)
	if err != nil {
		log.Printf("Warning: %v\n", ErrorMessage(fmt.Sprintf("can't read scell data with passed length=%v", length), err))
		return nil, hex_buf[:oct_n], err
	}
	if n != int(length) {
		log.Printf("Warning: read incorrect length, %v!=%v\n", n, length)
		return nil, hex_buf[:oct_n], base.FAKE_ACRA_STRUCT
	}
	decryptor.output_size += oct_n
	return buf, hex_buf[:oct_n], nil
}

func (decryptor *PgEscapeDecryptor) getFullDataLength() int {
	return decryptor.output_size
}

func (decryptor *PgEscapeDecryptor) ReadData(symmetric_key, zone_id []byte, reader io.Reader) ([]byte, error) {
	length, hex_length_buf, err := decryptor.readDataLength(reader)
	if err != nil {
		return hex_length_buf, err
	}
	data, oct_data, err := decryptor.readScellData(length, reader)
	if err != nil {
		return append(hex_length_buf, oct_data...), err
	}

	scell := cell.New(symmetric_key, cell.CELL_MODE_SEAL)
	decrypted, err := scell.Unprotect(data, nil, zone_id)
	// fill zero symmetric_key
	FillSlice(byte(0), symmetric_key[:])
	if err != nil {
		return append(hex_length_buf, oct_data...), base.FAKE_ACRA_STRUCT
	}
	return EncodeToOctal(decrypted), nil
}

func (decryptor *PgEscapeDecryptor) SetKeyStore(store keystore.KeyStore) {
	decryptor.key_store = store
}

func (decryptor *PgEscapeDecryptor) GetPrivateKey() (*keys.PrivateKey, error) {
	return decryptor.key_store.GetZonePrivateKey(decryptor.GetMatchedZoneId())
}

func (decryptor *PgEscapeDecryptor) SetZoneMatcher(zone_matcher *zone.ZoneIdMatcher) {
	decryptor.zone_matcher = zone_matcher
}

func (decryptor *PgEscapeDecryptor) MatchZone(c byte) bool {
	return decryptor.zone_matcher.Match(c)
}

func (decryptor *PgEscapeDecryptor) IsWithZone() bool {
	return decryptor.is_with_zone
}

func (decryptor *PgEscapeDecryptor) IsMatchedZone() bool {
	return decryptor.zone_matcher.IsMatched()
}

func (decryptor *PgEscapeDecryptor) ResetZoneMatch() {
	decryptor.zone_matcher.Reset()
}

func (decryptor *PgEscapeDecryptor) GetMatchedZoneId() []byte {
	if decryptor.IsWithZone() {
		return decryptor.zone_matcher.GetZoneId()
	} else {
		return nil
	}
}

func (decryptor *PgEscapeDecryptor) SetPoisonCallbackStorage(storage *base.PoisonCallbackStorage) {
	decryptor.callback_storage = storage
}

func (decryptor *PgEscapeDecryptor) GetPoisonCallbackStorage() *base.PoisonCallbackStorage {
	return decryptor.callback_storage
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/sha1"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strings"
)

var extensions = []string{".jpg", ".jpeg", ".JPG", ".JPEG"}
var metaDir = ".album"

var root = flag.String("root", "", "Album root")
var metaRoot string

type HashingTask struct {
	MetaDataPath string
	FilePath     string
}

func walker(path string, info os.FileInfo, err error) error {

	if info.IsDir() {

		if path == "." {
			return nil
		}

		if path == ".." {
			return nil
		}

		if strings.HasPrefix(info.Name(), ".") {
			return filepath.SkipDir
		}

		return nil
	}

	for _, ext := range extensions {
		if ext == filepath.Ext(info.Name()) {
			normalizedPath := strings.TrimPrefix(path, *root)
			normalizedPath = strings.TrimPrefix(normalizedPath, "/")

			metaDataPath := filepath.Join(metaRoot, "hash", normalizedPath) + ".sha1"

			hashFile, hashFileErr := os.Open(metaDataPath)

			if hashFileErr == nil {
				hashFile.Close()
				return nil
			}

			hasher <- HashingTask{metaDataPath, path}
			// hasher

			break
		}
	}

	return nil
}

var Usage = func() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	flag.PrintDefaults()
}

var hasher = make(chan HashingTask, runtime.NumCPU())

func main() {
	flag.Parse()

	if *root == "" {
		Usage()
		return
	}

	log.Printf("Setting up for %d CPUs", runtime.NumCPU())

	*root = filepath.Clean(*root)
	metaRoot = filepath.Join(*root, metaDir)

	if _, rootErr := ioutil.ReadDir(*root); rootErr != nil {
		log.Fatal("Root directory could not be read")
	}

	log.Println("Meta dir: " + metaDir)
	log.Println("Root: " + *root)

	go func() {

		for task := range hasher {

			go func(task HashingTask) {
				log.Println("Task starting for", task.FilePath)
				file, fileErr := os.Open(task.FilePath)

				if fileErr != nil {
					return
				}

				sha := sha1.New()
				io.Copy(sha, file)

				file.Close()

				// sum := sha.Sum(nil)

				/*os.MkdirAll(filepath.Dir(task.MetaDataPath), 0755)
				hashFile, hashFileErr := os.Create(task.MetaDataPath)

				if hashFileErr == nil {
					hashFile.Write(sum)
					hashFile.Close()
				} else {
					log.Println("Could not write hash file: ", hashFileErr)
				}*/

				log.Println("Task done for", task.FilePath)
			}(task)
		}

	}()

	walkErr := filepath.Walk(*root, walker)

	if walkErr != nil {
		log.Fatal(walkErr.Error())
	}
}
<commit_msg>Crawling concurrency probably done better<commit_after>package main

import (
	"crypto/sha1"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strings"
)

var extensions = []string{".jpg", ".jpeg", ".JPG", ".JPEG"}
var metaDir = ".album"

var root = flag.String("root", "", "Album root")
var testMode = flag.Bool("test", false, "Test mode")

var metaRoot string

type HashingTask struct {
	path     string
	info     os.FileInfo
	root     string
	metaRoot string
}

type PhotoWalker struct {
	walkFunc func(path string, info os.FileInfo)
}

func (w PhotoWalker) photoWalker() filepath.WalkFunc {
	return func(path string, info os.FileInfo, err error) error {

		if info.IsDir() {

			if path == "." {
				return nil
			}

			if path == ".." {
				return nil
			}

			if strings.HasPrefix(info.Name(), ".") {
				return filepath.SkipDir
			}

			return nil
		}

		for _, ext := range extensions {
			if ext == filepath.Ext(info.Name()) {
				w.walkFunc(path, info)
				return nil
			}
		}

		return nil
	}
}

func HashPath(path string, root string, metaRoot string) string {
	normalizedPath := strings.TrimPrefix(path, root)
	normalizedPath = strings.TrimPrefix(normalizedPath, "/")

	hashPath := filepath.Join(metaRoot, "hash", normalizedPath) + ".sha1"
	return hashPath
}

var Usage = func() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	flag.PrintDefaults()
}

// var hasher = make(chan HashingTask, runtime.NumCPU())

func main() {
	flag.Parse()

	if *root == "" {
		Usage()
		return
	}

	*root = filepath.Clean(*root)
	metaRoot = filepath.Join(*root, metaDir)

	if _, rootErr := ioutil.ReadDir(*root); rootErr != nil {
		log.Fatal("Root directory could not be read")
	}

	log.Println("Meta dir: " + metaDir)
	log.Println("Root: " + *root)

	var tasksCount int
	var fileCount int

	workerChan := make(chan HashingTask, runtime.NumCPU())
	counterChan := make(chan int)

	finishChan := make(chan int)

	go func() {
		for {
			val, ok := <-counterChan
			if !ok {
				return
			}

			if val > 0 {
				tasksCount++
			}

			if tasksCount == fileCount {
				finishChan <- 1
			}

		}
	}()

	for i := 0; i < runtime.NumCPU(); i++ {
		go func() {
			for {
				task := <-workerChan

				file, fileErr := os.Open(task.path)

				if fileErr != nil {
					log.Println("Could not read photo file at", task.path)
					continue
				}

				sha := sha1.New()
				io.Copy(sha, file)

				file.Close()

				sum := sha.Sum(nil)

				hashPath := HashPath(task.path, task.root, task.metaRoot)

				hashFile, hashFileErr := os.Open(hashPath)

				if hashFileErr == nil {
					currentSum := make([]byte, 20)
					hashFile.Read(currentSum)

					if string(currentSum) == string(sum) {
					}

					hashFile.Close()
					counterChan <- 1
					log.Println("Skipping", task.path)
					continue
				}

				if !(*testMode) {

					os.MkdirAll(filepath.Dir(hashPath), 0755)
					hashFile, hashFileErr := os.Create(hashPath)

					if hashFileErr == nil {
						hashFile.Write(sum)
						hashFile.Close()
					} else {
						log.Println("Could not write hash file: ", hashFileErr)
					}
				}

				log.Println("Task done for", task.path)
				counterChan <- 1
			}
		}()
	}

	photoWalker := PhotoWalker{}
	photoWalker.walkFunc = func(path string, info os.FileInfo) {
		fileCount++
		workerChan <- HashingTask{path, info, *root, metaRoot}
	}

	walkErr := filepath.Walk(*root, photoWalker.photoWalker())

	if walkErr != nil {
		log.Fatal(walkErr.Error())
	}

	<-finishChan
	close(workerChan)
	log.Printf("%d files processed", fileCount)
}
<|endoftext|>


<commit_before>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Spencer Kimball (spencer.kimball@gmail.com)
// Author: Tobias Schottdorf (tobias.schottdorf@gmail.com)

package engine

import (
	"bytes"
	"strings"

	"code.google.com/p/biogo.store/interval"
	"github.com/cockroachdb/cockroach/proto"
	"github.com/cockroachdb/cockroach/util/encoding"
	"github.com/cockroachdb/cockroach/util/log"
)

// Key defines the key in the key-value datastore.
type Key []byte

// MakeKey makes a new key which is the concatenation of the
// given inputs, in order.
func MakeKey(keys ...Key) Key {
	byteSlices := make([][]byte, len(keys))
	for i, k := range keys {
		byteSlices[i] = []byte(k)
	}
	return Key(bytes.Join(byteSlices, nil))
}

// MakeLocalKey is a simple passthrough to MakeKey, with verification
// that the first key has length KeyLocalPrefixLength.
func MakeLocalKey(keys ...Key) Key {
	if len(keys) == 0 {
		log.Fatal("no key components specified in call to MakeLocalKey")
	}
	if len(keys[0]) != KeyLocalPrefixLength {
		log.Fatalf("local key prefix length must be %d: %q", KeyLocalPrefixLength, keys[0])
	}
	return MakeKey(keys...)
}

// Address returns the address for the key, used to lookup the range
// containing the key. In the normal case, this is simply the key's
// value. However, for local keys, such as transaction records,
// range-spanning binary tree node pointers, and message queues, the
// address is the trailing suffix of the key, with the local key
// prefix removed. In this way, local keys address to the same range
// as non-local keys, but are stored separately so that they don't
// collide with user-space or global system keys.
func (k Key) Address() Key {
	if !bytes.HasPrefix(k, KeyLocalPrefix) {
		return k
	}
	if len(k) < KeyLocalPrefixLength {
		log.Fatalf("local key %q malformed; should contain prefix %q and four-character designation", k, KeyLocalPrefix)
	}
	return k[KeyLocalPrefixLength:]
}

// DecodeKey returns a Key initialized by decoding a binary-encoded
// prefix from the passed in bytes slice. Any leftover bytes are
// returned.
func DecodeKey(b []byte) ([]byte, Key) {
	if len(b) == 0 {
		panic("cannot decode an empty key")
	}
	var keyBytes []byte
	b, keyBytes = encoding.DecodeBinary(b)
	return b, Key(keyBytes)
}

// Encode returns a binary-encoded version of the key appended to the
// supplied byte string, b.
func (k Key) Encode(b []byte) []byte {
	return encoding.EncodeBinary(b, []byte(k))
}

// Next returns the next key in lexicographic sort order.
func (k Key) Next() Key {
	return MakeKey(k, Key{0})
}

// PrefixEnd determines the end key given key as a prefix, that is the
// key that sorts precisely behind all keys starting with prefix: "1"
// is added to the final byte and the carry propagated. The special
// cases of nil and KeyMin always returns KeyMax.
func (k Key) PrefixEnd() Key {
	if len(k) == 0 {
		return KeyMax
	}
	end := append([]byte(nil), k...)
	for i := len(end) - 1; i >= 0; i-- {
		end[i] = end[i] + 1
		if end[i] != 0 {
			return end
		}
	}
	// This statement will only be reached if the key is already a
	// maximal byte string (i.e. already \xff...).
	return k
}

// Less implements the util.Ordered interface.
func (k Key) Less(l Key) bool {
	return bytes.Compare(k, l) < 0
}

// Equal returns whether two keys are identical.
func (k Key) Equal(l Key) bool {
	return bytes.Equal(k, l)
}

// Compare implements the llrb.Comparable interface for tree nodes.
func (k Key) Compare(b interval.Comparable) int {
	return bytes.Compare(k, b.(Key))
}

// Value specifies the value at a key. Multiple values at the same key
// are supported based on timestamp.
type Value struct {
	// Bytes is the byte string value.
	Bytes []byte
	// Checksum is a CRC-32-IEEE checksum. A Value will only be used in
	// a write operation by the database if either its checksum is zero
	// or the CRC checksum of Bytes matches it.
	// Values returned by the database will contain a checksum of the
	// contained value.
	Checksum uint32
	// Timestamp of value.
	Timestamp proto.Timestamp
}

// KeyValue is a pair of Key and Value for returned Key/Value pairs
// from ScanRequest/ScanResponse. It embeds a Key and a Value.
type KeyValue struct {
	Key
	Value
}

// RangeMetaKey returns a range metadata key for the given key. For ordinary
// keys this returns a level 2 metadata key - for level 2 keys, it returns a
// level 1 key. For level 1 keys and local keys, KeyMin is returned.
func RangeMetaKey(key Key) Key {
	if len(key) == 0 {
		return KeyMin
	}
	addr := key.Address()
	if !bytes.HasPrefix(addr, KeyMetaPrefix) {
		return MakeKey(KeyMeta2Prefix, addr)
	}
	if bytes.HasPrefix(addr, KeyMeta2Prefix) {
		return MakeKey(KeyMeta1Prefix, addr[len(KeyMeta2Prefix):])
	}

	return KeyMin
}

// RangeMetadataLookupKey returns the metadata key at which this range
// descriptor should be stored as a value.
func RangeMetadataLookupKey(r *proto.RangeDescriptor) Key {
	return RangeMetaKey(r.EndKey)
}

// ValidateRangeMetaKey validates that the given key is a valid Range Metadata
// key. It must have an appropriate metadata range prefix, and the original key
// value must be less thas KeyMax. As a special case, KeyMin is considered a
// valid Range Metadata Key.
func ValidateRangeMetaKey(key Key) error {
	// KeyMin is a valid key.
	if len(key) == 0 {
		return nil
	}
	// Key must be at least as long as KeyMeta1Prefix.
	if len(key) < len(KeyMeta1Prefix) {
		return NewInvalidRangeMetaKeyError("too short", key)
	}

	prefix, body := key[:len(KeyMeta1Prefix)], key[len(KeyMeta1Prefix):]

	// The prefix must be equal to KeyMeta1Prefix or KeyMeta2Prefix
	if !bytes.HasPrefix(key, KeyMetaPrefix) {
		return NewInvalidRangeMetaKeyError("does not have \\x00\\x00meta[12] prefix", key)
	}
	if lvl := string(prefix[len(KeyMetaPrefix)]); lvl != "1" && lvl != "2" {
		return NewInvalidRangeMetaKeyError("meta level is not 1 or 2", key)
	}
	// Body of the key must sort before KeyMax
	if !body.Less(KeyMax) {
		return NewInvalidRangeMetaKeyError("body of range lookup is >= KeyMax", key)
	}

	return nil
}

func init() {
	if KeyLocalPrefixLength%7 != 0 {
		log.Fatal("local key prefix is not a multiple of 7: %d", KeyLocalPrefixLength)
	}
}

// Constants for system-reserved keys in the KV map.
var (
	// KeyMaxLength is the maximum key length.
	KeyMaxLength = 2048

	// KeyMin is a minimum key value which sorts before all other keys.
	KeyMin = Key("")
	// KeyMax is a maximum key value which sorts after all other keys.
	KeyMax = Key(strings.Repeat("\xff", KeyMaxLength))

	// KeyLocalPrefix is the prefix for keys which hold data local to a
	// RocksDB instance, such as range accounting information
	// (e.g. range metadata, range-spanning binary tree node pointers),
	// response cache values, transaction records, and message
	// queues. Some local data are replicated, such as transaction rows,
	// but are located in the local area so that they remain in
	// proximity to one or more keys which they affect, but without
	// unnecessarily polluting the key space. Further, some local data
	// are stored with MVCC and contribute to distributed transactions,
	// such as range metadata, range-spanning binary tree node pointers,
	// and message queues.
	//
	// The local key prefix has been deliberately chosen to sort before
	// the KeySystemPrefix, because these local keys are not addressable
	// via the meta range addressing indexes.
	KeyLocalPrefix = Key("\x00\x00\x00")

	// KeyLocalPrefixLength is the maximum length of the local prefix.
	// It includes both the standard prefix and an additional four
	// characters to designate the type of local data.
	//
	// NOTE: this is very important! In order to support prefix matches
	//   (e.g. for garbage collection of transaction and response cache
	//   rows), the number of bytes in the key local prefix must be a
	//   multiple of 7. This provides an encoded binary string with no
	//   leftover bits to "bleed" into the next byte in the non-prefix
	//   part of the local key.
	KeyLocalPrefixLength = len(KeyLocalPrefix) + 4

	// KeyLocalIdent stores an immutable identifier for this store,
	// created when the store is first bootstrapped.
	KeyLocalIdent = MakeKey(KeyLocalPrefix, Key("iden"))
	// KeyLocalRangeMetadataPrefix is the prefix for keys storing range metadata.
	// The value is a struct of type RangeMetadata.
	KeyLocalRangeMetadataPrefix = MakeKey(KeyLocalPrefix, Key("rng-"))
	// KeyLocalResponseCachePrefix is the prefix for keys storing command
	// responses used to guarantee idempotency (see ResponseCache). This key
	// prefix is duplicated in rocksdb_compaction.cc and must be kept in sync
	// if modified here.
	KeyLocalResponseCachePrefix = MakeKey(KeyLocalPrefix, Key("res-"))
	// KeyLocalTransactionPrefix specifies the key prefix for
	// transaction records. The suffix is the transaction id. This key
	// prefix is duplicated in rocksdb_compaction.cc and must be kept in
	// sync if modified here.
	KeyLocalTransactionPrefix = MakeKey(KeyLocalPrefix, Key("txn-"))
	// KeyLocalSnapshotIDGenerator is a snapshot ID generator sequence.
	// Snapshot IDs must be unique per store ID.
	KeyLocalSnapshotIDGenerator = MakeKey(KeyLocalPrefix, Key("ssid"))

	// KeySystemPrefix indicates the beginning of the key range for
	// global, system data which are replicated across the cluster.
	KeySystemPrefix = Key("\x00")
	KeySystemMax    = Key("\x01")

	// KeyMetaPrefix is the prefix for range metadata keys. Notice that
	// an extra null character in the prefix causes all range addressing
	// records to sort before any system tables which they might describe.
	KeyMetaPrefix = MakeKey(KeySystemPrefix, Key("\x00meta"))
	// KeyMeta1Prefix is the first level of key addressing. The value is a
	// RangeDescriptor struct.
	KeyMeta1Prefix = MakeKey(KeyMetaPrefix, Key("1"))
	// KeyMeta2Prefix is the second level of key addressing. The value is a
	// RangeDescriptor struct.
	KeyMeta2Prefix = MakeKey(KeyMetaPrefix, Key("2"))

	// KeyMetaMax is the end of the range of addressing keys.
	KeyMetaMax = MakeKey(KeySystemPrefix, Key("\x01"))

	// KeyConfigAccountingPrefix specifies the key prefix for accounting
	// configurations. The suffix is the affected key prefix.
	KeyConfigAccountingPrefix = MakeKey(KeySystemPrefix, Key("acct"))
	// KeyConfigPermissionPrefix specifies the key prefix for accounting
	// configurations. The suffix is the affected key prefix.
	KeyConfigPermissionPrefix = MakeKey(KeySystemPrefix, Key("perm"))
	// KeyConfigZonePrefix specifies the key prefix for zone
	// configurations. The suffix is the affected key prefix.
	KeyConfigZonePrefix = MakeKey(KeySystemPrefix, Key("zone"))
	// KeyNodeIDGenerator is the global node ID generator sequence.
	KeyNodeIDGenerator = MakeKey(KeySystemPrefix, Key("node-idgen"))
	// KeyRaftIDGenerator is the global Raft consensus group ID generator sequence.
	KeyRaftIDGenerator = MakeKey(KeySystemPrefix, Key("raft-idgen"))
	// KeyRangeIDGenerator is the global range ID generator sequence.
	KeyRangeIDGenerator = MakeKey(KeySystemPrefix, Key("range-idgen"))
	// KeySchemaPrefix specifies key prefixes for schema definitions.
	KeySchemaPrefix = MakeKey(KeySystemPrefix, Key("schema"))
	// KeyStoreIDGeneratorPrefix specifies key prefixes for sequence
	// generators, one per node, for store IDs.
	KeyStoreIDGeneratorPrefix = MakeKey(KeySystemPrefix, Key("store-idgen-"))
)
<commit_msg>Addressed Tobias feedback, increased key size limit to 4096<commit_after>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Spencer Kimball (spencer.kimball@gmail.com)
// Author: Tobias Schottdorf (tobias.schottdorf@gmail.com)

package engine

import (
	"bytes"
	"fmt"
	"strings"

	"code.google.com/p/biogo.store/interval"
	"github.com/cockroachdb/cockroach/proto"
	"github.com/cockroachdb/cockroach/util/encoding"
	"github.com/cockroachdb/cockroach/util/log"
)

// Key defines the key in the key-value datastore.
type Key []byte

// MakeKey makes a new key which is the concatenation of the
// given inputs, in order.
func MakeKey(keys ...Key) Key {
	byteSlices := make([][]byte, len(keys))
	for i, k := range keys {
		byteSlices[i] = []byte(k)
	}
	return Key(bytes.Join(byteSlices, nil))
}

// MakeLocalKey is a simple passthrough to MakeKey, with verification
// that the first key has length KeyLocalPrefixLength.
func MakeLocalKey(keys ...Key) Key {
	if len(keys) == 0 {
		log.Fatal("no key components specified in call to MakeLocalKey")
	}
	if len(keys[0]) != KeyLocalPrefixLength {
		log.Fatalf("local key prefix length must be %d: %q", KeyLocalPrefixLength, keys[0])
	}
	return MakeKey(keys...)
}

// Address returns the address for the key, used to lookup the range
// containing the key. In the normal case, this is simply the key's
// value. However, for local keys, such as transaction records,
// range-spanning binary tree node pointers, and message queues, the
// address is the trailing suffix of the key, with the local key
// prefix removed. In this way, local keys address to the same range
// as non-local keys, but are stored separately so that they don't
// collide with user-space or global system keys.
func (k Key) Address() Key {
	if !bytes.HasPrefix(k, KeyLocalPrefix) {
		return k
	}
	if len(k) < KeyLocalPrefixLength {
		log.Fatalf("local key %q malformed; should contain prefix %q and four-character designation", k, KeyLocalPrefix)
	}
	return k[KeyLocalPrefixLength:]
}

// DecodeKey returns a Key initialized by decoding a binary-encoded
// prefix from the passed in bytes slice. Any leftover bytes are
// returned.
func DecodeKey(b []byte) ([]byte, Key) {
	if len(b) == 0 {
		panic("cannot decode an empty key")
	}
	var keyBytes []byte
	b, keyBytes = encoding.DecodeBinary(b)
	return b, Key(keyBytes)
}

// Encode returns a binary-encoded version of the key appended to the
// supplied byte string, b.
func (k Key) Encode(b []byte) []byte {
	return encoding.EncodeBinary(b, []byte(k))
}

// Next returns the next key in lexicographic sort order.
func (k Key) Next() Key {
	return MakeKey(k, Key{0})
}

// PrefixEnd determines the end key given key as a prefix, that is the
// key that sorts precisely behind all keys starting with prefix: "1"
// is added to the final byte and the carry propagated. The special
// cases of nil and KeyMin always returns KeyMax.
func (k Key) PrefixEnd() Key {
	if len(k) == 0 {
		return KeyMax
	}
	end := append([]byte(nil), k...)
	for i := len(end) - 1; i >= 0; i-- {
		end[i] = end[i] + 1
		if end[i] != 0 {
			return end
		}
	}
	// This statement will only be reached if the key is already a
	// maximal byte string (i.e. already \xff...).
	return k
}

// Less implements the util.Ordered interface.
func (k Key) Less(l Key) bool {
	return bytes.Compare(k, l) < 0
}

// Equal returns whether two keys are identical.
func (k Key) Equal(l Key) bool {
	return bytes.Equal(k, l)
}

// Compare implements the llrb.Comparable interface for tree nodes.
func (k Key) Compare(b interval.Comparable) int {
	return bytes.Compare(k, b.(Key))
}

// Value specifies the value at a key. Multiple values at the same key
// are supported based on timestamp.
type Value struct {
	// Bytes is the byte string value.
	Bytes []byte
	// Checksum is a CRC-32-IEEE checksum. A Value will only be used in
	// a write operation by the database if either its checksum is zero
	// or the CRC checksum of Bytes matches it.
	// Values returned by the database will contain a checksum of the
	// contained value.
	Checksum uint32
	// Timestamp of value.
	Timestamp proto.Timestamp
}

// KeyValue is a pair of Key and Value for returned Key/Value pairs
// from ScanRequest/ScanResponse. It embeds a Key and a Value.
type KeyValue struct {
	Key
	Value
}

// RangeMetaKey returns a range metadata key for the given key. For ordinary
// keys this returns a level 2 metadata key - for level 2 keys, it returns a
// level 1 key. For level 1 keys and local keys, KeyMin is returned.
func RangeMetaKey(key Key) Key {
	if len(key) == 0 {
		return KeyMin
	}
	addr := key.Address()
	if !bytes.HasPrefix(addr, KeyMetaPrefix) {
		return MakeKey(KeyMeta2Prefix, addr)
	}
	if bytes.HasPrefix(addr, KeyMeta2Prefix) {
		return MakeKey(KeyMeta1Prefix, addr[len(KeyMeta2Prefix):])
	}

	return KeyMin
}

// RangeMetadataLookupKey returns the metadata key at which this range
// descriptor should be stored as a value.
func RangeMetadataLookupKey(r *proto.RangeDescriptor) Key {
	return RangeMetaKey(r.EndKey)
}

// ValidateRangeMetaKey validates that the given key is a valid Range Metadata
// key. It must have an appropriate metadata range prefix, and the original key
// value must be less than KeyMax. As a special case, KeyMin is considered a
// valid Range Metadata Key.
func ValidateRangeMetaKey(key Key) error {
	// KeyMin is a valid key.
	if len(key) == 0 {
		return nil
	}
	// Key must be at least as long as KeyMeta1Prefix.
	if len(key) < len(KeyMeta1Prefix) {
		return NewInvalidRangeMetaKeyError("too short", key)
	}

	prefix, body := key[:len(KeyMeta1Prefix)], key[len(KeyMeta1Prefix):]

	// The prefix must be equal to KeyMeta1Prefix or KeyMeta2Prefix
	if !bytes.HasPrefix(key, KeyMetaPrefix) {
		return NewInvalidRangeMetaKeyError(fmt.Sprintf("does not have %q prefix", KeyMetaPrefix), key)
	}
	if lvl := string(prefix[len(KeyMetaPrefix)]); lvl != "1" && lvl != "2" {
		return NewInvalidRangeMetaKeyError("meta level is not 1 or 2", key)
	}
	// Body of the key must sort before KeyMax
	if !body.Less(KeyMax) {
		return NewInvalidRangeMetaKeyError("body of range lookup is >= KeyMax", key)
	}

	return nil
}

func init() {
	if KeyLocalPrefixLength%7 != 0 {
		log.Fatal("local key prefix is not a multiple of 7: %d", KeyLocalPrefixLength)
	}
}

// Constants for system-reserved keys in the KV map.
var (
	// KeyMaxLength is the maximum key length.
	KeyMaxLength = 4096

	// KeyMin is a minimum key value which sorts before all other keys.
	KeyMin = Key("")
	// KeyMax is a maximum key value which sorts after all other keys.
	KeyMax = Key(strings.Repeat("\xff", KeyMaxLength))

	// KeyLocalPrefix is the prefix for keys which hold data local to a
	// RocksDB instance, such as range accounting information
	// (e.g. range metadata, range-spanning binary tree node pointers),
	// response cache values, transaction records, and message
	// queues. Some local data are replicated, such as transaction rows,
	// but are located in the local area so that they remain in
	// proximity to one or more keys which they affect, but without
	// unnecessarily polluting the key space. Further, some local data
	// are stored with MVCC and contribute to distributed transactions,
	// such as range metadata, range-spanning binary tree node pointers,
	// and message queues.
	//
	// The local key prefix has been deliberately chosen to sort before
	// the KeySystemPrefix, because these local keys are not addressable
	// via the meta range addressing indexes.
	KeyLocalPrefix = Key("\x00\x00\x00")

	// KeyLocalPrefixLength is the maximum length of the local prefix.
	// It includes both the standard prefix and an additional four
	// characters to designate the type of local data.
	//
	// NOTE: this is very important! In order to support prefix matches
	//   (e.g. for garbage collection of transaction and response cache
	//   rows), the number of bytes in the key local prefix must be a
	//   multiple of 7. This provides an encoded binary string with no
	//   leftover bits to "bleed" into the next byte in the non-prefix
	//   part of the local key.
	KeyLocalPrefixLength = len(KeyLocalPrefix) + 4

	// KeyLocalIdent stores an immutable identifier for this store,
	// created when the store is first bootstrapped.
	KeyLocalIdent = MakeKey(KeyLocalPrefix, Key("iden"))
	// KeyLocalRangeMetadataPrefix is the prefix for keys storing range metadata.
	// The value is a struct of type RangeMetadata.
	KeyLocalRangeMetadataPrefix = MakeKey(KeyLocalPrefix, Key("rng-"))
	// KeyLocalResponseCachePrefix is the prefix for keys storing command
	// responses used to guarantee idempotency (see ResponseCache). This key
	// prefix is duplicated in rocksdb_compaction.cc and must be kept in sync
	// if modified here.
	KeyLocalResponseCachePrefix = MakeKey(KeyLocalPrefix, Key("res-"))
	// KeyLocalTransactionPrefix specifies the key prefix for
	// transaction records. The suffix is the transaction id. This key
	// prefix is duplicated in rocksdb_compaction.cc and must be kept in
	// sync if modified here.
	KeyLocalTransactionPrefix = MakeKey(KeyLocalPrefix, Key("txn-"))
	// KeyLocalSnapshotIDGenerator is a snapshot ID generator sequence.
	// Snapshot IDs must be unique per store ID.
	KeyLocalSnapshotIDGenerator = MakeKey(KeyLocalPrefix, Key("ssid"))

	// KeySystemPrefix indicates the beginning of the key range for
	// global, system data which are replicated across the cluster.
	KeySystemPrefix = Key("\x00")
	KeySystemMax    = Key("\x01")

	// KeyMetaPrefix is the prefix for range metadata keys. Notice that
	// an extra null character in the prefix causes all range addressing
	// records to sort before any system tables which they might describe.
	KeyMetaPrefix = MakeKey(KeySystemPrefix, Key("\x00meta"))
	// KeyMeta1Prefix is the first level of key addressing. The value is a
	// RangeDescriptor struct.
	KeyMeta1Prefix = MakeKey(KeyMetaPrefix, Key("1"))
	// KeyMeta2Prefix is the second level of key addressing. The value is a
	// RangeDescriptor struct.
	KeyMeta2Prefix = MakeKey(KeyMetaPrefix, Key("2"))

	// KeyMetaMax is the end of the range of addressing keys.
	KeyMetaMax = MakeKey(KeySystemPrefix, Key("\x01"))

	// KeyConfigAccountingPrefix specifies the key prefix for accounting
	// configurations. The suffix is the affected key prefix.
	KeyConfigAccountingPrefix = MakeKey(KeySystemPrefix, Key("acct"))
	// KeyConfigPermissionPrefix specifies the key prefix for accounting
	// configurations. The suffix is the affected key prefix.
	KeyConfigPermissionPrefix = MakeKey(KeySystemPrefix, Key("perm"))
	// KeyConfigZonePrefix specifies the key prefix for zone
	// configurations. The suffix is the affected key prefix.
	KeyConfigZonePrefix = MakeKey(KeySystemPrefix, Key("zone"))
	// KeyNodeIDGenerator is the global node ID generator sequence.
	KeyNodeIDGenerator = MakeKey(KeySystemPrefix, Key("node-idgen"))
	// KeyRaftIDGenerator is the global Raft consensus group ID generator sequence.
	KeyRaftIDGenerator = MakeKey(KeySystemPrefix, Key("raft-idgen"))
	// KeyRangeIDGenerator is the global range ID generator sequence.
	KeyRangeIDGenerator = MakeKey(KeySystemPrefix, Key("range-idgen"))
	// KeySchemaPrefix specifies key prefixes for schema definitions.
	KeySchemaPrefix = MakeKey(KeySystemPrefix, Key("schema"))
	// KeyStoreIDGeneratorPrefix specifies key prefixes for sequence
	// generators, one per node, for store IDs.
	KeyStoreIDGeneratorPrefix = MakeKey(KeySystemPrefix, Key("store-idgen-"))
)
<|endoftext|>


<commit_before>package yaml

import (
	"bytes"
	"io"

	"github.com/goccy/go-yaml/internal/errors"
	"golang.org/x/xerrors"
)

// BytesMarshaler interface may be implemented by types to customize their
// behavior when being marshaled into a YAML document. The returned value
// is marshaled in place of the original value implementing Marshaler.
//
// If an error is returned by MarshalYAML, the marshaling procedure stops
// and returns with the provided error.
type BytesMarshaler interface {
	MarshalYAML() ([]byte, error)
}

// InterfaceMarshaler interface has MarshalYAML compatible with github.com/go-yaml/yaml package.
type InterfaceMarshaler interface {
	MarshalYAML() (interface{}, error)
}

// BytesUnmarshaler interface may be implemented by types to customize their
// behavior when being unmarshaled from a YAML document.
type BytesUnmarshaler interface {
	UnmarshalYAML([]byte) error
}

// InterfaceUnmarshaler interface has UnmarshalYAML compatible with github.com/go-yaml/yaml package.
type InterfaceUnmarshaler interface {
	UnmarshalYAML(func(interface{}) error) error
}

// MapItem is an item in a MapSlice.
type MapItem struct {
	Key, Value interface{}
}

// MapSlice encodes and decodes as a YAML map.
// The order of keys is preserved when encoding and decoding.
type MapSlice []MapItem

// ToMap convert to map[interface{}]interface{}.
func (s MapSlice) ToMap() map[interface{}]interface{} {
	v := map[interface{}]interface{}{}
	for _, item := range s {
		v[item.Key] = item.Value
	}
	return v
}

// Marshal serializes the value provided into a YAML document. The structure
// of the generated document will reflect the structure of the value itself.
// Maps and pointers (to struct, string, int, etc) are accepted as the in value.
//
// Struct fields are only marshalled if they are exported (have an upper case
// first letter), and are marshalled using the field name lowercased as the
// default key. Custom keys may be defined via the "yaml" name in the field
// tag: the content preceding the first comma is used as the key, and the
// following comma-separated options are used to tweak the marshalling process.
// Conflicting names result in a runtime error.
//
// The field tag format accepted is:
//
//     `(...) yaml:"[<key>][,<flag1>[,<flag2>]]" (...)`
//
// The following flags are currently supported:
//
//     omitempty    Only include the field if it's not set to the zero
//                  value for the type or to empty slices or maps.
//                  Zero valued structs will be omitted if all their public
//                  fields are zero, unless they implement an IsZero
//                  method (see the IsZeroer interface type), in which
//                  case the field will be included if that method returns true.
//
//     flow         Marshal using a flow style (useful for structs,
//                  sequences and maps).
//
//     inline       Inline the field, which must be a struct or a map,
//                  causing all of its fields or keys to be processed as if
//                  they were part of the outer struct. For maps, keys must
//                  not conflict with the yaml keys of other struct fields.
//
//     anchor       Marshal with anchor. If want to define anchor name explicitly, use anchor=name style.
//                  Otherwise, if used 'anchor' name only, used the field name lowercased as the anchor name
//
//     alias        Marshal with alias. If want to define alias name explicitly, use alias=name style.
//                  Otherwise, If omitted alias name and the field type is pointer type,
//                  assigned anchor name automatically from same pointer address.
//
// In addition, if the key is "-", the field is ignored.
//
// For example:
//
//     type T struct {
//         F int `yaml:"a,omitempty"`
//         B int
//     }
//     yaml.Marshal(&T{B: 2}) // Returns "b: 2\n"
//     yaml.Marshal(&T{F: 1}) // Returns "a: 1\nb: 0\n"
//
func Marshal(v interface{}) ([]byte, error) {
	var buf bytes.Buffer
	enc := NewEncoder(&buf)
	if err := enc.Encode(v); err != nil {
		return nil, errors.Wrapf(err, "failed to marshal")
	}
	return buf.Bytes(), nil
}

// Unmarshal decodes the first document found within the in byte slice
// and assigns decoded values into the out value.
//
// Struct fields are only unmarshalled if they are exported (have an
// upper case first letter), and are unmarshalled using the field name
// lowercased as the default key. Custom keys may be defined via the
// "yaml" name in the field tag: the content preceding the first comma
// is used as the key, and the following comma-separated options are
// used to tweak the marshalling process (see Marshal).
// Conflicting names result in a runtime error.
//
// For example:
//
//     type T struct {
//         F int `yaml:"a,omitempty"`
//         B int
//     }
//     var t T
//     yaml.Unmarshal([]byte("a: 1\nb: 2"), &t)
//
// See the documentation of Marshal for the format of tags and a list of
// supported tag options.
//
func Unmarshal(data []byte, v interface{}) error {
	dec := NewDecoder(bytes.NewBuffer(data))
	if err := dec.Decode(v); err != nil {
		if err == io.EOF {
			return nil
		}
		return errors.Wrapf(err, "failed to unmarshal")
	}
	return nil
}

// FormatError is a utility function that takes advantage of the metadata
// stored in the errors returned by this package's parser.
//
// If the second argument `colored` is true, the error message is colorized.
// If the third argument `inclSource` is true, the error message will
// contain snippets of the YAML source that was used.
func FormatError(e error, colored, inclSource bool) string {
	var pp errors.PrettyPrinter
	if xerrors.As(e, &pp) {
		var buf bytes.Buffer
		pp.PrettyPrint(&errors.Sink{&buf}, colored, inclSource)
		return buf.String()
	}

	return e.Error()

}
<commit_msg>Add MarshalWithOptions / UnmarshalWithOptions<commit_after>package yaml

import (
	"bytes"
	"io"

	"github.com/goccy/go-yaml/internal/errors"
	"golang.org/x/xerrors"
)

// BytesMarshaler interface may be implemented by types to customize their
// behavior when being marshaled into a YAML document. The returned value
// is marshaled in place of the original value implementing Marshaler.
//
// If an error is returned by MarshalYAML, the marshaling procedure stops
// and returns with the provided error.
type BytesMarshaler interface {
	MarshalYAML() ([]byte, error)
}

// InterfaceMarshaler interface has MarshalYAML compatible with github.com/go-yaml/yaml package.
type InterfaceMarshaler interface {
	MarshalYAML() (interface{}, error)
}

// BytesUnmarshaler interface may be implemented by types to customize their
// behavior when being unmarshaled from a YAML document.
type BytesUnmarshaler interface {
	UnmarshalYAML([]byte) error
}

// InterfaceUnmarshaler interface has UnmarshalYAML compatible with github.com/go-yaml/yaml package.
type InterfaceUnmarshaler interface {
	UnmarshalYAML(func(interface{}) error) error
}

// MapItem is an item in a MapSlice.
type MapItem struct {
	Key, Value interface{}
}

// MapSlice encodes and decodes as a YAML map.
// The order of keys is preserved when encoding and decoding.
type MapSlice []MapItem

// ToMap convert to map[interface{}]interface{}.
func (s MapSlice) ToMap() map[interface{}]interface{} {
	v := map[interface{}]interface{}{}
	for _, item := range s {
		v[item.Key] = item.Value
	}
	return v
}

// Marshal serializes the value provided into a YAML document. The structure
// of the generated document will reflect the structure of the value itself.
// Maps and pointers (to struct, string, int, etc) are accepted as the in value.
//
// Struct fields are only marshalled if they are exported (have an upper case
// first letter), and are marshalled using the field name lowercased as the
// default key. Custom keys may be defined via the "yaml" name in the field
// tag: the content preceding the first comma is used as the key, and the
// following comma-separated options are used to tweak the marshalling process.
// Conflicting names result in a runtime error.
//
// The field tag format accepted is:
//
//     `(...) yaml:"[<key>][,<flag1>[,<flag2>]]" (...)`
//
// The following flags are currently supported:
//
//     omitempty    Only include the field if it's not set to the zero
//                  value for the type or to empty slices or maps.
//                  Zero valued structs will be omitted if all their public
//                  fields are zero, unless they implement an IsZero
//                  method (see the IsZeroer interface type), in which
//                  case the field will be included if that method returns true.
//
//     flow         Marshal using a flow style (useful for structs,
//                  sequences and maps).
//
//     inline       Inline the field, which must be a struct or a map,
//                  causing all of its fields or keys to be processed as if
//                  they were part of the outer struct. For maps, keys must
//                  not conflict with the yaml keys of other struct fields.
//
//     anchor       Marshal with anchor. If want to define anchor name explicitly, use anchor=name style.
//                  Otherwise, if used 'anchor' name only, used the field name lowercased as the anchor name
//
//     alias        Marshal with alias. If want to define alias name explicitly, use alias=name style.
//                  Otherwise, If omitted alias name and the field type is pointer type,
//                  assigned anchor name automatically from same pointer address.
//
// In addition, if the key is "-", the field is ignored.
//
// For example:
//
//     type T struct {
//         F int `yaml:"a,omitempty"`
//         B int
//     }
//     yaml.Marshal(&T{B: 2}) // Returns "b: 2\n"
//     yaml.Marshal(&T{F: 1}) // Returns "a: 1\nb: 0\n"
//
func Marshal(v interface{}) ([]byte, error) {
	return MarshalWithOptions(v)
}

// MarshalWithOptions serializes the value provided into a YAML document with EncodeOptions.
func MarshalWithOptions(v interface{}, opts ...EncodeOption) ([]byte, error) {
	var buf bytes.Buffer
	enc := NewEncoder(&buf, opts...)
	if err := enc.Encode(v); err != nil {
		return nil, errors.Wrapf(err, "failed to marshal")
	}
	return buf.Bytes(), nil
}

// Unmarshal decodes the first document found within the in byte slice
// and assigns decoded values into the out value.
//
// Struct fields are only unmarshalled if they are exported (have an
// upper case first letter), and are unmarshalled using the field name
// lowercased as the default key. Custom keys may be defined via the
// "yaml" name in the field tag: the content preceding the first comma
// is used as the key, and the following comma-separated options are
// used to tweak the marshalling process (see Marshal).
// Conflicting names result in a runtime error.
//
// For example:
//
//     type T struct {
//         F int `yaml:"a,omitempty"`
//         B int
//     }
//     var t T
//     yaml.Unmarshal([]byte("a: 1\nb: 2"), &t)
//
// See the documentation of Marshal for the format of tags and a list of
// supported tag options.
//
func Unmarshal(data []byte, v interface{}) error {
	return UnmarshalWithOptions(data, v)
}

// UnmarshalWithOptions decodes with DecodeOptions the first document found within the in byte slice
// and assigns decoded values into the out value.
func UnmarshalWithOptions(data []byte, v interface{}, opts ...DecodeOption) error {
	dec := NewDecoder(bytes.NewBuffer(data), opts...)
	if err := dec.Decode(v); err != nil {
		if err == io.EOF {
			return nil
		}
		return errors.Wrapf(err, "failed to unmarshal")
	}
	return nil
}

// FormatError is a utility function that takes advantage of the metadata
// stored in the errors returned by this package's parser.
//
// If the second argument `colored` is true, the error message is colorized.
// If the third argument `inclSource` is true, the error message will
// contain snippets of the YAML source that was used.
func FormatError(e error, colored, inclSource bool) string {
	var pp errors.PrettyPrinter
	if xerrors.As(e, &pp) {
		var buf bytes.Buffer
		pp.PrettyPrint(&errors.Sink{&buf}, colored, inclSource)
		return buf.String()
	}

	return e.Error()
}
<|endoftext|>


<commit_before><commit_msg>add PAR1 header; not use HeadToUpper<commit_after><|endoftext|>


<commit_before>// Attributions
// some of the details below have been reproduced here from;
// Effective Go [http://golang.org/doc/effective_go.html]
// The Go Programming Language Specification [http://golang.org/ref/spec]
package main

import "fmt"

func main() {
	// In Go there are two distinct categories of types:
	//------------------------

	// 1. Value type variables point directly to their value contained in memory.
	// All types explored in previous sections except "pointer", "slice", "map",
	// and "channel" are value types.

	// 2. Reference types variable "pointer, slice, map, and channel", contains the
	// address of the memory location where the value is stored.

	// Merory allocation in Go
	//------------------------
	// Go has two allocation primitives, the built-in functions new and make.
	// They do different things and apply to different types, which can be
	// confusing, but the rules are simple.

	// allocation with new
	//------------------------
	// Let's talk about new first. It's a built-in function that allocates memory,
	// but unlike its namesakes in some other languages it does not initialize
	// the memory, it only zeros it. That is, new(T) allocates zeroed storage for
	// a new item of type T and returns its address, a value of type *T.
	// In Go terminology, it returns a pointer to a newly allocated zero value of type T.

	// Since the memory returned by new is zeroed, it's helpful to arrange when
	// designing your data structures that the zero value of each type can be used without further initialization.

	// Memory allocated as a result of declaring a variable of value type is zeroed. This is
	// know as the zero value of the type. This behavior is illustated in the following
	// code fragment

	var (
		a uint8
		b int
		c float64
		d bool
		e string
		f complex128
		g [4]bool
	)

	type car struct {
		make  string
		model string
		year  int16
	}

	h := car{}

	fmt.Printf("a = %d (%T)\n", a, a)
	fmt.Printf("b = %d (%T)\n", b, b)
	fmt.Printf("c = %f (%T)\n", c, c)
	fmt.Printf("d = %t (%T)\n", d, d)
	fmt.Printf("e = %#v (%T)\n", e, e)
	fmt.Printf("f = %f (%T)\n", f, f)
	fmt.Printf("g = %#v (%T)\n", g, g)
	fmt.Printf("h = %#v (%T)\n", h, h)

	// allocation with make
	//------------------------
	// The built-in function make(T, args) serves a purpose different from new(T).
	// It creates slices, maps, and channels only, and it returns an initialized (not zeroed)
	// value of type T (not *T). The reason for the distinction is that these three types
	// represent, under the covers, references to data structures that must be initialized
	// before use. A slice, for example, is a three-item descriptor containing a pointer to
	// the data (inside an array), the length, and the capacity, and until those items are
	// initialized, the slice is nil. For slices, maps, and channels, make initializes the
	// internal data structure and prepares the value for use.

	var i []int64
	if i == nil {
		fmt.Println("i is nil")
	}
	fmt.Printf("i = %#v (%T)\n", i, i)

	j := make([]int64, 5)
	if j == nil {
		fmt.Println("j is nil")
	}
	fmt.Printf("j = %#v (%T)\n", j, j)

	var k map[string]int64
	if k == nil {
		fmt.Println("k is nil")
	}
	fmt.Printf("k = %#v (%T)\n", k, k)

	l := make(map[string]int64, 5)
	fmt.Printf("l = %#v (%T)\n", l, l)

	var m chan string
	if m == nil {
		fmt.Println("m is nil")
	}
	fmt.Printf("m = %#v (%T)\n", m, m)

	n := make(chan string)
	fmt.Printf("n = %#v (%T)\n", n, n)
}
<commit_msg>add commnetray<commit_after>// Attributions
// some of the details below have been reproduced here from;
// Effective Go [http://golang.org/doc/effective_go.html]
// The Go Programming Language Specification [http://golang.org/ref/spec]
package main

import "fmt"

func main() {
	// In Go there are two distinct categories of types:
	//------------------------

	// 1. Value type variables point directly to their value contained in memory.
	// All types explored in previous sections except "pointer", "slice", "map",
	// and "channel" are value types.

	// 2. Reference types variable "pointer, slice, map, and channel", contains the
	// address of the memory location where the value is stored.

	// Merory allocation in Go
	//------------------------
	// Go has two allocation primitives, the built-in functions new and make.
	// They do different things and apply to different types, which can be
	// confusing, but the rules are simple.

	// allocation with new
	//------------------------
	// Let's talk about new first. It's a built-in function that allocates memory,
	// but unlike its namesakes in some other languages it does not initialize
	// the memory, it only zeros it. That is, new(T) allocates zeroed storage for
	// a new item of type T and returns its address, a value of type *T.
	// In Go terminology, it returns a pointer to a newly allocated zero value of type T.

	// Since the memory returned by new is zeroed, it's helpful to arrange when
	// designing your data structures that the zero value of each type can be used without
	// further initialization.

	// Memory allocated as a result of declaring a variable of value type is zeroed. This is
	// know as the zero value of the type. This behavior is illustated in the following
	// code fragment

	var (
		a uint8
		b int
		c float64
		d bool
		e string
		f complex128
		g [4]bool
	)

	type car struct {
		make  string
		model string
		year  int16
	}

	h := car{}

	fmt.Printf("a = %d (%T)\n", a, a)
	fmt.Printf("b = %d (%T)\n", b, b)
	fmt.Printf("c = %f (%T)\n", c, c)
	fmt.Printf("d = %t (%T)\n", d, d)
	fmt.Printf("e = %#v (%T)\n", e, e)
	fmt.Printf("f = %f (%T)\n", f, f)
	fmt.Printf("g = %#v (%T)\n", g, g)
	fmt.Printf("h = %#v (%T)\n", h, h)

	// allocation with make
	//------------------------
	// The built-in function make(T, args) serves a purpose different from new(T).
	// It creates slices, maps, and channels only, and it returns an initialized (not zeroed)
	// value of type T (not *T). The reason for the distinction is that these three types
	// represent, under the covers, references to data structures that must be initialized
	// before use. A slice, for example, is a three-item descriptor containing a pointer to
	// the data (inside an array), the length, and the capacity, and until those items are
	// initialized, the slice is nil. For slices, maps, and channels, make initializes the
	// internal data structure and prepares the value for use.

	var i []int64
	if i == nil {
		fmt.Println("i is nil")
	}
	fmt.Printf("i = %#v (%T)\n", i, i)

	j := make([]int64, 5)
	if j == nil {
		fmt.Println("j is nil")
	}
	fmt.Printf("j = %#v (%T)\n", j, j)

	var k map[string]int64
	if k == nil {
		fmt.Println("k is nil")
	}
	fmt.Printf("k = %#v (%T)\n", k, k)

	l := make(map[string]int64, 5)
	fmt.Printf("l = %#v (%T)\n", l, l)

	var m chan string
	if m == nil {
		fmt.Println("m is nil")
	}
	fmt.Printf("m = %#v (%T)\n", m, m)

	n := make(chan string)
	fmt.Printf("n = %#v (%T)\n", n, n)
}
<|endoftext|>