<commit_before>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package shim

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/containerd/containerd/namespaces"
	"github.com/gogo/protobuf/proto"
	"github.com/gogo/protobuf/types"
	"github.com/pkg/errors"
)

var runtimePaths sync.Map

// Command returns the shim command with the provided args and configuration
func Command(ctx context.Context, runtime, containerdAddress, containerdTTRPCAddress, path string, opts *types.Any, cmdArgs ...string) (*exec.Cmd, error) {
	ns, err := namespaces.NamespaceRequired(ctx)
	if err != nil {
		return nil, err
	}
	self, err := os.Executable()
	if err != nil {
		return nil, err
	}
	args := []string{
		"-namespace", ns,
		"-address", containerdAddress,
		"-publish-binary", self,
	}
	args = append(args, cmdArgs...)
	name := BinaryName(runtime)
	if name == "" {
		return nil, fmt.Errorf("invalid runtime name %s, correct runtime name should format like io.containerd.runc.v1", runtime)
	}

	var cmdPath string
	cmdPathI, cmdPathFound := runtimePaths.Load(name)
	if cmdPathFound {
		cmdPath = cmdPathI.(string)
	} else {
		var lerr error
		binaryPath := BinaryPath(runtime)
		if _, serr := os.Stat(binaryPath); serr == nil {
			cmdPath = binaryPath
		}

		if cmdPath == "" {
			if cmdPath, lerr = exec.LookPath(name); err != nil {
				if eerr, ok := lerr.(*exec.Error); ok {
					if eerr.Err == exec.ErrNotFound {
						// LookPath only finds current directory matches based on
						// the callers current directory but the caller is not
						// likely in the same directory as the containerd
						// executables. Instead match the calling binaries path
						// (containerd) and see if they are side by side. If so
						// execute the shim found there.
						testPath := filepath.Join(filepath.Dir(self), name)
						if _, serr := os.Stat(testPath); serr == nil {
							cmdPath = testPath
						}
						if cmdPath == "" {
							return nil, errors.Wrapf(os.ErrNotExist, "runtime %q binary not installed %q", runtime, name)
						}
					}
				}
			}
		}
		cmdPath, err = filepath.Abs(cmdPath)
		if err != nil {
			return nil, err
		}
		if cmdPathI, cmdPathFound = runtimePaths.LoadOrStore(name, cmdPath); cmdPathFound {
			// We didn't store cmdPath we loaded an already cached value. Use it.
			cmdPath = cmdPathI.(string)
		}
	}

	cmd := exec.Command(cmdPath, args...)
	cmd.Dir = path
	cmd.Env = append(
		os.Environ(),
		"GOMAXPROCS=2",
		fmt.Sprintf("%s=%s", ttrpcAddressEnv, containerdTTRPCAddress),
	)
	cmd.SysProcAttr = getSysProcAttr()
	if opts != nil {
		d, err := proto.Marshal(opts)
		if err != nil {
			return nil, err
		}
		cmd.Stdin = bytes.NewReader(d)
	}
	return cmd, nil
}

// BinaryName returns the shim binary name from the runtime name,
// empty string returns means runtime name is invalid
func BinaryName(runtime string) string {
	// runtime name should format like $prefix.name.version
	parts := strings.Split(runtime, ".")
	if len(parts) < 2 {
		return ""
	}

	return fmt.Sprintf(shimBinaryFormat, parts[len(parts)-2], parts[len(parts)-1])
}

// BinaryPath returns the full path for the shim binary from the runtime name,
// empty string returns means runtime name is invalid
func BinaryPath(runtime string) string {
	dir := filepath.Dir(runtime)
	binary := BinaryName(runtime)

	path, err := filepath.Abs(filepath.Join(dir, binary))
	if err != nil {
		return ""
	}

	return path
}

// Connect to the provided address
func Connect(address string, d func(string, time.Duration) (net.Conn, error)) (net.Conn, error) {
	return d(address, 100*time.Second)
}

// WritePidFile writes a pid file atomically
func WritePidFile(path string, pid int) error {
	path, err := filepath.Abs(path)
	if err != nil {
		return err
	}
	tempPath := filepath.Join(filepath.Dir(path), fmt.Sprintf(".%s", filepath.Base(path)))
	f, err := os.OpenFile(tempPath, os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_SYNC, 0666)
	if err != nil {
		return err
	}
	_, err = fmt.Fprintf(f, "%d", pid)
	f.Close()
	if err != nil {
		return err
	}
	return os.Rename(tempPath, path)
}

// WriteAddress writes a address file atomically
func WriteAddress(path, address string) error {
	path, err := filepath.Abs(path)
	if err != nil {
		return err
	}
	tempPath := filepath.Join(filepath.Dir(path), fmt.Sprintf(".%s", filepath.Base(path)))
	f, err := os.OpenFile(tempPath, os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_SYNC, 0666)
	if err != nil {
		return err
	}
	_, err = f.WriteString(address)
	f.Close()
	if err != nil {
		return err
	}
	return os.Rename(tempPath, path)
}

// ErrNoAddress is returned when the address file has no content
var ErrNoAddress = errors.New("no shim address")

// ReadAddress returns the shim's socket address from the path
func ReadAddress(path string) (string, error) {
	path, err := filepath.Abs(path)
	if err != nil {
		return "", err
	}
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return "", err
	}
	if len(data) == 0 {
		return "", ErrNoAddress
	}
	return string(data), nil
}
<commit_msg>Fix error checking when resolving shim binary path<commit_after>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package shim

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/containerd/containerd/namespaces"
	"github.com/gogo/protobuf/proto"
	"github.com/gogo/protobuf/types"
	"github.com/pkg/errors"
)

var runtimePaths sync.Map

// Command returns the shim command with the provided args and configuration
func Command(ctx context.Context, runtime, containerdAddress, containerdTTRPCAddress, path string, opts *types.Any, cmdArgs ...string) (*exec.Cmd, error) {
	ns, err := namespaces.NamespaceRequired(ctx)
	if err != nil {
		return nil, err
	}
	self, err := os.Executable()
	if err != nil {
		return nil, err
	}
	args := []string{
		"-namespace", ns,
		"-address", containerdAddress,
		"-publish-binary", self,
	}
	args = append(args, cmdArgs...)
	name := BinaryName(runtime)
	if name == "" {
		return nil, fmt.Errorf("invalid runtime name %s, correct runtime name should format like io.containerd.runc.v1", runtime)
	}

	var cmdPath string
	cmdPathI, cmdPathFound := runtimePaths.Load(name)
	if cmdPathFound {
		cmdPath = cmdPathI.(string)
	} else {
		var lerr error
		binaryPath := BinaryPath(runtime)
		if _, serr := os.Stat(binaryPath); serr == nil {
			cmdPath = binaryPath
		}

		if cmdPath == "" {
			if cmdPath, lerr = exec.LookPath(name); lerr != nil {
				if eerr, ok := lerr.(*exec.Error); ok {
					if eerr.Err == exec.ErrNotFound {
						// LookPath only finds current directory matches based on
						// the callers current directory but the caller is not
						// likely in the same directory as the containerd
						// executables. Instead match the calling binaries path
						// (containerd) and see if they are side by side. If so
						// execute the shim found there.
						testPath := filepath.Join(filepath.Dir(self), name)
						if _, serr := os.Stat(testPath); serr == nil {
							cmdPath = testPath
						}
						if cmdPath == "" {
							return nil, errors.Wrapf(os.ErrNotExist, "runtime %q binary not installed %q", runtime, name)
						}
					}
				}
			}
		}
		cmdPath, err = filepath.Abs(cmdPath)
		if err != nil {
			return nil, err
		}
		if cmdPathI, cmdPathFound = runtimePaths.LoadOrStore(name, cmdPath); cmdPathFound {
			// We didn't store cmdPath we loaded an already cached value. Use it.
			cmdPath = cmdPathI.(string)
		}
	}

	cmd := exec.Command(cmdPath, args...)
	cmd.Dir = path
	cmd.Env = append(
		os.Environ(),
		"GOMAXPROCS=2",
		fmt.Sprintf("%s=%s", ttrpcAddressEnv, containerdTTRPCAddress),
	)
	cmd.SysProcAttr = getSysProcAttr()
	if opts != nil {
		d, err := proto.Marshal(opts)
		if err != nil {
			return nil, err
		}
		cmd.Stdin = bytes.NewReader(d)
	}
	return cmd, nil
}

// BinaryName returns the shim binary name from the runtime name,
// empty string returns means runtime name is invalid
func BinaryName(runtime string) string {
	// runtime name should format like $prefix.name.version
	parts := strings.Split(runtime, ".")
	if len(parts) < 2 {
		return ""
	}

	return fmt.Sprintf(shimBinaryFormat, parts[len(parts)-2], parts[len(parts)-1])
}

// BinaryPath returns the full path for the shim binary from the runtime name,
// empty string returns means runtime name is invalid
func BinaryPath(runtime string) string {
	dir := filepath.Dir(runtime)
	binary := BinaryName(runtime)

	path, err := filepath.Abs(filepath.Join(dir, binary))
	if err != nil {
		return ""
	}

	return path
}

// Connect to the provided address
func Connect(address string, d func(string, time.Duration) (net.Conn, error)) (net.Conn, error) {
	return d(address, 100*time.Second)
}

// WritePidFile writes a pid file atomically
func WritePidFile(path string, pid int) error {
	path, err := filepath.Abs(path)
	if err != nil {
		return err
	}
	tempPath := filepath.Join(filepath.Dir(path), fmt.Sprintf(".%s", filepath.Base(path)))
	f, err := os.OpenFile(tempPath, os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_SYNC, 0666)
	if err != nil {
		return err
	}
	_, err = fmt.Fprintf(f, "%d", pid)
	f.Close()
	if err != nil {
		return err
	}
	return os.Rename(tempPath, path)
}

// WriteAddress writes a address file atomically
func WriteAddress(path, address string) error {
	path, err := filepath.Abs(path)
	if err != nil {
		return err
	}
	tempPath := filepath.Join(filepath.Dir(path), fmt.Sprintf(".%s", filepath.Base(path)))
	f, err := os.OpenFile(tempPath, os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_SYNC, 0666)
	if err != nil {
		return err
	}
	_, err = f.WriteString(address)
	f.Close()
	if err != nil {
		return err
	}
	return os.Rename(tempPath, path)
}

// ErrNoAddress is returned when the address file has no content
var ErrNoAddress = errors.New("no shim address")

// ReadAddress returns the shim's socket address from the path
func ReadAddress(path string) (string, error) {
	path, err := filepath.Abs(path)
	if err != nil {
		return "", err
	}
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return "", err
	}
	if len(data) == 0 {
		return "", ErrNoAddress
	}
	return string(data), nil
}
<|endoftext|>


<commit_before>/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
*/

package pbft

import (
	"reflect"
)

func (instance *Plugin) correctViewChange(vc *ViewChange) bool {
	for _, p := range append(vc.Pset, vc.Qset...) {
		if !(p.View < vc.View && p.SequenceNumber > vc.H && p.SequenceNumber <= vc.H+instance.L) {
			logger.Debug("invalid p entry in view-change: vc(v:%d h:%d) p(v:%d n:%d)",
				vc.View, vc.H, p.View, p.SequenceNumber)
			return false
		}
	}

	for _, c := range vc.Cset {
		// XXX the paper says c.n > vc.h
		if !(c.SequenceNumber >= vc.H && c.SequenceNumber <= vc.H+instance.L) {
			logger.Debug("invalid c entry in view-change: vc(v:%d h:%d) c(n:%d)",
				vc.View, vc.H, c.SequenceNumber)
			return false
		}
	}

	return true
}

func (instance *Plugin) sendViewChange() error {
	instance.view++
	instance.activeView = false

	// P set: requests that have prepared here
	//
	// "<n,d,v> has a prepared certificate, and no request
	// prepared in a later view with the same number"

	for idx, cert := range instance.certStore {
		if cert.prePrepare == nil {
			continue
		}

		digest := cert.prePrepare.RequestDigest
		if !instance.prepared(digest, idx.v, idx.n) {
			continue
		}

		if p, ok := instance.pset[idx.n]; ok && p.View > idx.v {
			continue
		}

		instance.pset[idx.n] = &ViewChange_PQ{
			SequenceNumber: idx.n,
			Digest:         digest,
			View:           idx.v,
		}
	}

	// Q set: requests that have pre-prepared here (pre-prepare or
	// prepare sent)
	//
	// "<n,d,v>: requests that pre-prepared here, and did not
	// pre-prepare in a later view with the same number"

	for idx, cert := range instance.certStore {
		if cert.prePrepare == nil {
			continue
		}

		digest := cert.prePrepare.RequestDigest
		if !instance.prePrepared(digest, idx.v, idx.n) {
			continue
		}

		qi := qidx{digest, idx.n}
		if q, ok := instance.qset[qi]; ok && q.View > idx.v {
			continue
		}

		instance.qset[qi] = &ViewChange_PQ{
			SequenceNumber: idx.n,
			Digest:         digest,
			View:           idx.v,
		}
	}

	// clear old messages
	for idx := range instance.certStore {
		if idx.v < instance.view {
			delete(instance.certStore, idx)
			// XXX how do we clear reqStore?
		}
	}
	for idx := range instance.viewChangeStore {
		if idx.v < instance.view {
			delete(instance.viewChangeStore, idx)
		}
	}

	vc := &ViewChange{
		View:      instance.view,
		H:         instance.h,
		ReplicaId: instance.id,
	}

	for n, state := range instance.chkpts {
		vc.Cset = append(vc.Cset, &ViewChange_C{
			SequenceNumber: n,
			Digest:         state,
		})
	}

	for _, p := range instance.pset {
		vc.Pset = append(vc.Pset, p)
	}

	for _, q := range instance.qset {
		vc.Qset = append(vc.Qset, q)
	}

	logger.Info("Replica %d sending view-change, v:%d, h:%d, |C|:%d, |P|:%d, |Q|:%d",
		instance.id, vc.View, vc.H, len(vc.Cset), len(vc.Pset), len(vc.Qset))

	return instance.broadcast(&Message{&Message_ViewChange{vc}}, true)
}

func (instance *Plugin) recvViewChange(vc *ViewChange) error {
	logger.Info("Replica %d received view-change from replica %d, v:%d, h:%d, |C|:%d, |P|:%d, |Q|:%d",
		instance.id, vc.ReplicaId, vc.View, vc.H, len(vc.Cset), len(vc.Pset), len(vc.Qset))

	if !(vc.View >= instance.view && instance.correctViewChange(vc) || instance.viewChangeStore[vcidx{vc.View, vc.ReplicaId}] != nil) {
		logger.Warning("View-change message incorrect")
		return nil
	}

	instance.viewChangeStore[vcidx{vc.View, vc.ReplicaId}] = vc

	if instance.getPrimary(instance.view) == instance.id {
		return instance.sendNewView()
	}

	return instance.processNewView()
}

func (instance *Plugin) sendNewView() (err error) {
	if instance.lastNewView.View == instance.view {
		return
	}

	vset := instance.getViewChanges()

	cp, ok := instance.selectInitialCheckpoint(vset)
	if !ok {
		return
	}

	msgList := instance.assignSequenceNumbers(vset, cp)
	if msgList == nil {
		return
	}

	nv := &NewView{
		View:      instance.view,
		Vset:      vset,
		Xset:      msgList,
		ReplicaId: instance.id,
	}

	logger.Info("New primary %d sending new-view, v:%d, X:%+v",
		instance.id, nv.View, nv.Xset)

	err = instance.broadcast(&Message{&Message_NewView{nv}}, false)
	if err != nil {
		return err
	}
	instance.lastNewView = *nv
	return instance.processNewView()
}

func (instance *Plugin) recvNewView(nv *NewView) error {
	logger.Info("Replica %d received new-view %d",
		instance.id, nv.View)

	if !(nv.View > 0 && nv.View >= instance.view && instance.getPrimary(nv.View) == nv.ReplicaId && instance.lastNewView.View != nv.View) {
		logger.Info("Replica %d rejecting invalid new-view from %d, v:%d",
			instance.id, nv.ReplicaId, nv.View)
		return nil
	}

	instance.lastNewView = *nv
	return instance.processNewView()
}

func (instance *Plugin) processNewView() error {
	nv := instance.lastNewView

	if nv.View == 0 {
		return nil
	}

	if instance.activeView {
		logger.Info("Replica %d ignoring new-view from %d, v:%d: we are active in view %d",
			instance.id, nv.ReplicaId, nv.View, instance.view)
		return nil
	}

	// XXX check new-view certificate

	cp, ok := instance.selectInitialCheckpoint(nv.Vset)
	if !ok {
		logger.Warning("could not determine initial checkpoint: %+v",
			instance.viewChangeStore)
		return instance.sendViewChange()
	}

	msgList := instance.assignSequenceNumbers(nv.Vset, cp)
	if msgList == nil {
		logger.Warning("could not assign sequence numbers: %+v",
			instance.viewChangeStore)
		return instance.sendViewChange()
	}

	if !reflect.DeepEqual(msgList, nv.Xset) {
		logger.Warning("failed to verify new-view Xset: computed %+v, received %+v",
			msgList, nv.Xset)
		return instance.sendViewChange()
	}

	for n, d := range nv.Xset {
		// XXX why should we use "h ≥ min{n | ∃d : (<n,d> ∈ X)}"?
		// "h ≥ min{n | ∃d : (<n,d> ∈ X)} ∧ ∀<n,d> ∈ X : (n ≤ h ∨ ∃m ∈ in : (D(m) = d))"
		if n <= instance.h {
			continue
		} else {
			if d == "" {
				// NULL request; skip
				continue
			}

			if _, ok := instance.reqStore[d]; !ok {
				logger.Warning("missing assigned, non-checkpointed request %s",
					d)
				// XXX fetch request?
				return nil
			}
		}
	}

	logger.Info("Replica %d accepting new-view to view %d", instance.id, instance.view)

	instance.activeView = true
	for n, d := range nv.Xset {
		preprep := &PrePrepare{
			View:           instance.view,
			SequenceNumber: n,
			RequestDigest:  d,
			ReplicaId:      instance.id,
		}
		cert := instance.getCert(instance.view, n)
		cert.prePrepare = preprep
		if n < instance.seqNo {
			instance.seqNo = n
		}
	}

	if instance.getPrimary(instance.view) != instance.id {
		for n, d := range nv.Xset {
			prep := &Prepare{
				View:           instance.view,
				SequenceNumber: n,
				RequestDigest:  d,
				ReplicaId:      instance.id,
			}
			cert := instance.getCert(instance.view, n)
			cert.prepare = append(cert.prepare, prep)
			cert.sentPrepare = true
			instance.broadcast(&Message{&Message_Prepare{prep}}, true)
		}
	}

	return nil
}

func (instance *Plugin) getViewChanges() (vset []*ViewChange) {
	vset = make([]*ViewChange, 0)
	for _, vc := range instance.viewChangeStore {
		vset = append(vset, vc)
	}

	return

}

func (instance *Plugin) selectInitialCheckpoint(vset []*ViewChange) (checkpoint uint64, ok bool) {
	checkpoints := make(map[ViewChange_C][]*ViewChange)
	for _, vc := range vset {
		for _, c := range vc.Cset {
			checkpoints[*c] = append(checkpoints[*c], vc)
		}
	}

	if len(checkpoints) == 0 {
		logger.Debug("no checkpoints to select from: %d %s",
			len(instance.viewChangeStore), checkpoints)
		return
	}

	for idx, vcList := range checkpoints {
		// need weak certificate for the checkpoint
		if uint(len(vcList)) <= instance.f { // type casting necessary to match types
			logger.Debug("no weak certificate for n:%d",
				idx.SequenceNumber)
			continue
		}

		quorum := uint(0)
		for _, vc := range vcList {
			if vc.H <= idx.SequenceNumber {
				quorum++
			}
		}

		if quorum <= 2*instance.f {
			logger.Debug("no quorum for n:%d",
				idx.SequenceNumber)
			continue
		}

		if checkpoint <= idx.SequenceNumber {
			checkpoint = idx.SequenceNumber
			ok = true
		}
	}

	return
}

func (instance *Plugin) assignSequenceNumbers(vset []*ViewChange, h uint64) (msgList map[uint64]string) {
	msgList = make(map[uint64]string)

	// "for all n such that h < n <= h + L"
nLoop:
	for n := h + 1; n <= h+instance.L; n++ {
		// "∃m ∈ S..."
		for _, m := range vset {
			// "...with <n,d,v> ∈ m.P"
			for _, em := range m.Pset {
				quorum := uint(0)
				// "A1. ∃2f+1 messages m' ∈ S"
			mpLoop:
				for _, mp := range vset {
					if mp.H >= n {
						continue
					}
					// "∀<n,d',v'> ∈ m'.P"
					for _, emp := range mp.Pset {
						if n == emp.SequenceNumber && !(emp.View < em.View || (emp.View == em.View && emp.Digest == em.Digest)) {
							continue mpLoop
						}
					}
					quorum++
				}

				if quorum < 2*instance.f+1 {
					continue
				}

				quorum = 0
				// "A2. ∃f+1 messages m' ∈ S"
				for _, mp := range vset {
					// "∃<n,d',v'> ∈ m'.Q"
					for _, emp := range mp.Qset {
						if n == emp.SequenceNumber && emp.View >= em.View && emp.Digest == em.Digest {
							quorum++
						}
					}
				}

				if quorum < instance.f+1 {
					continue
				}

				// "then select the null request with digest d for number n"
				msgList[n] = em.Digest

				continue nLoop
			}
		}

		quorum := uint(0)
		// "else if ∃2f+1 messages m ∈ S"
	nullLoop:
		for _, m := range vset {
			// "m.P has no entry"
			for _, em := range m.Pset {
				if em.SequenceNumber == n {
					continue nullLoop
				}
			}
			quorum++
		}

		if quorum >= 2*instance.f+1 {
			// "then select the null request for number n"
			msgList[n] = ""

			continue nLoop
		}

		return nil
	}

	return
}
<commit_msg>Remove vset slice initialization<commit_after>/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
*/

package pbft

import (
	"reflect"
)

func (instance *Plugin) correctViewChange(vc *ViewChange) bool {
	for _, p := range append(vc.Pset, vc.Qset...) {
		if !(p.View < vc.View && p.SequenceNumber > vc.H && p.SequenceNumber <= vc.H+instance.L) {
			logger.Debug("invalid p entry in view-change: vc(v:%d h:%d) p(v:%d n:%d)",
				vc.View, vc.H, p.View, p.SequenceNumber)
			return false
		}
	}

	for _, c := range vc.Cset {
		// XXX the paper says c.n > vc.h
		if !(c.SequenceNumber >= vc.H && c.SequenceNumber <= vc.H+instance.L) {
			logger.Debug("invalid c entry in view-change: vc(v:%d h:%d) c(n:%d)",
				vc.View, vc.H, c.SequenceNumber)
			return false
		}
	}

	return true
}

func (instance *Plugin) sendViewChange() error {
	instance.view++
	instance.activeView = false

	// P set: requests that have prepared here
	//
	// "<n,d,v> has a prepared certificate, and no request
	// prepared in a later view with the same number"

	for idx, cert := range instance.certStore {
		if cert.prePrepare == nil {
			continue
		}

		digest := cert.prePrepare.RequestDigest
		if !instance.prepared(digest, idx.v, idx.n) {
			continue
		}

		if p, ok := instance.pset[idx.n]; ok && p.View > idx.v {
			continue
		}

		instance.pset[idx.n] = &ViewChange_PQ{
			SequenceNumber: idx.n,
			Digest:         digest,
			View:           idx.v,
		}
	}

	// Q set: requests that have pre-prepared here (pre-prepare or
	// prepare sent)
	//
	// "<n,d,v>: requests that pre-prepared here, and did not
	// pre-prepare in a later view with the same number"

	for idx, cert := range instance.certStore {
		if cert.prePrepare == nil {
			continue
		}

		digest := cert.prePrepare.RequestDigest
		if !instance.prePrepared(digest, idx.v, idx.n) {
			continue
		}

		qi := qidx{digest, idx.n}
		if q, ok := instance.qset[qi]; ok && q.View > idx.v {
			continue
		}

		instance.qset[qi] = &ViewChange_PQ{
			SequenceNumber: idx.n,
			Digest:         digest,
			View:           idx.v,
		}
	}

	// clear old messages
	for idx := range instance.certStore {
		if idx.v < instance.view {
			delete(instance.certStore, idx)
			// XXX how do we clear reqStore?
		}
	}
	for idx := range instance.viewChangeStore {
		if idx.v < instance.view {
			delete(instance.viewChangeStore, idx)
		}
	}

	vc := &ViewChange{
		View:      instance.view,
		H:         instance.h,
		ReplicaId: instance.id,
	}

	for n, state := range instance.chkpts {
		vc.Cset = append(vc.Cset, &ViewChange_C{
			SequenceNumber: n,
			Digest:         state,
		})
	}

	for _, p := range instance.pset {
		vc.Pset = append(vc.Pset, p)
	}

	for _, q := range instance.qset {
		vc.Qset = append(vc.Qset, q)
	}

	logger.Info("Replica %d sending view-change, v:%d, h:%d, |C|:%d, |P|:%d, |Q|:%d",
		instance.id, vc.View, vc.H, len(vc.Cset), len(vc.Pset), len(vc.Qset))

	return instance.broadcast(&Message{&Message_ViewChange{vc}}, true)
}

func (instance *Plugin) recvViewChange(vc *ViewChange) error {
	logger.Info("Replica %d received view-change from replica %d, v:%d, h:%d, |C|:%d, |P|:%d, |Q|:%d",
		instance.id, vc.ReplicaId, vc.View, vc.H, len(vc.Cset), len(vc.Pset), len(vc.Qset))

	if !(vc.View >= instance.view && instance.correctViewChange(vc) || instance.viewChangeStore[vcidx{vc.View, vc.ReplicaId}] != nil) {
		logger.Warning("View-change message incorrect")
		return nil
	}

	instance.viewChangeStore[vcidx{vc.View, vc.ReplicaId}] = vc

	if instance.getPrimary(instance.view) == instance.id {
		return instance.sendNewView()
	}

	return instance.processNewView()
}

func (instance *Plugin) sendNewView() (err error) {
	if instance.lastNewView.View == instance.view {
		return
	}

	vset := instance.getViewChanges()

	cp, ok := instance.selectInitialCheckpoint(vset)
	if !ok {
		return
	}

	msgList := instance.assignSequenceNumbers(vset, cp)
	if msgList == nil {
		return
	}

	nv := &NewView{
		View:      instance.view,
		Vset:      vset,
		Xset:      msgList,
		ReplicaId: instance.id,
	}

	logger.Info("New primary %d sending new-view, v:%d, X:%+v",
		instance.id, nv.View, nv.Xset)

	err = instance.broadcast(&Message{&Message_NewView{nv}}, false)
	if err != nil {
		return err
	}
	instance.lastNewView = *nv
	return instance.processNewView()
}

func (instance *Plugin) recvNewView(nv *NewView) error {
	logger.Info("Replica %d received new-view %d",
		instance.id, nv.View)

	if !(nv.View > 0 && nv.View >= instance.view && instance.getPrimary(nv.View) == nv.ReplicaId && instance.lastNewView.View != nv.View) {
		logger.Info("Replica %d rejecting invalid new-view from %d, v:%d",
			instance.id, nv.ReplicaId, nv.View)
		return nil
	}

	instance.lastNewView = *nv
	return instance.processNewView()
}

func (instance *Plugin) processNewView() error {
	nv := instance.lastNewView

	if nv.View == 0 {
		return nil
	}

	if instance.activeView {
		logger.Info("Replica %d ignoring new-view from %d, v:%d: we are active in view %d",
			instance.id, nv.ReplicaId, nv.View, instance.view)
		return nil
	}

	// XXX check new-view certificate

	cp, ok := instance.selectInitialCheckpoint(nv.Vset)
	if !ok {
		logger.Warning("could not determine initial checkpoint: %+v",
			instance.viewChangeStore)
		return instance.sendViewChange()
	}

	msgList := instance.assignSequenceNumbers(nv.Vset, cp)
	if msgList == nil {
		logger.Warning("could not assign sequence numbers: %+v",
			instance.viewChangeStore)
		return instance.sendViewChange()
	}

	if !reflect.DeepEqual(msgList, nv.Xset) {
		logger.Warning("failed to verify new-view Xset: computed %+v, received %+v",
			msgList, nv.Xset)
		return instance.sendViewChange()
	}

	for n, d := range nv.Xset {
		// XXX why should we use "h ≥ min{n | ∃d : (<n,d> ∈ X)}"?
		// "h ≥ min{n | ∃d : (<n,d> ∈ X)} ∧ ∀<n,d> ∈ X : (n ≤ h ∨ ∃m ∈ in : (D(m) = d))"
		if n <= instance.h {
			continue
		} else {
			if d == "" {
				// NULL request; skip
				continue
			}

			if _, ok := instance.reqStore[d]; !ok {
				logger.Warning("missing assigned, non-checkpointed request %s",
					d)
				// XXX fetch request?
				return nil
			}
		}
	}

	logger.Info("Replica %d accepting new-view to view %d", instance.id, instance.view)

	instance.activeView = true
	for n, d := range nv.Xset {
		preprep := &PrePrepare{
			View:           instance.view,
			SequenceNumber: n,
			RequestDigest:  d,
			ReplicaId:      instance.id,
		}
		cert := instance.getCert(instance.view, n)
		cert.prePrepare = preprep
		if n < instance.seqNo {
			instance.seqNo = n
		}
	}

	if instance.getPrimary(instance.view) != instance.id {
		for n, d := range nv.Xset {
			prep := &Prepare{
				View:           instance.view,
				SequenceNumber: n,
				RequestDigest:  d,
				ReplicaId:      instance.id,
			}
			cert := instance.getCert(instance.view, n)
			cert.prepare = append(cert.prepare, prep)
			cert.sentPrepare = true
			instance.broadcast(&Message{&Message_Prepare{prep}}, true)
		}
	}

	return nil
}

func (instance *Plugin) getViewChanges() (vset []*ViewChange) {
	for _, vc := range instance.viewChangeStore {
		vset = append(vset, vc)
	}

	return

}

func (instance *Plugin) selectInitialCheckpoint(vset []*ViewChange) (checkpoint uint64, ok bool) {
	checkpoints := make(map[ViewChange_C][]*ViewChange)
	for _, vc := range vset {
		for _, c := range vc.Cset {
			checkpoints[*c] = append(checkpoints[*c], vc)
		}
	}

	if len(checkpoints) == 0 {
		logger.Debug("no checkpoints to select from: %d %s",
			len(instance.viewChangeStore), checkpoints)
		return
	}

	for idx, vcList := range checkpoints {
		// need weak certificate for the checkpoint
		if uint(len(vcList)) <= instance.f { // type casting necessary to match types
			logger.Debug("no weak certificate for n:%d",
				idx.SequenceNumber)
			continue
		}

		quorum := uint(0)
		for _, vc := range vcList {
			if vc.H <= idx.SequenceNumber {
				quorum++
			}
		}

		if quorum <= 2*instance.f {
			logger.Debug("no quorum for n:%d",
				idx.SequenceNumber)
			continue
		}

		if checkpoint <= idx.SequenceNumber {
			checkpoint = idx.SequenceNumber
			ok = true
		}
	}

	return
}

func (instance *Plugin) assignSequenceNumbers(vset []*ViewChange, h uint64) (msgList map[uint64]string) {
	msgList = make(map[uint64]string)

	// "for all n such that h < n <= h + L"
nLoop:
	for n := h + 1; n <= h+instance.L; n++ {
		// "∃m ∈ S..."
		for _, m := range vset {
			// "...with <n,d,v> ∈ m.P"
			for _, em := range m.Pset {
				quorum := uint(0)
				// "A1. ∃2f+1 messages m' ∈ S"
			mpLoop:
				for _, mp := range vset {
					if mp.H >= n {
						continue
					}
					// "∀<n,d',v'> ∈ m'.P"
					for _, emp := range mp.Pset {
						if n == emp.SequenceNumber && !(emp.View < em.View || (emp.View == em.View && emp.Digest == em.Digest)) {
							continue mpLoop
						}
					}
					quorum++
				}

				if quorum < 2*instance.f+1 {
					continue
				}

				quorum = 0
				// "A2. ∃f+1 messages m' ∈ S"
				for _, mp := range vset {
					// "∃<n,d',v'> ∈ m'.Q"
					for _, emp := range mp.Qset {
						if n == emp.SequenceNumber && emp.View >= em.View && emp.Digest == em.Digest {
							quorum++
						}
					}
				}

				if quorum < instance.f+1 {
					continue
				}

				// "then select the request with digest d for number n"
				msgList[n] = em.Digest

				continue nLoop
			}
		}

		quorum := uint(0)
		// "else if ∃2f+1 messages m ∈ S"
	nullLoop:
		for _, m := range vset {
			// "m.P has no entry"
			for _, em := range m.Pset {
				if em.SequenceNumber == n {
					continue nullLoop
				}
			}
			quorum++
		}

		if quorum >= 2*instance.f+1 {
			// "then select the null request for number n"
			msgList[n] = ""

			continue nLoop
		}

		return nil
	}

	return
}
<|endoftext|>


<commit_before>package app

import (
	"github.com/toonsevrin/simplechain/types"
	"time"
	"bytes"
	"fmt"
	"net/http"
	"encoding/json"
	"log"
	"io/ioutil"
)

type App struct {
	Blockchain    []types.Block
	Peers         map[string]*Peer //defined by url
	PeerAddresses map[string]bool //defined by ip
}

func (app *App) createAndAddNextBlock(data string) types.Block {
	previous := app.getLatestBlock()
	next := types.Block{Index: previous.Index, PreviousHash: previous.Hash, Timestamp: time.Now().Unix(), Hash: [32]byte{}, Data: data}
	next.Hash = *next.GenerateHash();
	app.Blockchain = append(app.Blockchain, next)
	return next
}
func (app *App) HasBlock(block types.Block) bool {
	return uint32(len(app.Blockchain)) > block.Index && app.Blockchain[block.Index].Hash == block.Hash
}

func (app *App) AddBlock(block types.Block) {
	app.Blockchain = append(app.Blockchain, block)
}
func (app *App) isValidNewBlock(block types.Block) bool {
	return block.IsValid() && bytes.Equal(block.PreviousHash[:], app.getLatestBlock().Hash[:]) && app.getLatestBlock().Index+1 == block.Index
}

func (app *App) getLatestBlock() *types.Block {
	return &app.Blockchain[len(app.Blockchain)-1]
}

func (app *App) isValidChain(chain []types.Block) bool {
	if *chain[0].GenerateHash() != app.Blockchain[0].Hash {
		return false
	}
	for i, block := range chain {
		if (i == 0) {
			continue
		}
		if block.PreviousHash != chain[i-1].Hash {
			return false
		}
		if !block.IsValid() {
			return false
		}
	}
	return true
}

func (app *App) pickLongestChain(newChain []types.Block) bool {
	if len(newChain) > len(app.Blockchain) && app.isValidChain(newChain) {
		fmt.Println("Received longer chain of length ", len(newChain))
		app.Blockchain = newChain
		return true
	} else {
		fmt.Println("Received invalid chain")
		return false
	}
}

func (app *App) broadcast(block types.Block) {
	marshalled, err := json.Marshal(block)
	if err != nil {
		panic(err.Error())
	}
	for _, peer := range app.Peers {
		response, err := http.NewRequest("POST", peer.getUrl()+"/addBlock", bytes.NewReader(marshalled))
		if err != nil {
			log.Println(err.Error())
			return
		}
		bytes, err := ioutil.ReadAll(response.Body)
		if err != nil {
			log.Println(err.Error())
			return
		}
		log.Println("Response when sharing new block: " + string(bytes))
	}
}
<commit_msg>Fixed block mining + broadcasting<commit_after>package app

import (
	"github.com/toonsevrin/simplechain/types"
	"time"
	"bytes"
	"fmt"
	"net/http"
	"encoding/json"
	"log"
	"io/ioutil"
)

type App struct {
	Blockchain    []types.Block
	Peers         map[string]*Peer //defined by url
	PeerAddresses map[string]bool //defined by ip
}

func (app *App) createAndAddNextBlock(data string) types.Block {
	previous := app.getLatestBlock()
	next := types.Block{Index: previous.Index + 1, PreviousHash: previous.Hash, Timestamp: time.Now().Unix(), Hash: [32]byte{}, Data: data}
	next.Hash = *next.GenerateHash();
	app.Blockchain = append(app.Blockchain, next)
	return next
}
func (app *App) HasBlock(block types.Block) bool {
	return uint32(len(app.Blockchain)) > block.Index && app.Blockchain[block.Index].Hash == block.Hash
}

func (app *App) AddBlock(block types.Block) {
	app.Blockchain = append(app.Blockchain, block)
}
func (app *App) isValidNewBlock(block types.Block) bool {
	return block.IsValid() && bytes.Equal(block.PreviousHash[:], app.getLatestBlock().Hash[:]) && app.getLatestBlock().Index+1 == block.Index
}

func (app *App) getLatestBlock() *types.Block {
	return &app.Blockchain[len(app.Blockchain)-1]
}

func (app *App) isValidChain(chain []types.Block) bool {
	if *chain[0].GenerateHash() != app.Blockchain[0].Hash {
		return false
	}
	for i, block := range chain {
		if (i == 0) {
			continue
		}
		if block.PreviousHash != chain[i-1].Hash {
			return false
		}
		if !block.IsValid() {
			return false
		}
	}
	return true
}

func (app *App) pickLongestChain(newChain []types.Block) bool {
	if len(newChain) > len(app.Blockchain) && app.isValidChain(newChain) {
		fmt.Println("Received longer chain of length ", len(newChain))
		app.Blockchain = newChain
		return true
	} else {
		fmt.Println("Received invalid chain")
		return false
	}
}

func (app *App) broadcast(block types.Block) {
	marshalled, err := json.Marshal(block)
	if err != nil {
		panic(err.Error())
	}
	for _, peer := range app.Peers {
		req, err := http.NewRequest("POST", peer.getUrl()+"/addBlock", bytes.NewReader(marshalled))
		if err != nil {
			log.Println(err.Error())
			return
		}
		response := req.Response

		bytes, err := ioutil.ReadAll(response.Body)
		if err != nil {
			log.Println(err.Error())
			return
		}
		log.Println("Response when sharing new block: " + string(bytes))
	}
}
<|endoftext|>


<commit_before>package mitlru

import (
	"container/list"
	"sync"
	"time"
)

// TTLRUCache stores up to capacity triples of keys, values, and timestamps,
// then begins removing the least recently read/created
// triples for every new triple beyond insert.
type TTLRUCache struct {
	capacity  int
	mapping   map[interface{}]*triple
	order     *list.List
	timeorder *list.List
	ttl       time.Duration
	lock      sync.RWMutex
}

type triple struct {
	key          interface{}
	val          interface{}
	expiration   time.Time
	timeelement  *list.Element
	orderelement *list.Element
}

// NewTTLRUCache returns a cache that starts removing elements after capacity
// elements have been added
func NewTTLRUCache(capacity int, ttl time.Duration) *TTLRUCache {
	rtn := &TTLRUCache{
		capacity:  capacity,
		mapping:   make(map[interface{}]*triple),
		order:     list.New(),
		timeorder: list.New(),
		ttl:       ttl,
	}
	go rtn.timer()
	return rtn
}

func (lru *TTLRUCache) timer() {
	for {
		lru.lock.Lock()
		now := time.Now()
		for lru.order.Len() > 0 {
			efront := lru.timeorder.Front()
			tfront := efront.Value.(*triple)
			if tfront.expiration.After(now) {
				break
			}
			lru.order.Remove(tfront.orderelement)
			lru.timeorder.Remove(efront)
			delete(lru.mapping, tfront.key)
		}
		lru.lock.Unlock()
		time.Sleep(1 * time.Second)
	}
}

// Purge clears everything from the cache
func (lru *TTLRUCache) Purge() {
	lru.lock.Lock()
	defer lru.lock.Unlock()

	lru.mapping = make(map[interface{}]*triple)
	lru.order = list.New()
	lru.timeorder = list.New()
}

// Add stores a key value pair in the cache. If the key was already present,
// it is moved to the front
func (lru *TTLRUCache) Add(key, val interface{}) {
	lru.AddWithExpire(key, val, time.Now().Add(lru.ttl))
}

// AddWithTTL stores a key value pair in the cache. If the key was already present,
// it is moved to the front.  Additionally, the given TTL is used (overriding the
// default ttl).
func (lru *TTLRUCache) AddWithTTL(key, val interface{}, ttl time.Duration) {
	lru.AddWithExpire(key, val, time.Now().Add(ttl))
}

// AddWithExpire stores a key value pair in the cache. If the key was already present,
// it is moved to the front.  Additionally, the given expiration time is used (overriding the
// default ttl).
func (lru *TTLRUCache) AddWithExpire(key, val interface{}, expiration time.Time) {
	lru.lock.Lock()
	defer lru.lock.Unlock()

	if etriple, ok := lru.mapping[key]; ok {
		etriple.val = val
		lru.order.MoveToFront(etriple.orderelement)
		lru.timeorder.MoveToBack(etriple.timeelement)
		etriple.expiration = expiration
		return
	}

	etriple := &triple{key, val, expiration, nil, nil}
	lru.mapping[key] = etriple
	etriple.orderelement = lru.order.PushFront(etriple)
	etriple.timeelement = lru.timeorder.PushBack(etriple)

	for lru.order.Len() > lru.capacity {
		element := lru.order.Back()
		etriple := element.Value.(*triple)
		lru.order.Remove(element)
		lru.timeorder.Remove(etriple.timeelement)
		delete(lru.mapping, etriple.key)
	}
}

// Get returns the value if the keyh exists in the cache
func (lru *TTLRUCache) Get(key interface{}) (value interface{}, foundInCache bool) {
	lru.lock.Lock()
	defer lru.lock.Unlock()

	if element, ok := lru.mapping[key]; ok {
		lru.order.MoveToFront(element.orderelement)
		return element.val, true
	}

	return nil, false
}

// Remove removes a key / value combination
func (lru *TTLRUCache) Remove(key interface{}) {
	lru.lock.Lock()
	defer lru.lock.Unlock()

	if element, ok := lru.mapping[key]; ok {
		lru.order.Remove(element.orderelement)
		lru.timeorder.Remove(element.timeelement)
		delete(lru.mapping, element.key)
	}
}

// Len returns the number of triples in the cache
func (lru *TTLRUCache) Len() int {
	lru.lock.RLock()
	defer lru.lock.RUnlock()
	return lru.order.Len()
}

// Capacity returns the capacity of the cache
func (lru *TTLRUCache) Capacity() int {
	return lru.capacity
}
<commit_msg>pooling for internal structure allocation<commit_after>package mitlru

import (
	"container/list"
	"sync"
	"time"
)

// TTLRUCache stores up to capacity triples of keys, values, and timestamps,
// then begins removing the least recently read/created
// triples for every new triple beyond insert.
type TTLRUCache struct {
	capacity   int
	mapping    map[interface{}]*triple
	order      *list.List
	timeorder  *list.List
	ttl        time.Duration
	lock       sync.RWMutex
	triplePool sync.Pool
}

type triple struct {
	key          interface{}
	val          interface{}
	expiration   time.Time
	timeelement  *list.Element
	orderelement *list.Element
}

func newTriple() interface{} {
	return &triple{}
}

// NewTTLRUCache returns a cache that starts removing elements after capacity
// elements have been added
func NewTTLRUCache(capacity int, ttl time.Duration) *TTLRUCache {
	rtn := &TTLRUCache{
		capacity:  capacity,
		mapping:   make(map[interface{}]*triple),
		order:     list.New(),
		timeorder: list.New(),
		ttl:       ttl,
	}
	rtn.triplePool.New = newTriple
	go rtn.timer()
	return rtn
}

func (lru *TTLRUCache) timer() {
	for {
		lru.lock.Lock()
		now := time.Now()
		for lru.order.Len() > 0 {
			efront := lru.timeorder.Front()
			tfront := efront.Value.(*triple)
			if tfront.expiration.After(now) {
				break
			}
			lru.order.Remove(tfront.orderelement)
			lru.timeorder.Remove(efront)
			delete(lru.mapping, tfront.key)
		}
		lru.lock.Unlock()
		time.Sleep(1 * time.Second)
	}
}

// Purge clears everything from the cache
func (lru *TTLRUCache) Purge() {
	lru.lock.Lock()
	defer lru.lock.Unlock()

	lru.mapping = make(map[interface{}]*triple)
	lru.order = list.New()
	lru.timeorder = list.New()
}

// Add stores a key value pair in the cache. If the key was already present,
// it is moved to the front
func (lru *TTLRUCache) Add(key, val interface{}) {
	lru.AddWithExpire(key, val, time.Now().Add(lru.ttl))
}

// AddWithTTL stores a key value pair in the cache. If the key was already present,
// it is moved to the front.  Additionally, the given TTL is used (overriding the
// default ttl).
func (lru *TTLRUCache) AddWithTTL(key, val interface{}, ttl time.Duration) {
	lru.AddWithExpire(key, val, time.Now().Add(ttl))
}

// AddWithExpire stores a key value pair in the cache. If the key was already present,
// it is moved to the front.  Additionally, the given expiration time is used (overriding the
// default ttl).
func (lru *TTLRUCache) AddWithExpire(key, val interface{}, expiration time.Time) {
	lru.lock.Lock()
	defer lru.lock.Unlock()

	if etriple, ok := lru.mapping[key]; ok {
		etriple.val = val
		lru.order.MoveToFront(etriple.orderelement)
		lru.timeorder.MoveToBack(etriple.timeelement)
		etriple.expiration = expiration
		return
	}

	etriple := lru.triplePool.Get().(*triple)
	etriple.key = key
	etriple.val = val
	etriple.expiration = expiration
	etriple.orderelement = lru.order.PushFront(etriple)
	etriple.timeelement = lru.timeorder.PushBack(etriple)
	lru.mapping[key] = etriple

	for lru.order.Len() > lru.capacity {
		element := lru.order.Back()
		etriple := element.Value.(*triple)
		lru.order.Remove(element)
		lru.timeorder.Remove(etriple.timeelement)
		delete(lru.mapping, etriple.key)
		lru.triplePool.Put(etriple)
	}
}

// Get returns the value if the keyh exists in the cache
func (lru *TTLRUCache) Get(key interface{}) (value interface{}, foundInCache bool) {
	lru.lock.Lock()
	defer lru.lock.Unlock()

	if element, ok := lru.mapping[key]; ok {
		lru.order.MoveToFront(element.orderelement)
		return element.val, true
	}

	return nil, false
}

// Remove removes a key / value combination
func (lru *TTLRUCache) Remove(key interface{}) {
	lru.lock.Lock()
	defer lru.lock.Unlock()

	if element, ok := lru.mapping[key]; ok {
		lru.order.Remove(element.orderelement)
		lru.timeorder.Remove(element.timeelement)
		delete(lru.mapping, element.key)
		lru.triplePool.Put(element)
	}
}

// Len returns the number of triples in the cache
func (lru *TTLRUCache) Len() int {
	lru.lock.RLock()
	defer lru.lock.RUnlock()
	return lru.order.Len()
}

// Capacity returns the capacity of the cache
func (lru *TTLRUCache) Capacity() int {
	return lru.capacity
}
<|endoftext|>


<commit_before>package gomarquise

import (
	"fmt"
	"unsafe"
)

// #include <anchor_stats.h>
// #include <stdint.h>
// #cgo LDFLAGS: -lanchor_stats
import "C"

type MarquiseContext struct {
	consumer   C.as_consumer
	connection C.as_connection
}

func newMarquiseWriteError(value string) error {
	return fmt.Errorf("libmarquise returned -1 whilst trying to write frame with value %v", value)
}

// zmqBroker is a string taking the form of a ZeroMQ URI. batchPeriod
// is the interval at which the worker thread will poll/empty the queue
// of messages.
//
// Wraps C functions from anchor_stats.h:
//
// - as_consumer_new
// - as_connect
func Dial(zmqBroker string, batchPeriod float64) (MarquiseContext, error) {
	context := new(MarquiseContext)
	broker := C.CString(zmqBroker)
	defer C.free(unsafe.Pointer(broker))
	interval := C.double(batchPeriod)
	context.consumer = C.as_consumer_new(broker, interval)
	context.connection = C.as_connect(context.consumer)
	return *context, nil
}

// Need to free tagFields and tagValues as they're malloced
func translateSource(source map[string]string) ([]*C.char, []*C.char, C.size_t) {
	nTags := len(source)
	tagFields := make([]*C.char, nTags)
	tagValues := make([]*C.char, nTags)
	idx := 0
	for field, value := range source {
		tagFields[idx] = C.CString(field)
		tagValues[idx] = C.CString(value)
		idx += 1
	}
	return tagFields, tagValues, C.size_t(nTags)
}

func (c MarquiseContext) WriteText(source map[string]string, data string, timestamp uint64) error {
	tagFields, tagValues, tagCount := translateSource(source)
	for idx, _ := range tagFields {
		defer C.free(unsafe.Pointer(tagFields[idx]))
		defer C.free(unsafe.Pointer(tagValues[idx]))
	}
	cFields := &tagFields[0]
	cValues := &tagValues[0]
	cStr := C.CString(data)
	defer C.free(unsafe.Pointer(cStr))
	cLen := C.size_t(len(data))
	cTimestamp := C.uint64_t(timestamp)
	writeResult := C.as_send_text(c.connection, cFields, cValues, tagCount, cStr, cLen, cTimestamp)
	if writeResult == -1 {
		return newMarquiseWriteError(data)
	}
	return nil
}

func (c MarquiseContext) WriteInt(source map[string]string, data int64, timestamp uint64) error {
	tagFields, tagValues, tagCount := translateSource(source)
	for idx, _ := range tagFields {
		defer C.free(unsafe.Pointer(tagFields[idx]))
		defer C.free(unsafe.Pointer(tagValues[idx]))
	}
	cFields := &tagFields[0]
	cValues := &tagValues[0]
	cInt := C.int64_t(data)
	cTimestamp := C.uint64_t(timestamp)
	writeResult := C.as_send_int(c.connection, cFields, cValues, tagCount, cInt, cTimestamp)
	if writeResult == -1 {
		return newMarquiseWriteError(fmt.Sprintf("%v", data))
	}
	return nil
}

func (c MarquiseContext) WriteReal(source map[string]string, data float64, timestamp uint64) error {
	tagFields, tagValues, tagCount := translateSource(source)
	for idx, _ := range tagFields {
		defer C.free(unsafe.Pointer(tagFields[idx]))
		defer C.free(unsafe.Pointer(tagValues[idx]))
	}
	cFields := &tagFields[0]
	cValues := &tagValues[0]
	cFloat := C.double(data)
	cTimestamp := C.uint64_t(timestamp)
	writeResult := C.as_send_real(c.connection, cFields, cValues, tagCount, cFloat, cTimestamp)
	if writeResult == -1 {
		return newMarquiseWriteError(fmt.Sprintf("%v", data))
	}
	return nil
}

func (c MarquiseContext) WriteCounter(source map[string]string, timestamp uint64) error {
	tagFields, tagValues, tagCount := translateSource(source)
	for idx, _ := range tagFields {
		defer C.free(unsafe.Pointer(tagFields[idx]))
		defer C.free(unsafe.Pointer(tagValues[idx]))
	}
	cFields := &tagFields[0]
	cValues := &tagValues[0]
	cTimestamp := C.uint64_t(timestamp)
	writeResult := C.as_send_counter(c.connection, cFields, cValues, tagCount, cTimestamp)
	if writeResult == -1 {
		return newMarquiseWriteError("<EMPTY>")
	}
	return nil
}

func (c MarquiseContext) WriteBinary(source map[string]string, data []byte, timestamp uint64) error {
	tagFields, tagValues, tagCount := translateSource(source)
	for idx, _ := range tagFields {
		defer C.free(unsafe.Pointer(tagFields[idx]))
		defer C.free(unsafe.Pointer(tagValues[idx]))
	}
	cFields := &tagFields[0]
	cValues := &tagValues[0]
	cTimestamp := C.uint64_t(timestamp)
	// FIXME: do less wrong
	buf := make([]C.uint8_t, len(data))
	for i, val := range data {
		buf[i] = C.uint8_t(val)
	}
	nBytes := C.size_t(len(data))
	writeResult := C.as_send_binary(c.connection, cFields, cValues, tagCount, &buf[0], nBytes, cTimestamp)
	if writeResult == -1 {
		return newMarquiseWriteError(fmt.Sprintf("%v", data))
	}
	return nil
}

<commit_msg>Build against the right library name<commit_after>package gomarquise

import (
	"fmt"
	"unsafe"
)

// #include <marquise.h>
// #include <stdint.h>
// #cgo LDFLAGS: -lmarquise
import "C"

type MarquiseContext struct {
	consumer   C.as_consumer
	connection C.as_connection
}

func newMarquiseWriteError(value string) error {
	return fmt.Errorf("libmarquise returned -1 whilst trying to write frame with value %v", value)
}

// zmqBroker is a string taking the form of a ZeroMQ URI. batchPeriod
// is the interval at which the worker thread will poll/empty the queue
// of messages.
//
// Wraps C functions from anchor_stats.h:
//
// - as_consumer_new
// - as_connect
func Dial(zmqBroker string, batchPeriod float64) (MarquiseContext, error) {
	context := new(MarquiseContext)
	broker := C.CString(zmqBroker)
	defer C.free(unsafe.Pointer(broker))
	interval := C.double(batchPeriod)
	context.consumer = C.as_consumer_new(broker, interval)
	context.connection = C.as_connect(context.consumer)
	return *context, nil
}

// Need to free tagFields and tagValues as they're malloced
func translateSource(source map[string]string) ([]*C.char, []*C.char, C.size_t) {
	nTags := len(source)
	tagFields := make([]*C.char, nTags)
	tagValues := make([]*C.char, nTags)
	idx := 0
	for field, value := range source {
		tagFields[idx] = C.CString(field)
		tagValues[idx] = C.CString(value)
		idx += 1
	}
	return tagFields, tagValues, C.size_t(nTags)
}

func (c MarquiseContext) WriteText(source map[string]string, data string, timestamp uint64) error {
	tagFields, tagValues, tagCount := translateSource(source)
	for idx, _ := range tagFields {
		defer C.free(unsafe.Pointer(tagFields[idx]))
		defer C.free(unsafe.Pointer(tagValues[idx]))
	}
	cFields := &tagFields[0]
	cValues := &tagValues[0]
	cStr := C.CString(data)
	defer C.free(unsafe.Pointer(cStr))
	cLen := C.size_t(len(data))
	cTimestamp := C.uint64_t(timestamp)
	writeResult := C.as_send_text(c.connection, cFields, cValues, tagCount, cStr, cLen, cTimestamp)
	if writeResult == -1 {
		return newMarquiseWriteError(data)
	}
	return nil
}

func (c MarquiseContext) WriteInt(source map[string]string, data int64, timestamp uint64) error {
	tagFields, tagValues, tagCount := translateSource(source)
	for idx, _ := range tagFields {
		defer C.free(unsafe.Pointer(tagFields[idx]))
		defer C.free(unsafe.Pointer(tagValues[idx]))
	}
	cFields := &tagFields[0]
	cValues := &tagValues[0]
	cInt := C.int64_t(data)
	cTimestamp := C.uint64_t(timestamp)
	writeResult := C.as_send_int(c.connection, cFields, cValues, tagCount, cInt, cTimestamp)
	if writeResult == -1 {
		return newMarquiseWriteError(fmt.Sprintf("%v", data))
	}
	return nil
}

func (c MarquiseContext) WriteReal(source map[string]string, data float64, timestamp uint64) error {
	tagFields, tagValues, tagCount := translateSource(source)
	for idx, _ := range tagFields {
		defer C.free(unsafe.Pointer(tagFields[idx]))
		defer C.free(unsafe.Pointer(tagValues[idx]))
	}
	cFields := &tagFields[0]
	cValues := &tagValues[0]
	cFloat := C.double(data)
	cTimestamp := C.uint64_t(timestamp)
	writeResult := C.as_send_real(c.connection, cFields, cValues, tagCount, cFloat, cTimestamp)
	if writeResult == -1 {
		return newMarquiseWriteError(fmt.Sprintf("%v", data))
	}
	return nil
}

func (c MarquiseContext) WriteCounter(source map[string]string, timestamp uint64) error {
	tagFields, tagValues, tagCount := translateSource(source)
	for idx, _ := range tagFields {
		defer C.free(unsafe.Pointer(tagFields[idx]))
		defer C.free(unsafe.Pointer(tagValues[idx]))
	}
	cFields := &tagFields[0]
	cValues := &tagValues[0]
	cTimestamp := C.uint64_t(timestamp)
	writeResult := C.as_send_counter(c.connection, cFields, cValues, tagCount, cTimestamp)
	if writeResult == -1 {
		return newMarquiseWriteError("<EMPTY>")
	}
	return nil
}

func (c MarquiseContext) WriteBinary(source map[string]string, data []byte, timestamp uint64) error {
	tagFields, tagValues, tagCount := translateSource(source)
	for idx, _ := range tagFields {
		defer C.free(unsafe.Pointer(tagFields[idx]))
		defer C.free(unsafe.Pointer(tagValues[idx]))
	}
	cFields := &tagFields[0]
	cValues := &tagValues[0]
	cTimestamp := C.uint64_t(timestamp)
	// FIXME: do less wrong
	buf := make([]C.uint8_t, len(data))
	for i, val := range data {
		buf[i] = C.uint8_t(val)
	}
	nBytes := C.size_t(len(data))
	writeResult := C.as_send_binary(c.connection, cFields, cValues, tagCount, &buf[0], nBytes, cTimestamp)
	if writeResult == -1 {
		return newMarquiseWriteError(fmt.Sprintf("%v", data))
	}
	return nil
}

<|endoftext|>


<commit_before>package atlas

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/sendgrid/rest"
)

var (
	allTypes = []string{
		"dns",
		"http",
		"ntp",
		"ping",
		"sslcert",
		"traceroute",
		"wifi",
	}
)

// ErrInvalidMeasurementType is a new error
var ErrInvalidMeasurementType = errors.New("invalid measurement type")

// ErrInvalidAPIKey is returned when the key is invalid
var ErrInvalidAPIKey = errors.New("invalid API key")

// -- private

// checkType verify that the type is valid
func checkType(d Definition) (valid bool) {
	valid = false
	for _, t := range allTypes {
		if d.Type == t {
			valid = true
			break
		}
	}
	return
}

// checkTypeAs is a shortcut
func checkTypeAs(d Definition, t string) bool {
	valid := checkType(d)
	return valid && d.Type == t
}

// checkAllTypesAs is a generalization of checkTypeAs
func checkAllTypesAs(dl []Definition, t string) (valid bool) {
	valid = true
	for _, d := range dl {
		if d.Type != t {
			valid = false
			break
		}
	}
	return
}

// measurementList is our main answer
type measurementList struct {
	Count    int
	Next     string
	Previous string
	Results  []Measurement
}

// fetch the given resource
func fetchOneMeasurementPage(MeasurementEP string, opts map[string]string) (raw *measurementList, err error) {
	hdrs := make(map[string]string)
	req := rest.Request{
		BaseURL:     MeasurementEP,
		Method:      rest.Get,
		Headers:     hdrs,
		QueryParams: opts,
	}

	r, err := rest.API(req)
	if err != nil {
		err = fmt.Errorf("err: %v - r:%v\n", err, r)
		return
	}

	raw = &measurementList{}
	err = json.Unmarshal([]byte(r.Body), raw)
	//log.Printf(">> rawlist=%+v r=%+v Next=|%s|", rawlist, r, rawlist.Next)
	return
}

// -- public

// GetMeasurement gets info for a single one
func GetMeasurement(id int) (m *Measurement, err error) {
	measurementEP := apiEndpoint + "/measurements"

	key, ok := HasAPIKey()

	// Add at least one option, the APIkey if present
	hdrs := make(map[string]string)
	opts := make(map[string]string)

	if ok {
		opts["key"] = key
	}

	req := rest.Request{
		BaseURL:     measurementEP + fmt.Sprintf("/%d", id),
		Method:      rest.Get,
		Headers:     hdrs,
		QueryParams: opts,
	}

	//log.Printf("req: %#v", req)
	r, err := rest.API(req)
	if err != nil {
		err = fmt.Errorf("err: %v - r:%v\n", err, r)
		return
	}

	m = &Measurement{}
	err = json.Unmarshal([]byte(r.Body), m)
	//log.Printf("json: %#v\n", m)
	return
}

// GetMeasurements gets info for a set
func GetMeasurements(opts map[string]string) (m []Measurement, err error) {
	measurementEP := apiEndpoint + "/measurements"

	key, ok := HasAPIKey()

	// Add APIKey if set
	if ok {
		opts["key"] = key
	}

	// First call
	rawlist, err := fetchOneMeasurementPage(measurementEP, opts)

	// Empty answer
	if rawlist.Count == 0 {
		return nil, fmt.Errorf("empty measurement list")
	}

	var res []Measurement

	res = append(res, rawlist.Results...)
	if rawlist.Next != "" {
		// We have pagination
		for pn := getPageNum(rawlist.Next); rawlist.Next != ""; pn = getPageNum(rawlist.Next) {
			opts["page"] = pn

			rawlist, err = fetchOneMeasurementPage(measurementEP, opts)
			if err != nil {
				return
			}

			res = append(res, rawlist.Results...)
		}
	}
	m = res
	return
}

// DNS creates a measurement
func DNS(d MeasurementRequest) (m *Measurement, err error) {
	// Check that all Definition.Type are the same and compliant
	if !checkAllTypesAs(d.Definitions, "dns") {
		err = ErrInvalidMeasurementType
		return
	}
	return
}

// HTTP creates a measurement
func HTTP(d MeasurementRequest) (m *Measurement, err error) {
	// Check that all Definition.Type are the same and compliant
	if !checkAllTypesAs(d.Definitions, "http") {
		err = ErrInvalidMeasurementType
		return
	}
	return
}

// NTP creates a measurement
func NTP(d MeasurementRequest) (m *Measurement, err error) {
	// Check that all Definition.Type are the same and compliant
	if !checkAllTypesAs(d.Definitions, "ntp") {
		err = ErrInvalidMeasurementType
		return
	}
	return
}

type pingResp struct {
	Measurements []int
}

type pingError struct {
	Error struct {
		Status int
		Code   int
		Detail string
		Title  string
	}
}

// Ping creates a measurement
func Ping(d MeasurementRequest) (m *pingResp, err error) {
	// Check that all Definition.Type are the same and compliant
	if !checkAllTypesAs(d.Definitions, "ping") {
		err = ErrInvalidMeasurementType
		return
	}

	pingEP := apiEndpoint + "/measurements/ping"

	// Add at least one option, the APIkey if present
	hdrs := make(map[string]string)
	opts := make(map[string]string)

	key, ok := HasAPIKey()
	if ok {
		opts["key"] = key
	} else {
		err = ErrInvalidAPIKey
		return
	}

	body, err := json.Marshal(d)
	if err != nil {
		return
	}

	req := rest.Request{
		BaseURL:     pingEP,
		Method:      rest.Post,
		Headers:     hdrs,
		QueryParams: opts,
		Body:        body,
	}

	resp, err := rest.API(req)

	m = &pingResp{}
	err = json.Unmarshal([]byte(resp.Body), m)
	//r, err := api.Res(base, &resp).Post(d)
	fmt.Printf("m: %v\nresp: %#v\nd: %v\n", m, string(resp.Body), d)
	if err != nil {
		err = fmt.Errorf("err: %v - m:%v\n", err, m)
		return
	}
	return
}

// SSLCert creates a measurement
func SSLCert(d MeasurementRequest) (m *Measurement, err error) {
	// Check that all Definition.Type are the same and compliant
	if !checkAllTypesAs(d.Definitions, "sslcert") {
		err = ErrInvalidMeasurementType
		return
	}
	return
}

// Traceroute creates a measurement
func Traceroute(d MeasurementRequest) (m *Measurement, err error) {
	// Check that all Definition.Type are the same and compliant
	if !checkAllTypesAs(d.Definitions, "traceroute") {
		err = ErrInvalidMeasurementType
		return
	}
	return
}

// Measurement-related methods

// Start is for starting a given measurement
func (m *Measurement) Start(id int) (err error) {
	return nil
}

// Stop is for stopping a given measurement
func (m *Measurement) Stop(id int) (err error) {
	return nil
}
<commit_msg>Add the DeleteMeasurement() method.<commit_after>package atlas

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/sendgrid/rest"
)

var (
	allTypes = []string{
		"dns",
		"http",
		"ntp",
		"ping",
		"sslcert",
		"traceroute",
		"wifi",
	}
)

// ErrInvalidMeasurementType is a new error
var ErrInvalidMeasurementType = errors.New("invalid measurement type")

// ErrInvalidAPIKey is returned when the key is invalid
var ErrInvalidAPIKey = errors.New("invalid API key")

// -- private

// checkType verify that the type is valid
func checkType(d Definition) (valid bool) {
	valid = false
	for _, t := range allTypes {
		if d.Type == t {
			valid = true
			break
		}
	}
	return
}

// checkTypeAs is a shortcut
func checkTypeAs(d Definition, t string) bool {
	valid := checkType(d)
	return valid && d.Type == t
}

// checkAllTypesAs is a generalization of checkTypeAs
func checkAllTypesAs(dl []Definition, t string) (valid bool) {
	valid = true
	for _, d := range dl {
		if d.Type != t {
			valid = false
			break
		}
	}
	return
}

// measurementList is our main answer
type measurementList struct {
	Count    int
	Next     string
	Previous string
	Results  []Measurement
}

// fetch the given resource
func fetchOneMeasurementPage(MeasurementEP string, opts map[string]string) (raw *measurementList, err error) {
	hdrs := make(map[string]string)
	req := rest.Request{
		BaseURL:     MeasurementEP,
		Method:      rest.Get,
		Headers:     hdrs,
		QueryParams: opts,
	}

	r, err := rest.API(req)
	if err != nil {
		err = fmt.Errorf("err: %v - r:%v\n", err, r)
		return
	}

	raw = &measurementList{}
	err = json.Unmarshal([]byte(r.Body), raw)
	//log.Printf(">> rawlist=%+v r=%+v Next=|%s|", rawlist, r, rawlist.Next)
	return
}

// -- public

// GetMeasurement gets info for a single one
func GetMeasurement(id int) (m *Measurement, err error) {
	measurementEP := apiEndpoint + "/measurements"

	key, ok := HasAPIKey()

	// Add at least one option, the APIkey if present
	hdrs := make(map[string]string)
	opts := make(map[string]string)

	if ok {
		opts["key"] = key
	}

	req := rest.Request{
		BaseURL:     measurementEP + fmt.Sprintf("/%d", id),
		Method:      rest.Get,
		Headers:     hdrs,
		QueryParams: opts,
	}

	//log.Printf("req: %#v", req)
	r, err := rest.API(req)
	if err != nil {
		err = fmt.Errorf("err: %v - r:%v\n", err, r)
		return
	}

	m = &Measurement{}
	err = json.Unmarshal([]byte(r.Body), m)
	//log.Printf("json: %#v\n", m)
	return
}

// DeleteMeasurement stops (not really deletes) a given measurement
func DeleteMeasurement(id int) (err error) {
	measurementEP := apiEndpoint + "/measurements"

	key, ok := HasAPIKey()

	// Add at least one option, the APIkey if present
	hdrs := make(map[string]string)
	opts := make(map[string]string)

	if ok {
		opts["key"] = key
	}

	req := rest.Request{
		BaseURL:     measurementEP + fmt.Sprintf("/%d", id),
		Method:      rest.Delete,
		Headers:     hdrs,
		QueryParams: opts,
	}

	//log.Printf("req: %#v", req)
	r, err := rest.API(req)
	if err != nil {
		err = fmt.Errorf("err: %v - r:%v\n", err, r)
	}
	return
}

// GetMeasurements gets info for a set
func GetMeasurements(opts map[string]string) (m []Measurement, err error) {
	measurementEP := apiEndpoint + "/measurements"

	key, ok := HasAPIKey()

	// Add APIKey if set
	if ok {
		opts["key"] = key
	}

	// First call
	rawlist, err := fetchOneMeasurementPage(measurementEP, opts)

	// Empty answer
	if rawlist.Count == 0 {
		return nil, fmt.Errorf("empty measurement list")
	}

	var res []Measurement

	res = append(res, rawlist.Results...)
	if rawlist.Next != "" {
		// We have pagination
		for pn := getPageNum(rawlist.Next); rawlist.Next != ""; pn = getPageNum(rawlist.Next) {
			opts["page"] = pn

			rawlist, err = fetchOneMeasurementPage(measurementEP, opts)
			if err != nil {
				return
			}

			res = append(res, rawlist.Results...)
		}
	}
	m = res
	return
}

// DNS creates a measurement
func DNS(d MeasurementRequest) (m *Measurement, err error) {
	// Check that all Definition.Type are the same and compliant
	if !checkAllTypesAs(d.Definitions, "dns") {
		err = ErrInvalidMeasurementType
		return
	}
	return
}

// HTTP creates a measurement
func HTTP(d MeasurementRequest) (m *Measurement, err error) {
	// Check that all Definition.Type are the same and compliant
	if !checkAllTypesAs(d.Definitions, "http") {
		err = ErrInvalidMeasurementType
		return
	}
	return
}

// NTP creates a measurement
func NTP(d MeasurementRequest) (m *Measurement, err error) {
	// Check that all Definition.Type are the same and compliant
	if !checkAllTypesAs(d.Definitions, "ntp") {
		err = ErrInvalidMeasurementType
		return
	}
	return
}

type pingResp struct {
	Measurements []int
}

type pingError struct {
	Error struct {
		Status int
		Code   int
		Detail string
		Title  string
	}
}

// Ping creates a measurement
func Ping(d MeasurementRequest) (m *pingResp, err error) {
	// Check that all Definition.Type are the same and compliant
	if !checkAllTypesAs(d.Definitions, "ping") {
		err = ErrInvalidMeasurementType
		return
	}

	pingEP := apiEndpoint + "/measurements/ping"

	// Add at least one option, the APIkey if present
	hdrs := make(map[string]string)
	opts := make(map[string]string)

	key, ok := HasAPIKey()
	if ok {
		opts["key"] = key
	} else {
		err = ErrInvalidAPIKey
		return
	}

	body, err := json.Marshal(d)
	if err != nil {
		return
	}

	req := rest.Request{
		BaseURL:     pingEP,
		Method:      rest.Post,
		Headers:     hdrs,
		QueryParams: opts,
		Body:        body,
	}

	resp, err := rest.API(req)

	m = &pingResp{}
	err = json.Unmarshal([]byte(resp.Body), m)
	//r, err := api.Res(base, &resp).Post(d)
	fmt.Printf("m: %v\nresp: %#v\nd: %v\n", m, string(resp.Body), d)
	if err != nil {
		err = fmt.Errorf("err: %v - m:%v\n", err, m)
		return
	}
	return
}

// SSLCert creates a measurement
func SSLCert(d MeasurementRequest) (m *Measurement, err error) {
	// Check that all Definition.Type are the same and compliant
	if !checkAllTypesAs(d.Definitions, "sslcert") {
		err = ErrInvalidMeasurementType
		return
	}
	return
}

// Traceroute creates a measurement
func Traceroute(d MeasurementRequest) (m *Measurement, err error) {
	// Check that all Definition.Type are the same and compliant
	if !checkAllTypesAs(d.Definitions, "traceroute") {
		err = ErrInvalidMeasurementType
		return
	}
	return
}

// Measurement-related methods

// Start is for starting a given measurement
func (m *Measurement) Start(id int) (err error) {
	return nil
}

// Stop is for stopping a given measurement
func (m *Measurement) Stop(id int) (err error) {
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"path/filepath"
	"regexp"
	"strings"
)

type Matcher interface {
	Matches(string) bool
}

type ExtMatcher string

func (e ExtMatcher) Matches(s string) bool {
	return strings.HasSuffix(s, string(e))
}

var numStartRegex = regexp.MustCompile(`\A\d+`)
var numEndRegex = regexp.MustCompile(`\d+\z`)

// If true, look from the start.
type NumberMatcher bool

func (n NumberMatcher) Matches(s string) bool {
	if n {
		return numStartRegex.MatchString(s)
	}
	ext := filepath.Ext(s)
	if ext != "" {
		i := strings.LastIndex(s, ext)
		s = s[:i]
	}
	return numEndRegex.MatchString(s)
}

type StringMatcher string

func (m StringMatcher) Matches(s string) bool {
	return strings.Contains(s, string(m))
}

type RegexMatcher regexp.Regexp

func NewRegexMatcher(s string) *RegexMatcher {
	buf := new(bytes.Buffer)
	for {
		i := strings.Index(s, "%")
		if i < 0 {
			break
		}

		buf.WriteString(s[:i])

		if len(s) == i+1 {
			buf.WriteString("%")
			break
		}

		switch s[i+1] {
		case 'd', 'D', 's', 'S', 'w', 'W':
			buf.Write([]byte{'\\', s[i+1]})
			s = s[i+2:]
			continue

		case '%':
			buf.WriteString("%")
			s = s[i+2:]
			continue

		default:
			complain("Error: Could not parse regex. " + s[i:i+2] + " is not a valid combination.")
		}
	}
	return (*RegexMatcher)(regexp.MustCompile(buf.String()))
}

func (r *RegexMatcher) Matches(s string) bool {
	return (*regexp.Regexp)(r).MatchString(s)
}
<commit_msg>Improved --regex-from handling<commit_after>package main

import (
	"bytes"
	"path/filepath"
	"regexp"
	"strings"
)

type Matcher interface {
	Matches(string) bool
}

type ExtMatcher string

func (e ExtMatcher) Matches(s string) bool {
	return strings.HasSuffix(s, string(e))
}

var numStartRegex = regexp.MustCompile(`\A\d+`)
var numEndRegex = regexp.MustCompile(`\d+\z`)

// If true, look from the start.
type NumberMatcher bool

func (n NumberMatcher) Matches(s string) bool {
	if n {
		return numStartRegex.MatchString(s)
	}
	ext := filepath.Ext(s)
	if ext != "" {
		i := strings.LastIndex(s, ext)
		s = s[:i]
	}
	return numEndRegex.MatchString(s)
}

type StringMatcher string

func (m StringMatcher) Matches(s string) bool {
	return strings.Contains(s, string(m))
}

type RegexMatcher regexp.Regexp

func NewRegexMatcher(s string) *RegexMatcher {
	buf := new(bytes.Buffer)
	for {
		i := strings.Index(s, "%")
		if i < 0 {
			break
		}

		buf.WriteString(s[:i])

		if len(s) == i+1 {
			buf.WriteString("%")
			break
		}

		switch s[i+1] {
		case 'd', 'D', 's', 'S', 'w', 'W', // Character classes.
			'A', 'b', 'B', 'z', // Position indicators.
			'a', 'f', 'n', 'r', 'v', '.', '*', '?': // Escape sequences.
			buf.Write([]byte{'\\', s[i+1]})
			s = s[i+2:]
			continue

		case '%':
			buf.WriteString("%")
			s = s[i+2:]
			continue

		default:
			complain("Error: Could not parse regex. " + s[i:i+2] + " is not a valid combination.")
		}
	}
	return (*RegexMatcher)(regexp.MustCompile(buf.String()))
}

func (r *RegexMatcher) Matches(s string) bool {
	return (*regexp.Regexp)(r).MatchString(s)
}
<|endoftext|>


<commit_before>// Copyright 2017 TiKV Project Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package checker

import (
	"bytes"
	"context"
	"time"

	"github.com/pingcap/log"
	"github.com/tikv/pd/pkg/cache"
	"github.com/tikv/pd/pkg/codec"
	"github.com/tikv/pd/pkg/errs"
	"github.com/tikv/pd/pkg/logutil"
	"github.com/tikv/pd/server/config"
	"github.com/tikv/pd/server/core"
	"github.com/tikv/pd/server/schedule"
	"github.com/tikv/pd/server/schedule/labeler"
	"github.com/tikv/pd/server/schedule/operator"
	"github.com/tikv/pd/server/schedule/opt"
	"github.com/tikv/pd/server/schedule/placement"
)

const maxTargetRegionSize = 500

// When a region has label `merge_option=deny`, skip merging the region.
// If label value is `allow` or other value, it will be treated as `allow`.
const (
	mergeOptionLabel     = "merge_option"
	mergeOptionValueDeny = "deny"
)

// MergeChecker ensures region to merge with adjacent region when size is small
type MergeChecker struct {
	PauseController
	cluster    opt.Cluster
	opts       *config.PersistOptions
	splitCache *cache.TTLUint64
	startTime  time.Time // it's used to judge whether server recently start.
}

// NewMergeChecker creates a merge checker.
func NewMergeChecker(ctx context.Context, cluster opt.Cluster) *MergeChecker {
	opts := cluster.GetOpts()
	splitCache := cache.NewIDTTL(ctx, time.Minute, opts.GetSplitMergeInterval())
	return &MergeChecker{
		cluster:    cluster,
		opts:       opts,
		splitCache: splitCache,
		startTime:  time.Now(),
	}
}

// GetType return MergeChecker's type
func (m *MergeChecker) GetType() string {
	return "merge-checker"
}

// RecordRegionSplit put the recently split region into cache. MergeChecker
// will skip check it for a while.
func (m *MergeChecker) RecordRegionSplit(regionIDs []uint64) {
	for _, regionID := range regionIDs {
		m.splitCache.PutWithTTL(regionID, nil, m.opts.GetSplitMergeInterval())
	}
}

// Check verifies a region's replicas, creating an Operator if need.
func (m *MergeChecker) Check(region *core.RegionInfo) []*operator.Operator {
	checkerCounter.WithLabelValues("merge_checker", "check").Inc()

	if m.IsPaused() {
		checkerCounter.WithLabelValues("merge_checker", "paused").Inc()
		return nil
	}

	expireTime := m.startTime.Add(m.opts.GetSplitMergeInterval())
	if time.Now().Before(expireTime) {
		checkerCounter.WithLabelValues("merge_checker", "recently-start").Inc()
		return nil
	}

	if m.splitCache.Exists(region.GetID()) {
		checkerCounter.WithLabelValues("merge_checker", "recently-split").Inc()
		return nil
	}

	// when pd just started, it will load region meta from region storage,
	// but the size for these loaded region info is 0
	// pd don't know the real size of one region until the first heartbeat of the region
	// thus here when size is 0, just skip.
	if region.GetApproximateSize() == 0 {
		checkerCounter.WithLabelValues("merge_checker", "skip").Inc()
		return nil
	}

	// region is not small enough
	if region.GetApproximateSize() > int64(m.opts.GetMaxMergeRegionSize()) ||
		region.GetApproximateKeys() > int64(m.opts.GetMaxMergeRegionKeys()) {
		checkerCounter.WithLabelValues("merge_checker", "no-need").Inc()
		return nil
	}

	// skip region has down peers or pending peers
	if !schedule.IsRegionHealthy(region) {
		checkerCounter.WithLabelValues("merge_checker", "special-peer").Inc()
		return nil
	}

	if !schedule.IsRegionReplicated(m.cluster, region) {
		checkerCounter.WithLabelValues("merge_checker", "abnormal-replica").Inc()
		return nil
	}

	// skip hot region
	if m.cluster.IsRegionHot(region) {
		checkerCounter.WithLabelValues("merge_checker", "hot-region").Inc()
		return nil
	}

	prev, next := m.cluster.GetAdjacentRegions(region)

	var target *core.RegionInfo
	if m.checkTarget(region, next) {
		target = next
	}
	if !m.opts.IsOneWayMergeEnabled() && m.checkTarget(region, prev) { // allow a region can be merged by two ways.
		if target == nil || prev.GetApproximateSize() < next.GetApproximateSize() { // pick smaller
			target = prev
		}
	}

	if target == nil {
		checkerCounter.WithLabelValues("merge_checker", "no-target").Inc()
		return nil
	}

	if target.GetApproximateSize() > maxTargetRegionSize {
		checkerCounter.WithLabelValues("merge_checker", "target-too-large").Inc()
		return nil
	}

	log.Debug("try to merge region",
		logutil.ZapRedactStringer("from", core.RegionToHexMeta(region.GetMeta())),
		logutil.ZapRedactStringer("to", core.RegionToHexMeta(target.GetMeta())))
	ops, err := operator.CreateMergeRegionOperator("merge-region", m.cluster, region, target, operator.OpMerge)
	if err != nil {
		log.Warn("create merge region operator failed", errs.ZapError(err))
		return nil
	}
	checkerCounter.WithLabelValues("merge_checker", "new-operator").Inc()
	if region.GetApproximateSize() > target.GetApproximateSize() ||
		region.GetApproximateKeys() > target.GetApproximateKeys() {
		checkerCounter.WithLabelValues("merge_checker", "larger-source").Inc()
	}
	return ops
}

func (m *MergeChecker) checkTarget(region, adjacent *core.RegionInfo) bool {
	return adjacent != nil && !m.splitCache.Exists(adjacent.GetID()) && !m.cluster.IsRegionHot(adjacent) &&
		AllowMerge(m.cluster, region, adjacent) && checkPeerStore(m.cluster, region, adjacent) &&
		schedule.IsRegionHealthy(adjacent) && schedule.IsRegionReplicated(m.cluster, adjacent)
}

// AllowMerge returns true if two regions can be merged according to the key type.
func AllowMerge(cluster opt.Cluster, region, adjacent *core.RegionInfo) bool {
	var start, end []byte
	if bytes.Equal(region.GetEndKey(), adjacent.GetStartKey()) && len(region.GetEndKey()) != 0 {
		start, end = region.GetStartKey(), adjacent.GetEndKey()
	} else if bytes.Equal(adjacent.GetEndKey(), region.GetStartKey()) && len(adjacent.GetEndKey()) != 0 {
		start, end = adjacent.GetStartKey(), region.GetEndKey()
	} else {
		return false
	}

	// The interface probe is used here to get the rule manager and region
	// labeler because AllowMerge is also used by the random merge scheduler,
	// where it is not easy to get references to concrete objects.
	// We can consider using dependency injection techniques to optimize in
	// the future.

	if cluster.GetOpts().IsPlacementRulesEnabled() {
		cl, ok := cluster.(interface{ GetRuleManager() *placement.RuleManager })
		if !ok || len(cl.GetRuleManager().GetSplitKeys(start, end)) > 0 {
			return false
		}
	}

	if cl, ok := cluster.(interface{ GetRegionLabeler() *labeler.RegionLabeler }); ok {
		l := cl.GetRegionLabeler()
		if len(l.GetSplitKeys(start, end)) > 0 {
			return false
		}
		if l.GetRegionLabel(region, mergeOptionLabel) == mergeOptionValueDeny || l.GetRegionLabel(adjacent, mergeOptionLabel) == mergeOptionValueDeny {
			return false
		}
	}

	policy := cluster.GetOpts().GetKeyType()
	switch policy {
	case core.Table:
		if cluster.GetOpts().IsCrossTableMergeEnabled() {
			return true
		}
		return isTableIDSame(region, adjacent)
	case core.Raw:
		return true
	case core.Txn:
		return true
	default:
		return isTableIDSame(region, adjacent)
	}
}

func isTableIDSame(region, adjacent *core.RegionInfo) bool {
	return codec.Key(region.GetStartKey()).TableID() == codec.Key(adjacent.GetStartKey()).TableID()
}

// Check whether there is a peer of the adjacent region on an offline store,
// while the source region has no peer on it. This is to prevent from bringing
// any other peer into an offline store to slow down the offline process.
func checkPeerStore(cluster opt.Cluster, region, adjacent *core.RegionInfo) bool {
	regionStoreIDs := region.GetStoreIds()
	for _, peer := range adjacent.GetPeers() {
		storeID := peer.GetStoreId()
		store := cluster.GetStore(storeID)
		if store == nil || store.IsOffline() {
			if _, ok := regionStoreIDs[storeID]; !ok {
				return false
			}
		}
	}
	return true
}
<commit_msg>checker: enhance the observability for no-target metrics (#4559)<commit_after>// Copyright 2017 TiKV Project Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package checker

import (
	"bytes"
	"context"
	"time"

	"github.com/pingcap/log"
	"github.com/tikv/pd/pkg/cache"
	"github.com/tikv/pd/pkg/codec"
	"github.com/tikv/pd/pkg/errs"
	"github.com/tikv/pd/pkg/logutil"
	"github.com/tikv/pd/server/config"
	"github.com/tikv/pd/server/core"
	"github.com/tikv/pd/server/schedule"
	"github.com/tikv/pd/server/schedule/labeler"
	"github.com/tikv/pd/server/schedule/operator"
	"github.com/tikv/pd/server/schedule/opt"
	"github.com/tikv/pd/server/schedule/placement"
)

const maxTargetRegionSize = 500

// When a region has label `merge_option=deny`, skip merging the region.
// If label value is `allow` or other value, it will be treated as `allow`.
const (
	mergeOptionLabel     = "merge_option"
	mergeOptionValueDeny = "deny"
)

// MergeChecker ensures region to merge with adjacent region when size is small
type MergeChecker struct {
	PauseController
	cluster    opt.Cluster
	opts       *config.PersistOptions
	splitCache *cache.TTLUint64
	startTime  time.Time // it's used to judge whether server recently start.
}

// NewMergeChecker creates a merge checker.
func NewMergeChecker(ctx context.Context, cluster opt.Cluster) *MergeChecker {
	opts := cluster.GetOpts()
	splitCache := cache.NewIDTTL(ctx, time.Minute, opts.GetSplitMergeInterval())
	return &MergeChecker{
		cluster:    cluster,
		opts:       opts,
		splitCache: splitCache,
		startTime:  time.Now(),
	}
}

// GetType return MergeChecker's type
func (m *MergeChecker) GetType() string {
	return "merge-checker"
}

// RecordRegionSplit put the recently split region into cache. MergeChecker
// will skip check it for a while.
func (m *MergeChecker) RecordRegionSplit(regionIDs []uint64) {
	for _, regionID := range regionIDs {
		m.splitCache.PutWithTTL(regionID, nil, m.opts.GetSplitMergeInterval())
	}
}

// Check verifies a region's replicas, creating an Operator if need.
func (m *MergeChecker) Check(region *core.RegionInfo) []*operator.Operator {
	checkerCounter.WithLabelValues("merge_checker", "check").Inc()

	if m.IsPaused() {
		checkerCounter.WithLabelValues("merge_checker", "paused").Inc()
		return nil
	}

	expireTime := m.startTime.Add(m.opts.GetSplitMergeInterval())
	if time.Now().Before(expireTime) {
		checkerCounter.WithLabelValues("merge_checker", "recently-start").Inc()
		return nil
	}

	if m.splitCache.Exists(region.GetID()) {
		checkerCounter.WithLabelValues("merge_checker", "recently-split").Inc()
		return nil
	}

	// when pd just started, it will load region meta from region storage,
	// but the size for these loaded region info is 0
	// pd don't know the real size of one region until the first heartbeat of the region
	// thus here when size is 0, just skip.
	if region.GetApproximateSize() == 0 {
		checkerCounter.WithLabelValues("merge_checker", "skip").Inc()
		return nil
	}

	// region is not small enough
	if region.GetApproximateSize() > int64(m.opts.GetMaxMergeRegionSize()) ||
		region.GetApproximateKeys() > int64(m.opts.GetMaxMergeRegionKeys()) {
		checkerCounter.WithLabelValues("merge_checker", "no-need").Inc()
		return nil
	}

	// skip region has down peers or pending peers
	if !schedule.IsRegionHealthy(region) {
		checkerCounter.WithLabelValues("merge_checker", "special-peer").Inc()
		return nil
	}

	if !schedule.IsRegionReplicated(m.cluster, region) {
		checkerCounter.WithLabelValues("merge_checker", "abnormal-replica").Inc()
		return nil
	}

	// skip hot region
	if m.cluster.IsRegionHot(region) {
		checkerCounter.WithLabelValues("merge_checker", "hot-region").Inc()
		return nil
	}

	prev, next := m.cluster.GetAdjacentRegions(region)

	var target *core.RegionInfo
	if m.checkTarget(region, next) {
		target = next
	}
	if !m.opts.IsOneWayMergeEnabled() && m.checkTarget(region, prev) { // allow a region can be merged by two ways.
		if target == nil || prev.GetApproximateSize() < next.GetApproximateSize() { // pick smaller
			target = prev
		}
	}

	if target == nil {
		checkerCounter.WithLabelValues("merge_checker", "no-target").Inc()
		return nil
	}

	if target.GetApproximateSize() > maxTargetRegionSize {
		checkerCounter.WithLabelValues("merge_checker", "target-too-large").Inc()
		return nil
	}

	log.Debug("try to merge region",
		logutil.ZapRedactStringer("from", core.RegionToHexMeta(region.GetMeta())),
		logutil.ZapRedactStringer("to", core.RegionToHexMeta(target.GetMeta())))
	ops, err := operator.CreateMergeRegionOperator("merge-region", m.cluster, region, target, operator.OpMerge)
	if err != nil {
		log.Warn("create merge region operator failed", errs.ZapError(err))
		return nil
	}
	checkerCounter.WithLabelValues("merge_checker", "new-operator").Inc()
	if region.GetApproximateSize() > target.GetApproximateSize() ||
		region.GetApproximateKeys() > target.GetApproximateKeys() {
		checkerCounter.WithLabelValues("merge_checker", "larger-source").Inc()
	}
	return ops
}

func (m *MergeChecker) checkTarget(region, adjacent *core.RegionInfo) bool {
	if adjacent == nil {
		checkerCounter.WithLabelValues("merge_checker", "adj-not-exist").Inc()
		return false
	}

	if m.splitCache.Exists(adjacent.GetID()) {
		checkerCounter.WithLabelValues("merge_checker", "adj-recently-split").Inc()
		return false
	}

	if m.cluster.IsRegionHot(adjacent) {
		checkerCounter.WithLabelValues("merge_checker", "adj-region-hot").Inc()
		return false
	}

	if !AllowMerge(m.cluster, region, adjacent) {
		checkerCounter.WithLabelValues("merge_checker", "adj-disallow-merge").Inc()
		return false
	}

	if !checkPeerStore(m.cluster, region, adjacent) {
		checkerCounter.WithLabelValues("merge_checker", "adj-abnormal-peerstore").Inc()
		return false
	}

	if !schedule.IsRegionHealthy(adjacent) {
		checkerCounter.WithLabelValues("merge_checker", "adj-special-peer").Inc()
		return false
	}

	if !schedule.IsRegionReplicated(m.cluster, adjacent) {
		checkerCounter.WithLabelValues("merge_checker", "adj-abnormal-replica").Inc()
		return false
	}

	return true
}

// AllowMerge returns true if two regions can be merged according to the key type.
func AllowMerge(cluster opt.Cluster, region, adjacent *core.RegionInfo) bool {
	var start, end []byte
	if bytes.Equal(region.GetEndKey(), adjacent.GetStartKey()) && len(region.GetEndKey()) != 0 {
		start, end = region.GetStartKey(), adjacent.GetEndKey()
	} else if bytes.Equal(adjacent.GetEndKey(), region.GetStartKey()) && len(adjacent.GetEndKey()) != 0 {
		start, end = adjacent.GetStartKey(), region.GetEndKey()
	} else {
		return false
	}

	// The interface probe is used here to get the rule manager and region
	// labeler because AllowMerge is also used by the random merge scheduler,
	// where it is not easy to get references to concrete objects.
	// We can consider using dependency injection techniques to optimize in
	// the future.

	if cluster.GetOpts().IsPlacementRulesEnabled() {
		cl, ok := cluster.(interface{ GetRuleManager() *placement.RuleManager })
		if !ok || len(cl.GetRuleManager().GetSplitKeys(start, end)) > 0 {
			return false
		}
	}

	if cl, ok := cluster.(interface{ GetRegionLabeler() *labeler.RegionLabeler }); ok {
		l := cl.GetRegionLabeler()
		if len(l.GetSplitKeys(start, end)) > 0 {
			return false
		}
		if l.GetRegionLabel(region, mergeOptionLabel) == mergeOptionValueDeny || l.GetRegionLabel(adjacent, mergeOptionLabel) == mergeOptionValueDeny {
			return false
		}
	}

	policy := cluster.GetOpts().GetKeyType()
	switch policy {
	case core.Table:
		if cluster.GetOpts().IsCrossTableMergeEnabled() {
			return true
		}
		return isTableIDSame(region, adjacent)
	case core.Raw:
		return true
	case core.Txn:
		return true
	default:
		return isTableIDSame(region, adjacent)
	}
}

func isTableIDSame(region, adjacent *core.RegionInfo) bool {
	return codec.Key(region.GetStartKey()).TableID() == codec.Key(adjacent.GetStartKey()).TableID()
}

// Check whether there is a peer of the adjacent region on an offline store,
// while the source region has no peer on it. This is to prevent from bringing
// any other peer into an offline store to slow down the offline process.
func checkPeerStore(cluster opt.Cluster, region, adjacent *core.RegionInfo) bool {
	regionStoreIDs := region.GetStoreIds()
	for _, peer := range adjacent.GetPeers() {
		storeID := peer.GetStoreId()
		store := cluster.GetStore(storeID)
		if store == nil || store.IsOffline() {
			if _, ok := regionStoreIDs[storeID]; !ok {
				return false
			}
		}
	}
	return true
}
<|endoftext|>