<commit_before>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tester

import (
	"regexp"
	"strings"

	"k8s.io/kops/pkg/apis/kops/util"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/utils"
)

const (
	skipRegexBase = "\\[Slow\\]|\\[Serial\\]|\\[Disruptive\\]|\\[Flaky\\]|\\[Feature:.+\\]|nfs|NFS|Gluster"
)

func (t *Tester) setSkipRegexFlag() error {
	if t.SkipRegex != "" {
		return nil
	}

	cluster, err := t.getKopsCluster()
	if err != nil {
		return err
	}
	k8sVersion, err := util.ParseKubernetesVersion(cluster.Spec.KubernetesVersion)
	if err != nil {
		return err
	}

	skipRegex := skipRegexBase

	networking := cluster.Spec.Networking
	switch {
	case networking.Kubenet != nil, networking.Canal != nil, networking.Weave != nil, networking.Cilium != nil:
		skipRegex += "|Services.*rejected.*endpoints"
	}
	if networking.Cilium != nil {
		// https://github.com/cilium/cilium/issues/10002
		skipRegex += "|TCP.CLOSE_WAIT"
		// https://github.com/cilium/cilium/issues/15361
		skipRegex += "|external.IP.is.not.assigned.to.a.node"
		// https://github.com/cilium/cilium/issues/14287
		skipRegex += "|same.port.number.but.different.protocols|same.hostPort.but.different.hostIP.and.protocol"
		if strings.Contains(cluster.Spec.KubernetesVersion, "v1.23") || strings.Contains(cluster.Spec.KubernetesVersion, "v1.24") || strings.Contains(cluster.Spec.KubernetesVersion, "v1.25") {
			// Reassess after https://github.com/kubernetes/kubernetes/pull/102643 is merged
			// ref:
			// https://github.com/kubernetes/kubernetes/issues/96717
			// https://github.com/cilium/cilium/issues/5719
			skipRegex += "|should.create.a.Pod.with.SCTP.HostPort"
		}
		// https://github.com/cilium/cilium/issues/18241
		skipRegex += "|Services.should.create.endpoints.for.unready.pods"
		skipRegex += "|Services.should.be.able.to.connect.to.terminating.and.unready.endpoints.if.PublishNotReadyAddresses.is.true"
	} else if networking.Kuberouter != nil {
		skipRegex += "|load-balancer|hairpin|affinity\\stimeout|service\\.kubernetes\\.io|CLOSE_WAIT"
	} else if networking.Kubenet != nil {
		skipRegex += "|Services.*affinity"
	}

	if cluster.Spec.LegacyCloudProvider == "gce" {
		// Firewall tests expect a specific format for cluster and control plane host names
		// which kOps does not match
		// ref: https://github.com/kubernetes/kubernetes/blob/1bd00776b5d78828a065b5c21e7003accc308a06/test/e2e/framework/providers/gce/firewall.go#L92-L100
		skipRegex += "|Firewall"
		// kube-dns tests are not skipped automatically if a cluster uses CoreDNS instead
		skipRegex += "|kube-dns"
		// this test assumes the cluster runs COS but kOps uses Ubuntu by default
		// ref: https://github.com/kubernetes/test-infra/pull/22190
		skipRegex += "|should.be.mountable.when.non-attachable"
		// The in-tree driver and its E2E tests use `topology.kubernetes.io/zone` but the CSI driver uses `topology.gke.io/zone`
		skipRegex += "|In-tree.Volumes.\\[Driver:.gcepd\\].*topology.should.provision.a.volume.and.schedule.a.pod.with.AllowedTopologies"
	}

	if k8sVersion.Minor <= 22 {
		// this tests assumes a custom config for containerd:
		// https://github.com/kubernetes/test-infra/blob/578d86a7be187214be6ccd60e6ea7317b51aeb15/jobs/e2e_node/containerd/config.toml#L19-L21
		// ref: https://github.com/kubernetes/kubernetes/pull/104803
		skipRegex += "|RuntimeClass.should.run"
	}

	if k8sVersion.Minor == 23 && cluster.Spec.LegacyCloudProvider == "aws" && utils.IsIPv6CIDR(cluster.Spec.NonMasqueradeCIDR) {
		// ref: https://github.com/kubernetes/kubernetes/pull/106992
		skipRegex += "|should.not.disrupt.a.cloud.load-balancer.s.connectivity.during.rollout"
	}

	if k8sVersion.Minor == 23 {
		// beta feature not enabled by default
		skipRegex += "|Topology.Hints"
	}

	if k8sVersion.Minor == 25 {
		// this test was being skipped automatically because it isn't applicable with CSIMigration=true which is default
		// but skipping logic has been changed and now the test is planned for removal
		// ref: https://github.com/kubernetes/kubernetes/pull/109649#issuecomment-1108574843
		skipRegex += "|should.verify.that.all.nodes.have.volume.limits"
	}

	if cluster.Spec.CloudConfig != nil && cluster.Spec.CloudConfig.AWSEBSCSIDriver != nil && fi.BoolValue(cluster.Spec.CloudConfig.AWSEBSCSIDriver.Enabled) {
		skipRegex += "|In-tree.Volumes.\\[Driver:.aws\\]"
	}

	// Ensure it is valid regex
	if _, err := regexp.Compile(skipRegex); err != nil {
		return err
	}
	t.SkipRegex = skipRegex
	return nil
}
<commit_msg>Fix more e2e skips<commit_after>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tester

import (
	"regexp"

	"k8s.io/kops/pkg/apis/kops/util"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/utils"
)

const (
	skipRegexBase = "\\[Slow\\]|\\[Serial\\]|\\[Disruptive\\]|\\[Flaky\\]|\\[Feature:.+\\]|nfs|NFS|Gluster"
)

func (t *Tester) setSkipRegexFlag() error {
	if t.SkipRegex != "" {
		return nil
	}

	cluster, err := t.getKopsCluster()
	if err != nil {
		return err
	}
	k8sVersion, err := util.ParseKubernetesVersion(cluster.Spec.KubernetesVersion)
	if err != nil {
		return err
	}

	skipRegex := skipRegexBase

	networking := cluster.Spec.Networking
	switch {
	case networking.Kubenet != nil, networking.Canal != nil, networking.Weave != nil, networking.Cilium != nil:
		skipRegex += "|Services.*rejected.*endpoints"
	}
	if networking.Cilium != nil {
		// https://github.com/cilium/cilium/issues/10002
		skipRegex += "|TCP.CLOSE_WAIT"
		// https://github.com/cilium/cilium/issues/15361
		skipRegex += "|external.IP.is.not.assigned.to.a.node"
		// https://github.com/cilium/cilium/issues/14287
		skipRegex += "|same.port.number.but.different.protocols|same.hostPort.but.different.hostIP.and.protocol"
		if k8sVersion.Minor >= 22 {
			// ref:
			// https://github.com/kubernetes/kubernetes/issues/96717
			// https://github.com/cilium/cilium/issues/5719
			skipRegex += "|should.create.a.Pod.with.SCTP.HostPort"
		}
		// https://github.com/cilium/cilium/issues/18241
		skipRegex += "|Services.should.create.endpoints.for.unready.pods"
		skipRegex += "|Services.should.be.able.to.connect.to.terminating.and.unready.endpoints.if.PublishNotReadyAddresses.is.true"
	} else if networking.Kuberouter != nil {
		skipRegex += "|load-balancer|hairpin|affinity\\stimeout|service\\.kubernetes\\.io|CLOSE_WAIT"
	} else if networking.Kubenet != nil {
		skipRegex += "|Services.*affinity"
	}

	if cluster.Spec.LegacyCloudProvider == "gce" {
		// Firewall tests expect a specific format for cluster and control plane host names
		// which kOps does not match
		// ref: https://github.com/kubernetes/kubernetes/blob/1bd00776b5d78828a065b5c21e7003accc308a06/test/e2e/framework/providers/gce/firewall.go#L92-L100
		skipRegex += "|Firewall"
		// kube-dns tests are not skipped automatically if a cluster uses CoreDNS instead
		skipRegex += "|kube-dns"
		// this test assumes the cluster runs COS but kOps uses Ubuntu by default
		// ref: https://github.com/kubernetes/test-infra/pull/22190
		skipRegex += "|should.be.mountable.when.non-attachable"
		// The in-tree driver and its E2E tests use `topology.kubernetes.io/zone` but the CSI driver uses `topology.gke.io/zone`
		skipRegex += "|In-tree.Volumes.\\[Driver:.gcepd\\].*topology.should.provision.a.volume.and.schedule.a.pod.with.AllowedTopologies"
	}

	if k8sVersion.Minor <= 23 {
		// this tests assumes a custom config for containerd:
		// https://github.com/kubernetes/test-infra/blob/578d86a7be187214be6ccd60e6ea7317b51aeb15/jobs/e2e_node/containerd/config.toml#L19-L21
		// ref: https://github.com/kubernetes/kubernetes/pull/104803
		skipRegex += "|RuntimeClass.should.run"
	}

	if k8sVersion.Minor == 23 && cluster.Spec.LegacyCloudProvider == "aws" && utils.IsIPv6CIDR(cluster.Spec.NonMasqueradeCIDR) {
		// ref: https://github.com/kubernetes/kubernetes/pull/106992
		skipRegex += "|should.not.disrupt.a.cloud.load-balancer.s.connectivity.during.rollout"
	}

	if k8sVersion.Minor == 23 {
		// beta feature not enabled by default
		skipRegex += "|Topology.Hints"
	}

	if k8sVersion.Minor >= 22 {
		// this test was being skipped automatically because it isn't applicable with CSIMigration=true which is default
		// but skipping logic has been changed and now the test is planned for removal
		// Should be skipped on all versions we enable CSI drivers on
		// ref: https://github.com/kubernetes/kubernetes/pull/109649#issuecomment-1108574843
		skipRegex += "|should.verify.that.all.nodes.have.volume.limits"
	}

	if cluster.Spec.CloudConfig != nil && cluster.Spec.CloudConfig.AWSEBSCSIDriver != nil && fi.BoolValue(cluster.Spec.CloudConfig.AWSEBSCSIDriver.Enabled) {
		skipRegex += "|In-tree.Volumes.\\[Driver:.aws\\]"
	}

	// Ensure it is valid regex
	if _, err := regexp.Compile(skipRegex); err != nil {
		return err
	}
	t.SkipRegex = skipRegex
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tester

import (
	"regexp"
	"strings"
)

const (
	skipRegexBase = "\\[Slow\\]|\\[Serial\\]|\\[Disruptive\\]|\\[Flaky\\]|\\[Feature:.+\\]|\\[HPA\\]|\\[Driver:.nfs\\]|Dashboard|RuntimeClass|RuntimeHandler"
)

func (t *Tester) setSkipRegexFlag() error {
	if t.SkipRegex != "" {
		return nil
	}

	cluster, err := t.getKopsCluster()
	if err != nil {
		return err
	}

	skipRegex := skipRegexBase

	networking := cluster.Spec.Networking
	switch {
	case networking.Kubenet != nil, networking.Canal != nil, networking.Weave != nil, networking.Cilium != nil:
		skipRegex += "|Services.*rejected.*endpoints"
	}
	if networking.Cilium != nil {
		// https://github.com/cilium/cilium/issues/10002
		skipRegex += "|TCP.CLOSE_WAIT"
		// https://github.com/cilium/cilium/issues/15361
		skipRegex += "|external.IP.is.not.assigned.to.a.node"
		// https://github.com/cilium/cilium/issues/14287
		skipRegex += "|same.port.number.but.different.protocols|same.hostPort.but.different.hostIP.and.protocol"
	} else if networking.Calico != nil {
		skipRegex += "|Services.*functioning.*NodePort"
	} else if networking.Kuberouter != nil {
		skipRegex += "|load-balancer|hairpin|affinity\\stimeout|service\\.kubernetes\\.io|CLOSE_WAIT"
	} else if networking.Kubenet != nil {
		skipRegex += "|Services.*affinity"
	}

	if strings.Contains(cluster.Spec.KubernetesVersion, "v1.22.") {
		// TODO(rifelpet): Remove once k8s tags has been created that include
		// https://github.com/kubernetes/kubernetes/pull/104061
		skipRegex += "|MetricsGrabber.should.grab.all.metrics.from.a.ControllerManager"
	}

	// Ensure it is valid regex
	if _, err := regexp.Compile(skipRegex); err != nil {
		return err
	}
	t.SkipRegex = skipRegex
	return nil
}
<commit_msg>Also skip MetricsGrabber tests on 1.23 version marker<commit_after>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tester

import (
	"regexp"
	"strings"
)

const (
	skipRegexBase = "\\[Slow\\]|\\[Serial\\]|\\[Disruptive\\]|\\[Flaky\\]|\\[Feature:.+\\]|\\[HPA\\]|\\[Driver:.nfs\\]|Dashboard|RuntimeClass|RuntimeHandler"
)

func (t *Tester) setSkipRegexFlag() error {
	if t.SkipRegex != "" {
		return nil
	}

	cluster, err := t.getKopsCluster()
	if err != nil {
		return err
	}

	skipRegex := skipRegexBase

	networking := cluster.Spec.Networking
	switch {
	case networking.Kubenet != nil, networking.Canal != nil, networking.Weave != nil, networking.Cilium != nil:
		skipRegex += "|Services.*rejected.*endpoints"
	}
	if networking.Cilium != nil {
		// https://github.com/cilium/cilium/issues/10002
		skipRegex += "|TCP.CLOSE_WAIT"
		// https://github.com/cilium/cilium/issues/15361
		skipRegex += "|external.IP.is.not.assigned.to.a.node"
		// https://github.com/cilium/cilium/issues/14287
		skipRegex += "|same.port.number.but.different.protocols|same.hostPort.but.different.hostIP.and.protocol"
	} else if networking.Calico != nil {
		skipRegex += "|Services.*functioning.*NodePort"
	} else if networking.Kuberouter != nil {
		skipRegex += "|load-balancer|hairpin|affinity\\stimeout|service\\.kubernetes\\.io|CLOSE_WAIT"
	} else if networking.Kubenet != nil {
		skipRegex += "|Services.*affinity"
	}

	if strings.Contains(cluster.Spec.KubernetesVersion, "v1.22.") || strings.Contains(cluster.Spec.KubernetesVersion, "v1.23.") {
		// TODO(rifelpet): Remove once k8s tags has been created that include
		// https://github.com/kubernetes/kubernetes/pull/104061
		skipRegex += "|MetricsGrabber.should.grab.all.metrics.from.a.ControllerManager"
	}

	// Ensure it is valid regex
	if _, err := regexp.Compile(skipRegex); err != nil {
		return err
	}
	t.SkipRegex = skipRegex
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build ignore

/*
The headscan command extracts comment headings from package files;
it is used to detect false positives which may require an adjustment
to the comment formatting heuristics in comment.go.

Usage: headscan [-root root_directory]

By default, the $GOROOT/src directory is scanned.
*/
package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/doc"
	"go/parser"
	"go/token"
	"io/fs"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
)

var (
	root    = flag.String("root", filepath.Join(runtime.GOROOT(), "src"), "root of filesystem tree to scan")
	verbose = flag.Bool("v", false, "verbose mode")
)

// ToHTML in comment.go assigns a (possibly blank) ID to each heading
var html_h = regexp.MustCompile(`<h3 id="[^"]*">`)

const html_endh = "
\n"

func isGoFile(fi fs.FileInfo) bool {
	return strings.HasSuffix(fi.Name(), ".go") &&
		!strings.HasSuffix(fi.Name(), "_test.go")
}

func appendHeadings(list []string, comment string) []string {
	var buf bytes.Buffer
	doc.ToHTML(&buf, comment, nil)
	for s := buf.String(); s != ""; {
		loc := html_h.FindStringIndex(s)
		if len(loc) == 0 {
			break
		}
		var inner string
		inner, s, _ = strings.Cut(s[loc[1]:], html_endh)
		list = append(list, inner)
	}
	return list
}

func main() {
	flag.Parse()
	fset := token.NewFileSet()
	nheadings := 0
	err := filepath.WalkDir(*root, func(path string, info fs.DirEntry, err error) error {
		if !info.IsDir() {
			return nil
		}
		pkgs, err := parser.ParseDir(fset, path, isGoFile, parser.ParseComments)
		if err != nil {
			if *verbose {
				fmt.Fprintln(os.Stderr, err)
			}
			return nil
		}
		for _, pkg := range pkgs {
			d := doc.New(pkg, path, doc.Mode(0))
			list := appendHeadings(nil, d.Doc)
			for _, d := range d.Consts {
				list = appendHeadings(list, d.Doc)
			}
			for _, d := range d.Types {
				list = appendHeadings(list, d.Doc)
			}
			for _, d := range d.Vars {
				list = appendHeadings(list, d.Doc)
			}
			for _, d := range d.Funcs {
				list = appendHeadings(list, d.Doc)
			}
			if len(list) > 0 {
				// directories may contain multiple packages;
				// print path and package name
				fmt.Printf("%s (package %s)\n", path, pkg.Name)
				for _, h := range list {
					fmt.Printf("\t%s\n", h)
				}
				nheadings += len(list)
			}
		}
		return nil
	})
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	fmt.Println(nheadings, "headings found")
}
<commit_msg>go/doc: using strings.Builder<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build ignore

/*
The headscan command extracts comment headings from package files;
it is used to detect false positives which may require an adjustment
to the comment formatting heuristics in comment.go.

Usage: headscan [-root root_directory]

By default, the $GOROOT/src directory is scanned.
*/
package main

import (
	"flag"
	"fmt"
	"go/doc"
	"go/parser"
	"go/token"
	"io/fs"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
)

var (
	root    = flag.String("root", filepath.Join(runtime.GOROOT(), "src"), "root of filesystem tree to scan")
	verbose = flag.Bool("v", false, "verbose mode")
)

// ToHTML in comment.go assigns a (possibly blank) ID to each heading
var html_h = regexp.MustCompile(`<h3 id="[^"]*">`)

const html_endh = "
\n"

func isGoFile(fi fs.FileInfo) bool {
	return strings.HasSuffix(fi.Name(), ".go") &&
		!strings.HasSuffix(fi.Name(), "_test.go")
}

func appendHeadings(list []string, comment string) []string {
	var buf strings.Builder
	doc.ToHTML(&buf, comment, nil)
	for s := buf.String(); s != ""; {
		loc := html_h.FindStringIndex(s)
		if len(loc) == 0 {
			break
		}
		var inner string
		inner, s, _ = strings.Cut(s[loc[1]:], html_endh)
		list = append(list, inner)
	}
	return list
}

func main() {
	flag.Parse()
	fset := token.NewFileSet()
	nheadings := 0
	err := filepath.WalkDir(*root, func(path string, info fs.DirEntry, err error) error {
		if !info.IsDir() {
			return nil
		}
		pkgs, err := parser.ParseDir(fset, path, isGoFile, parser.ParseComments)
		if err != nil {
			if *verbose {
				fmt.Fprintln(os.Stderr, err)
			}
			return nil
		}
		for _, pkg := range pkgs {
			d := doc.New(pkg, path, doc.Mode(0))
			list := appendHeadings(nil, d.Doc)
			for _, d := range d.Consts {
				list = appendHeadings(list, d.Doc)
			}
			for _, d := range d.Types {
				list = appendHeadings(list, d.Doc)
			}
			for _, d := range d.Vars {
				list = appendHeadings(list, d.Doc)
			}
			for _, d := range d.Funcs {
				list = appendHeadings(list, d.Doc)
			}
			if len(list) > 0 {
				// directories may contain multiple packages;
				// print path and package name
				fmt.Printf("%s (package %s)\n", path, pkg.Name)
				for _, h := range list {
					fmt.Printf("\t%s\n", h)
				}
				nheadings += len(list)
			}
		}
		return nil
	})
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	fmt.Println(nheadings, "headings found")
}
<|endoftext|>


<commit_before>// Copyright (c) Ilia Kravets, 2014-2016. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package efhsim

import (
	"io"
	"log"
	"net"

	"github.com/google/gopacket"
	"github.com/google/gopacket/layers"
	"github.com/google/gopacket/pcap"
	"github.com/ikravets/errs"

	"my/ev/channels"
	"my/ev/packet"
	"my/ev/packet/processor"
	"my/ev/sim"
)

type EfhSim struct {
	inputFileName    string
	inputPacketLimit int
	packetNum        int
	simu             sim.Sim
	observer         *sim.MuxObserver
}

func NewEfhSim(shallow bool) *EfhSim {
	s := &EfhSim{
		simu:     sim.NewSim(shallow),
		observer: sim.NewMuxObserver(),
	}
	return s
}

func (s *EfhSim) SetInput(fileName string, limit int) {
	s.inputFileName = fileName
	s.inputPacketLimit = limit
}
func (s *EfhSim) SubscribeFromReader(r io.Reader) error {
	return s.simu.Subscr().SubscribeFromReader(r)
}
func (s *EfhSim) SubscriptionsNum() int {
	return s.simu.Subscr().Num()
}
func (s *EfhSim) RegisterChannels(cc channels.Config) (err error) {
	defer errs.PassE(&err)
	s.simu.SessionsIgnoreSrc(true)
	for _, c := range cc.Addrs() {
		a, err := net.ResolveUDPAddr("udp", c)
		errs.CheckE(err)
		ipFlow := gopacket.NewFlow(layers.EndpointIPv4, nil, a.IP.To4())
		portBytes := []byte{byte(a.Port >> 8), byte(a.Port)}
		udpFlow := gopacket.NewFlow(layers.EndpointUDPPort, nil, portBytes)
		s.simu.Session([]gopacket.Flow{ipFlow, udpFlow})
	}
	return
}

func (s *EfhSim) AddLogger(logger sim.Observer) error {
	s.observer.AppendSlave(logger)
	return nil
}

func (s *EfhSim) AnalyzeInput() (err error) {
	defer errs.PassE(&err)
	handle, err := pcap.OpenOffline(s.inputFileName)
	errs.CheckE(err)
	defer handle.Close()
	pp := processor.NewProcessor()
	pp.LimitPacketNumber(s.inputPacketLimit)
	pp.SetObtainer(handle)
	pp.SetHandler(s)
	errs.CheckE(pp.ProcessAll())
	return
}

func (s *EfhSim) HandlePacket(packet packet.Packet) {
	s.packetNum++
	if s.packetNum%10000 == 0 {
		type Stats struct {
			Packets      int
			OrderDbStats sim.OrderDbStats
			Options      int
			Sessions     int
		}
		s := Stats{
			Packets:      s.packetNum,
			OrderDbStats: s.simu.OrderDb().Stats(),
			Options:      s.simu.Book().NumOptions(),
			Sessions:     len(s.simu.Sessions()),
		}
		log.Printf("%#v", s)
	}
}

func (s *EfhSim) HandleMessage(message packet.ApplicationMessage) {
	//log.Println(message.Layer())
	m := s.simu.NewMessage(message)
	s.observer.MessageArrived(m)
	ops := m.MessageOperations()
	for _, op := range ops {
		//log.Println(op)
		if op.CanAffect(sim.OA_ORDERS) {
			s.simu.OrderDb().ApplyOperation(op)
			s.observer.OperationAppliedToOrders(op)
		}
		if op.CanAffect(sim.OA_BOOKS) {
			s.observer.BeforeBookUpdate(s.simu.Book(), op)
			s.simu.Book().ApplyOperation(op)
			s.observer.AfterBookUpdate(s.simu.Book(), op)
		}
	}
}
<commit_msg>efhsim: support Options<commit_after>// Copyright (c) Ilia Kravets, 2014-2016. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package efhsim

import (
	"io"
	"log"
	"net"

	"github.com/google/gopacket"
	"github.com/google/gopacket/layers"
	"github.com/google/gopacket/pcap"
	"github.com/ikravets/errs"

	"my/ev/channels"
	"my/ev/packet"
	"my/ev/packet/processor"
	"my/ev/sim"
)

type EfhSim struct {
	inputFileName    string
	inputPacketLimit int
	packetNum        int
	simu             sim.Sim
	observer         *sim.MuxObserver
}

func NewEfhSim(shallow bool) *EfhSim {
	s := &EfhSim{
		simu:     sim.NewSim(shallow),
		observer: sim.NewMuxObserver(),
	}
	return s
}

func (s *EfhSim) SetInput(fileName string, limit int) {
	s.inputFileName = fileName
	s.inputPacketLimit = limit
}
func (s *EfhSim) SubscribeFromReader(r io.Reader) error {
	return s.simu.Subscr().SubscribeFromReader(r)
}
func (s *EfhSim) SubscriptionsNum() int {
	return s.simu.Subscr().Num()
}
func (s *EfhSim) RegisterChannels(cc channels.Config) (err error) {
	defer errs.PassE(&err)
	s.simu.SessionsIgnoreSrc(true)
	for _, c := range cc.Addrs() {
		a, err := net.ResolveUDPAddr("udp", c)
		errs.CheckE(err)
		ipFlow := gopacket.NewFlow(layers.EndpointIPv4, nil, a.IP.To4())
		portBytes := []byte{byte(a.Port >> 8), byte(a.Port)}
		udpFlow := gopacket.NewFlow(layers.EndpointUDPPort, nil, portBytes)
		s.simu.Session([]gopacket.Flow{ipFlow, udpFlow})
	}
	return
}

func (s *EfhSim) AddLogger(logger sim.Observer) error {
	s.observer.AppendSlave(logger)
	return nil
}

func (s *EfhSim) AnalyzeInput() (err error) {
	defer errs.PassE(&err)
	handle, err := pcap.OpenOffline(s.inputFileName)
	errs.CheckE(err)
	defer handle.Close()
	pp := processor.NewProcessor()
	pp.LimitPacketNumber(s.inputPacketLimit)
	pp.SetObtainer(handle)
	pp.SetHandler(s)
	errs.CheckE(pp.ProcessAll())
	return
}

func (s *EfhSim) HandlePacket(packet packet.Packet) {
	s.packetNum++
	if s.packetNum%10000 == 0 {
		type Stats struct {
			Packets      int
			OrderDbStats sim.OrderDbStats
			Options      int
			Sessions     int
		}
		s := Stats{
			Packets:      s.packetNum,
			OrderDbStats: s.simu.OrderDb().Stats(),
			Options:      s.simu.Book().NumOptions(),
			Sessions:     len(s.simu.Sessions()),
		}
		log.Printf("%#v", s)
	}
}

func (s *EfhSim) HandleMessage(message packet.ApplicationMessage) {
	//log.Println(message.Layer())
	m := s.simu.NewMessage(message)
	s.observer.MessageArrived(m)
	ops := m.MessageOperations()
	for _, op := range ops {
		//log.Println(op)
		if op.CanAffect(sim.OA_OPTIONS) {
			s.simu.Options().ApplyOperation(op)
		}
		if op.CanAffect(sim.OA_ORDERS) {
			s.simu.OrderDb().ApplyOperation(op)
			s.observer.OperationAppliedToOrders(op)
		}
		if op.CanAffect(sim.OA_BOOKS) {
			s.observer.BeforeBookUpdate(s.simu.Book(), op)
			s.simu.Book().ApplyOperation(op)
			s.observer.AfterBookUpdate(s.simu.Book(), op)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Koichi Shiraishi. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package pathutil

import (
	"go/build"
	"path/filepath"

	"github.com/pkg/errors"
)

// PackagePath returns the package full directory path estimated from the path p directory structure.
//  return "github.com/pkg/errors", nil
// TODO(zchee): duplicate function behavior of PackageID.
func PackagePath(dir string) (string, error) {
	dir = filepath.Clean(dir)

	savePkg := new(build.Package)
	for {
		// Get the current files package information
		pkg, err := build.Default.ImportDir(dir, build.IgnoreVendor)
		// noGoError := &build.NoGoError{Dir: dir}
		if _, ok := err.(*build.NoGoError); ok {
			// if err == noGoError {
			return savePkg.Dir, nil
		} else if err != nil {
			return "", errors.Wrap(err, pkgPathutil)
		}

		if pkg.IsCommand() {
			return pkg.Dir, nil
		} else if savePkg.Name != "" && pkg.Name != savePkg.Name {
			return savePkg.Dir, nil
		}

		if dir == "/" {
			return "", errors.Errorf("cannot find the package path from %s", dir)
		}

		// Save the current package name
		savePkg = pkg
		dir = filepath.Dir(dir)
	}
}

// PackageID returns the package ID estimated from the path p directory structure.
//  return "github.com/pkg/errors", nil
// TODO(zchee): duplicate function behavior of PackagePath.
func PackageID(dir string) (string, error) {
	savePkg := new(build.Package)
	for {
		// Get the current files package information
		pkg, err := build.Default.ImportDir(dir, build.IgnoreVendor)
		// noGoError := &build.NoGoError{Dir: dir}
		if _, ok := err.(*build.NoGoError); ok {
			// if err == noGoError {
			return savePkg.ImportPath, nil
		} else if err != nil {
			return "", errors.Wrap(err, pkgPathutil)
		}

		if savePkg.Name != "" && pkg.Name != savePkg.Name {
			return savePkg.ImportPath, nil
		} else if pkg.IsCommand() {
			return pkg.ImportPath, nil
		}

		if dir == "/" {
			return "", errors.Errorf("cannot find the package path from %s", dir)
		}

		// Save the current package name
		savePkg = pkg
		dir = filepath.Dir(dir)
	}
}
<commit_msg>patutil/PackagePath: fix check savePkg first and godoc comment<commit_after>// Copyright 2016 Koichi Shiraishi. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package pathutil

import (
	"go/build"
	"path/filepath"

	"github.com/pkg/errors"
)

// PackagePath returns the package full directory path estimated from the path p directory structure.
// return "/Users/zchee/go/src/github.com/pkg/errors", nil
// TODO(zchee): duplicate function behavior of PackageID.
func PackagePath(dir string) (string, error) {
	dir = filepath.Clean(dir)

	savePkg := new(build.Package)
	for {
		// Get the current files package information
		pkg, err := build.Default.ImportDir(dir, build.IgnoreVendor)
		// noGoError := &build.NoGoError{Dir: dir}
		if _, ok := err.(*build.NoGoError); ok {
			// if err == noGoError {
			return savePkg.Dir, nil
		} else if err != nil {
			return "", errors.Wrap(err, pkgPathutil)
		}

		if savePkg.Name != "" && pkg.Name != savePkg.Name {
			return savePkg.Dir, nil
		} else if pkg.IsCommand() {
			return pkg.Dir, nil
		}

		if dir == "/" {
			return "", errors.Errorf("cannot find the package path from %s", dir)
		}

		// Save the current package name
		savePkg = pkg
		dir = filepath.Dir(dir)
	}
}

// PackageID returns the package ID estimated from the path p directory structure.
//  return "github.com/pkg/errors", nil
// TODO(zchee): duplicate function behavior of PackagePath.
func PackageID(dir string) (string, error) {
	savePkg := new(build.Package)
	for {
		// Get the current files package information
		pkg, err := build.Default.ImportDir(dir, build.IgnoreVendor)
		// noGoError := &build.NoGoError{Dir: dir}
		if _, ok := err.(*build.NoGoError); ok {
			// if err == noGoError {
			return savePkg.ImportPath, nil
		} else if err != nil {
			return "", errors.Wrap(err, pkgPathutil)
		}

		if savePkg.Name != "" && pkg.Name != savePkg.Name {
			return savePkg.ImportPath, nil
		} else if pkg.IsCommand() {
			return pkg.ImportPath, nil
		}

		if dir == "/" {
			return "", errors.Errorf("cannot find the package path from %s", dir)
		}

		// Save the current package name
		savePkg = pkg
		dir = filepath.Dir(dir)
	}
}
<|endoftext|>


<commit_before>package configparser

import (
	"io"
	"os"
	"path/filepath"
	"strings"
)

var absPath string
var data map[string][]byte
var forceType = "auto"
var glue string

func Parse(path string, dataDest map[string][]byte, arrayGlue string) error {
	absPath, _ = filepath.Abs(path)
	data = dataDest
	glue = arrayGlue
	_, err := os.Stat(absPath)
	if err != nil {
		return err
	}
	err = filepath.Walk(absPath, walk)
	return err
}

func ParseAsJSON(path string, dataDest map[string][]byte, arrayGlue string) error {
	forceType = "json"
	return Parse(path, dataDest, arrayGlue)
}

func ParseAsYAML(path string, dataDest map[string][]byte, arrayGlue string) error {
	forceType = "yaml"
	return Parse(path, dataDest, arrayGlue)
}

func walk(path string, fstat os.FileInfo, err error) error {
	var fp *os.File
	if fstat.Mode().IsDir() {
		return nil
	} else {
		fp, err = os.Open(path)
	}
	if err != nil {
		return err
	}
	keyPrefix := strings.Split(
		// remove leading '/'
		strings.TrimPrefix(
			// remove the file extension
			strings.TrimSuffix(
				// remove the base path that was passed as -path
				strings.TrimPrefix(path, absPath),
				filepath.Ext(path)),
			string(os.PathSeparator)),
		string(os.PathSeparator))
	if keyPrefix[0] == "" {
		// remove blank token
		keyPrefix = []string{}
	}
	switch {
	case strings.HasSuffix(strings.ToLower(path), ".json") || forceType == "json":
		err := parseJson(fp, keyPrefix, glue)
		if err != nil {
			return err
		}
	case strings.HasSuffix(strings.ToLower(path), ".yml"):
		fallthrough
	case strings.HasSuffix(strings.ToLower(path), ".yaml") || forceType == "yaml":
		// yaml handling based on https://github.com/bronze1man/yaml2json
		yamlR, yamlW := io.Pipe()
		go func() {
			defer yamlW.Close()
			yamlToJson(fp, yamlW)
		}()
		err := parseJson(yamlR, keyPrefix, glue)
		if err != nil {
			return err
		}
	//case strings.HasSuffix(strings.ToLower(path), ".properties"):
	// TODO: .properties parsing
	//case strings.HasSuffix(strings.ToLower(path), ".ini"):
	// TODO: .ini parsing
	// filenames with no type, or .txt should be handled as raw data
	case ! strings.Contains(fstat.Name(), ".") || strings.HasSuffix(strings.ToLower(path), ".txt"):
		err := parseRaw(fp, keyPrefix, glue)
		if err != nil {
			return err
		}
	default:
	}
	return nil
}
<commit_msg>skip dotfiles/dirs<commit_after>package configparser

import (
	"io"
	"os"
	"path/filepath"
	"strings"
)

var absPath string
var data map[string][]byte
var forceType = "auto"
var glue string

func Parse(path string, dataDest map[string][]byte, arrayGlue string) error {
	absPath, _ = filepath.Abs(path)
	data = dataDest
	glue = arrayGlue
	_, err := os.Stat(absPath)
	if err != nil {
		return err
	}
	err = filepath.Walk(absPath, walk)
	return err
}

func ParseAsJSON(path string, dataDest map[string][]byte, arrayGlue string) error {
	forceType = "json"
	return Parse(path, dataDest, arrayGlue)
}

func ParseAsYAML(path string, dataDest map[string][]byte, arrayGlue string) error {
	forceType = "yaml"
	return Parse(path, dataDest, arrayGlue)
}

func walk(path string, fstat os.FileInfo, err error) error {
	// skip .git etc
	if fstat.IsDir() && strings.HasPrefix(fstat.Name(), ".") {
		return filepath.SkipDir
	}
	var fp *os.File
	if fstat.Mode().IsDir() {
		return nil
	} else {
		fp, err = os.Open(path)
	}
	if err != nil {
		return err
	}
	keyPrefix := strings.Split(
		// remove leading '/'
		strings.TrimPrefix(
			// remove the file extension
			strings.TrimSuffix(
				// remove the base path that was passed as -path
				strings.TrimPrefix(path, absPath),
				filepath.Ext(path)),
			string(os.PathSeparator)),
		string(os.PathSeparator))
	if keyPrefix[0] == "" {
		// remove blank token
		keyPrefix = []string{}
	}
	switch {
	// skip dotfiles
	case strings.HasPrefix(fstat.Name(), "."):
		return nil
	case strings.HasSuffix(strings.ToLower(path), ".json") || forceType == "json":
		err := parseJson(fp, keyPrefix, glue)
		if err != nil {
			return err
		}
	case strings.HasSuffix(strings.ToLower(path), ".yml"):
		fallthrough
	case strings.HasSuffix(strings.ToLower(path), ".yaml") || forceType == "yaml":
		// yaml handling based on https://github.com/bronze1man/yaml2json
		yamlR, yamlW := io.Pipe()
		go func() {
			defer yamlW.Close()
			yamlToJson(fp, yamlW)
		}()
		err := parseJson(yamlR, keyPrefix, glue)
		if err != nil {
			return err
		}
	//case strings.HasSuffix(strings.ToLower(path), ".properties"):
	// TODO: .properties parsing
	//case strings.HasSuffix(strings.ToLower(path), ".ini"):
	// TODO: .ini parsing
	// filenames with no type, or .txt should be handled as raw data
	case ! strings.Contains(fstat.Name(), ".") || strings.HasSuffix(strings.ToLower(path), ".txt"):
		err := parseRaw(fp, keyPrefix, glue)
		if err != nil {
			return err
		}
	default:
	}
	return nil
}
<|endoftext|>


<commit_before>package suite

type preAuthCodeInfo struct {
	Code      string `json:"pre_auth_code"`
	ExpiresIn int64  `json:"expires_in"`
}

type corpTokenInfo struct {
	Token     string `json:"access_token"`
	ExpiresIn int64  `json:"expires_in"`
}

// Corporation 用于表示授权方企业信息
type Corporation struct {
	ID            string `json:"corpid"`
	Name          string `json:"corp_name"`
	Type          string `json:"corp_type"`
	RoundLogoURI  string `json:"corp_round_logo_url"`
	SquareLogoURI string `json:"corp_square_logo_url"`
	UserMax       int    `json:"corp_user_max"`
	AgentMax      int    `json:"corp_agent_max"`
	QRCode        string `json:"corp_wxqrcode"`
}

// Agent 用于表示应用基本信息
type Agent struct {
	ID                   int64  `json:"agentid"`
	Name                 string `json:"name"`
	RoundLogoURI         string `json:"round_logo_url"`
	SquareLogoURI        string `json:"square_logo_url"`
	Description          string `json:"description"`
	RedirectDomain       string `json:"redirect_domain"`
	RedirectLocationFlag int64  `json:"report_location_flag"`
	IsReportUser         int64  `json:"isreportuser"`
	IsReportEnter        int64  `json:"isreportenter"`
}

type authorizedAgent struct {
	Agent
	AppID    int64    `json:"appid"`
	APIGroup []string `json:"api_group"`
}

type authorizedDepartment struct {
	ID       int64  `json:"id"`
	Name     string `json:"name"`
	ParentID int64  `json:"parentid"`
	Writable string `json:"writable"`
}

// AuthInfo 表示授权基本信息
type AuthInfo struct {
	Agent      []*authorizedAgent      `json:"agent"`
	Department []*authorizedDepartment `json:"department"`
}

// PermanentCodeInfo 代表获取企业号永久授权码时的响应信息
type PermanentCodeInfo struct {
	AccessToken   string       `json:"access_token"`
	ExpiresIn     int64        `json:"expires_in"`
	PermanentCode string       `json:"permanent_code"`
	AuthCorpInfo  *Corporation `json:"auth_corp_info"`
	AuthInfo      *AuthInfo    `json:"auth_info"`
}

type operator struct {
	Email  string `json:"email"`
	Mobile string `json:"mobile"`
}

// CorpAuthInfo 代表企业号的授权信息
type CorpAuthInfo struct {
	AuthCorpInfo *Corporation `json:"auth_corp_info"`
	AuthInfo     *AuthInfo    `json:"auth_info"`
	AuthUserInfo *operator    `json:"auth_user_info"`
}

type allowUser struct {
	UserID string `json:"userid"`
	Status string `json:"status"`
}

type allowUsers struct {
	User []*allowUser `json:"user"`
}

type allowPartys struct {
	PartyID []int64 `json:"partyid"`
}

type allowTags struct {
	TagID []int64 `json:"tagid"`
}

// CorpAgent 用于表示授权方企业号某个应用的基本信息
type CorpAgent struct {
	Agent
	AllowUsers  *allowUsers  `json:"allow_userinfos"`
	AllowPartys *allowPartys `json:"allow_partys"`
	AllowTags   *allowTags   `json:"allow_tags"`
	Close       int64        `json:"close"`
}

// AgentEditInfo 代表设置授权方企业号某个应用时的应用信息
type AgentEditInfo struct {
	Agent
	LogoMediaID string `json:"logo_mediaid"`
}

// RecvSuiteTicket 用于记录应用套件 ticket 的被动响应结果
type RecvSuiteTicket struct {
	SuiteId     string
	InfoType    string
	TimeStamp   float64
	SuiteTicket string
}

// RecvSuiteAuth 用于记录应用套件授权变更和授权撤销的被动响应结果
type RecvSuiteAuth struct {
	SuiteId    string
	InfoType   string
	TimeStamp  float64
	AuthCorpId string
}
<commit_msg>调整部门 struct 中属性的类型<commit_after>package suite

type preAuthCodeInfo struct {
	Code      string `json:"pre_auth_code"`
	ExpiresIn int64  `json:"expires_in"`
}

type corpTokenInfo struct {
	Token     string `json:"access_token"`
	ExpiresIn int64  `json:"expires_in"`
}

// Corporation 用于表示授权方企业信息
type Corporation struct {
	ID            string `json:"corpid"`
	Name          string `json:"corp_name"`
	Type          string `json:"corp_type"`
	RoundLogoURI  string `json:"corp_round_logo_url"`
	SquareLogoURI string `json:"corp_square_logo_url"`
	UserMax       int    `json:"corp_user_max"`
	AgentMax      int    `json:"corp_agent_max"`
	QRCode        string `json:"corp_wxqrcode"`
}

// Agent 用于表示应用基本信息
type Agent struct {
	ID                   int64  `json:"agentid"`
	Name                 string `json:"name"`
	RoundLogoURI         string `json:"round_logo_url"`
	SquareLogoURI        string `json:"square_logo_url"`
	Description          string `json:"description"`
	RedirectDomain       string `json:"redirect_domain"`
	RedirectLocationFlag int64  `json:"report_location_flag"`
	IsReportUser         int64  `json:"isreportuser"`
	IsReportEnter        int64  `json:"isreportenter"`
}

type authorizedAgent struct {
	Agent
	AppID    int64    `json:"appid"`
	APIGroup []string `json:"api_group"`
}

type authorizedDepartment struct {
	ID       int64  `json:"id"`
	Name     string `json:"name"`
	ParentID int64  `json:"parentid"`
	Writable bool   `json:"writable"`
}

// AuthInfo 表示授权基本信息
type AuthInfo struct {
	Agent      []*authorizedAgent      `json:"agent"`
	Department []*authorizedDepartment `json:"department"`
}

// PermanentCodeInfo 代表获取企业号永久授权码时的响应信息
type PermanentCodeInfo struct {
	AccessToken   string       `json:"access_token"`
	ExpiresIn     int64        `json:"expires_in"`
	PermanentCode string       `json:"permanent_code"`
	AuthCorpInfo  *Corporation `json:"auth_corp_info"`
	AuthInfo      *AuthInfo    `json:"auth_info"`
}

type operator struct {
	Email  string `json:"email"`
	Mobile string `json:"mobile"`
}

// CorpAuthInfo 代表企业号的授权信息
type CorpAuthInfo struct {
	AuthCorpInfo *Corporation `json:"auth_corp_info"`
	AuthInfo     *AuthInfo    `json:"auth_info"`
	AuthUserInfo *operator    `json:"auth_user_info"`
}

type allowUser struct {
	UserID string `json:"userid"`
	Status string `json:"status"`
}

type allowUsers struct {
	User []*allowUser `json:"user"`
}

type allowPartys struct {
	PartyID []int64 `json:"partyid"`
}

type allowTags struct {
	TagID []int64 `json:"tagid"`
}

// CorpAgent 用于表示授权方企业号某个应用的基本信息
type CorpAgent struct {
	Agent
	AllowUsers  *allowUsers  `json:"allow_userinfos"`
	AllowPartys *allowPartys `json:"allow_partys"`
	AllowTags   *allowTags   `json:"allow_tags"`
	Close       int64        `json:"close"`
}

// AgentEditInfo 代表设置授权方企业号某个应用时的应用信息
type AgentEditInfo struct {
	Agent
	LogoMediaID string `json:"logo_mediaid"`
}

// RecvSuiteTicket 用于记录应用套件 ticket 的被动响应结果
type RecvSuiteTicket struct {
	SuiteId     string
	InfoType    string
	TimeStamp   float64
	SuiteTicket string
}

// RecvSuiteAuth 用于记录应用套件授权变更和授权撤销的被动响应结果
type RecvSuiteAuth struct {
	SuiteId    string
	InfoType   string
	TimeStamp  float64
	AuthCorpId string
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"math/rand"
	"os"
	"strconv"
	"time"

	"golang.org/x/net/context"

	log "github.com/Sirupsen/logrus"
	etcd "github.com/coreos/etcd/client"

	"lib/etcdclient"
	pb "lib/proto/snowflake"
)

const (
	service      = "[SNOWFLAKE]"
	envMachineID = "MACHINE_ID" // specific machine id
	path         = "/seqs/"
	uuidKey      = "/seqs/snowflake-uuid"
	backoff      = 100  // max backoff delay millisecond
	concurrent   = 128  // max concurrent connections to etcd
	uuidQueue    = 1024 // uuid process queue
)

const (
	tsMask        = 0x1FFFFFFFFFF // 41bit
	snMask        = 0xFFF         // 12bit
	machineIDMask = 0x3FF         // 10bit
)

type server struct {
	machineID  uint64 // 10-bit machine append
	clientPool chan etcd.KeysAPI
	procCh     chan chan uint64
}

func (p *server) init() {
	p.clientPool = make(chan etcd.KeysAPI, concurrent)
	p.procCh = make(chan chan uint64, uuidQueue)

	// init client pool
	for i := 0; i < concurrent; i++ {
		p.clientPool <- etcdclient.KeysAPI()
	}

	// check if user specified machine id is set
	if env := os.Getenv(envMachineID); env != "" {
		id, err := strconv.Atoi(env)
		if err != nil {
			log.Panic(err)
			os.Exit(-1)
		}

		p.machineID = (uint64(id) & machineIDMask) << 12
		log.Info("machine id specified:", id)
	} else {
		p.initMachineID()
	}

	go p.uuidTask()
}

func (p *server) initMachineID() {
	client := <-p.clientPool
	defer func() { p.clientPool <- client }()

	for {
		// get the key
		resp, err := client.Get(context.Background(), uuidKey, nil)
		if err != nil {
			log.Panic(err)
			os.Exit(-1)
		}

		// get preValue & preIndex
		prevValue, err := strconv.Atoi(resp.Node.Value)
		if err != nil {
			log.Panic(err)
			os.Exit(-1)
		}
		prevIndex := resp.Node.ModifiedIndex

		// compareAndSwap
		resp, err = client.Set(context.Background(), uuidKey, fmt.Sprint(prevValue+1), &etcd.SetOptions{PrevIndex: prevIndex})
		if err != nil {
			casDelay()
			continue
		}

		// record serial number of this service, already shifted
		p.machineID = (uint64(prevValue+1) & machineIDMask) << 12
		return
	}
}

// Next is get next value of a key, like auto-incrememt in mysql
func (p *server) Next(ctx context.Context, in *pb.Snowflake_Key) (*pb.Snowflake_Value, error) {
	client := <-p.clientPool
	defer func() { p.clientPool <- client }()
	key := path + in.Name
	for {
		// get the key
		resp, err := client.Get(context.Background(), key, nil)
		if err != nil {
			log.Error(err)
			return nil, errors.New("Key not exists, need to create first")
		}

		// get prevValue & prevIndex
		prevValue, err := strconv.Atoi(resp.Node.Value)
		if err != nil {
			log.Error(err)
			return nil, errors.New("marlformed value")
		}
		prevIndex := resp.Node.ModifiedIndex

		// compareAndSwap
		resp, err = client.Set(context.Background(), key, fmt.Sprint(prevValue+1), &etcd.SetOptions{PrevIndex: prevIndex})
		if err != nil {
			casDelay()
			continue
		}
		return &pb.Snowflake_Value{Value: int64(prevValue + 1)}, nil
	}
}

// GetUUID is generate an unique uuid
func (p *server) GetUUID(context.Context, *pb.Snowflake_NullRequest) (*pb.Snowflake_UUID, error) {
	req := make(chan uint64, 1)
	p.procCh <- req
	return &pb.Snowflake_UUID{Uuid: <-req}, nil
}

// uuid generator
func (p *server) uuidTask() {
	var sn uint64    // 12-bit serial no
	var lastts int64 // last timestamp
	for {
		ret := <-p.procCh
		// get a correct serial number
		t := ts()
		// clock shift backward
		if t < lastts {
			log.Error("clock shift happened, waiting until the clock moving to the next millisecond.")
			t = p.waitMs(lastts)
		}

		// same millisecond
		if lastts == t {
			sn = (sn + 1) & snMask
			// serial number overflows, wait until next ms
			if sn == 0 {
				t = p.waitMs(lastts)
			}
		} else { // new millsecond, reset serial number to 0
			sn = 0
		}
		// remember last timestamp
		lastts = t

		// generate uuid, format:
		//
		// 0		0.................0		0..............0	0........0
		// 1-bit	41bit timestamp			10bit machine-id	12bit sn
		var uuid uint64
		uuid |= (uint64(t) & tsMask) << 22
		uuid |= p.machineID
		uuid |= sn
		ret <- uuid
	}
}

// waitMs will spin wait till next millisecond
func (p *server) waitMs(lastts int64) int64 {
	t := ts()
	for t <= lastts {
		t = ts()
	}
	return t
}

// random delay
func casDelay() {
	<-time.After(time.Duration(rand.Int63n(backoff)) * time.Millisecond)
}

// get timestamp
func ts() int64 {
	return time.Now().UnixNano() / int64(time.Millisecond)
}
<commit_msg>upd snowflake<commit_after>package main

import (
	"errors"
	"fmt"
	"math/rand"
	"os"
	"strconv"
	"time"

	log "github.com/Sirupsen/logrus"
	"golang.org/x/net/context"

	pb "lib/proto/snowflake"
	sp "lib/services"
	"lib/utils"
)

const (
	service      = "[SNOWFLAKE]"
	envMachineID = "MACHINE_ID" // specific machine id
	path         = "/seqs/"
	uuidKey      = "/seqs/snowflake-uuid"
	backoff      = 100  // max backoff delay millisecond
	concurrent   = 128  // max concurrent connections
	uuidQueue    = 1024 // uuid process queue
)

const (
	tsMask        = 0x1FFFFFFFFFF // 41bit
	snMask        = 0xFFF         // 12bit
	machineIDMask = 0x3FF         // 10bit
)

type server struct {
	machineID  uint64 // 10-bit machine append
	clientPool chan *sp.ConsulKV
	procCh     chan chan uint64
}

func (p *server) init() {
	p.clientPool = make(chan *sp.ConsulKV, concurrent)
	p.procCh = make(chan chan uint64, uuidQueue)

	// init client pool
	for i := 0; i < concurrent; i++ {
		p.clientPool <- sp.NewDefaultKVAPI()
	}

	// check if user specified machine id is set
	if env := os.Getenv(envMachineID); env != "" {
		id, err := strconv.Atoi(env)
		if err != nil {
			log.Panic(err)
			os.Exit(-1)
		}

		p.machineID = (uint64(id) & machineIDMask) << 12
		log.Info("machine id specified:", id)
	} else {
		p.initMachineID()
	}

	go p.uuidTask()
}

func (p *server) initMachineID() {
	client := <-p.clientPool
	defer func() { p.clientPool <- client }()

	for {
		// get the key
		kvpair, _, err := client.GetKVPair(uuidKey, nil)
		if err != nil {
			log.Panic(err)
			os.Exit(-1)
		}

		// get preValue & preIndex
		prevValue, err := strconv.Atoi(utils.Bytes2Str(kvpair.Value))
		if err != nil {
			log.Panic(err)
			os.Exit(-1)
		}
		prevIndex := kvpair.ModifyIndex

		// compareAndSwap

		_, _, err = client.CAS(uuidKey, fmt.Sprint(prevValue+1), prevIndex, nil)
		if err != nil {
			casDelay()
			continue
		}

		// record serial number of this service, already shifted
		p.machineID = (uint64(prevValue+1) & machineIDMask) << 12
		return
	}
}

// Next is get next value of a key, like auto-incrememt in mysql
func (p *server) Next(ctx context.Context, in *pb.Snowflake_Key) (*pb.Snowflake_Value, error) {
	client := <-p.clientPool
	defer func() { p.clientPool <- client }()
	key := path + in.Name
	for {
		// get the key
		kvpair, _, err := client.GetKVPair(key, nil)
		if err != nil {
			log.Error(err)
			return nil, errors.New("Key not exists, need to create first")
		}

		// get prevValue & prevIndex
		prevValue, err := strconv.Atoi(utils.Bytes2Str(kvpair.Value))
		if err != nil {
			log.Error(err)
			return nil, errors.New("marlformed value")
		}
		prevIndex := kvpair.ModifyIndex

		// compareAndSwap
		_, _, err = client.CAS(key, fmt.Sprint(prevValue+1), prevIndex, nil)
		if err != nil {
			casDelay()
			continue
		}
		return &pb.Snowflake_Value{Value: int64(prevValue + 1)}, nil
	}
}

// GetUUID is generate an unique uuid
func (p *server) GetUUID(context.Context, *pb.Snowflake_NullRequest) (*pb.Snowflake_UUID, error) {
	req := make(chan uint64, 1)
	p.procCh <- req
	return &pb.Snowflake_UUID{Uuid: <-req}, nil
}

// uuid generator
func (p *server) uuidTask() {
	var sn uint64    // 12-bit serial no
	var lastts int64 // last timestamp
	for {
		ret := <-p.procCh
		// get a correct serial number
		t := ts()
		// clock shift backward
		if t < lastts {
			log.Error("clock shift happened, waiting until the clock moving to the next millisecond.")
			t = p.waitMs(lastts)
		}

		// same millisecond
		if lastts == t {
			sn = (sn + 1) & snMask
			// serial number overflows, wait until next ms
			if sn == 0 {
				t = p.waitMs(lastts)
			}
		} else { // new millsecond, reset serial number to 0
			sn = 0
		}
		// remember last timestamp
		lastts = t

		// generate uuid, format:
		//
		// 0		0.................0		0..............0	0........0
		// 1-bit	41bit timestamp			10bit machine-id	12bit sn
		var uuid uint64
		uuid |= (uint64(t) & tsMask) << 22
		uuid |= p.machineID
		uuid |= sn
		ret <- uuid
	}
}

// waitMs will spin wait till next millisecond
func (p *server) waitMs(lastts int64) int64 {
	t := ts()
	for t <= lastts {
		t = ts()
	}
	return t
}

// random delay
func casDelay() {
	<-time.After(time.Duration(rand.Int63n(backoff)) * time.Millisecond)
}

// get timestamp
func ts() int64 {
	return time.Now().UnixNano() / int64(time.Millisecond)
}
<|endoftext|>


<commit_before>// Copyright 2017 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package ratelimit

import (
	"net/http"
	"time"
)

// GenFunc 用于生成用户唯一 ID 的函数
type GenFunc func(*http.Request) string

// Store 存储 Bucket 的接口
type Store interface {
	// 设置或是添加指定名称的令牌桶
	Set(name string, val *Bucket) error

	// 删除指定的令牌桶
	Delete(name string) error

	// 获取指定的令牌桶，为空则返回 nil
	Get(name string) *Bucket
}

// Server 提供操作 Bucket 的一系列服务。
type Server struct {
	store    Store
	capacity int64
	rate     time.Duration
	genFunc  GenFunc
}

func genIP(r *http.Request) string {
	return r.RemoteAddr
}

// NewServer 声明一个新的 Server。
// fn 为令牌桶名称的产生方法，默认为用户的访问 IP。
func NewServer(store Store, capacity int64, rate time.Duration, fn GenFunc) *Server {
	if fn == nil {
		fn = genIP
	}

	return &Server{
		store:    store,
		capacity: capacity,
		rate:     rate,
		genFunc:  fn,
	}
}

// 获取与前请求相对应的令牌桶。
func (srv *Server) bucket(r *http.Request) (*Bucket, error) {
	name := srv.genFunc(r)

	b := srv.store.Get(name)
	if b == nil {
		b = newBucket(srv.capacity, srv.rate)
		if err := srv.store.Set(name, b); err != nil {
			return nil, err
		}
	}

	return b, nil
}

// Transfer 将 oldName 的数据传送给 newName。
func (srv *Server) Transfer(oldName, newName string) error {
	b := srv.store.Get(oldName)
	if b != nil {
		if err := srv.store.Delete(oldName); err != nil {
			return err
		}
	}

	return srv.store.Set(newName, b)
}
<commit_msg>GenFunc 多了个返回参数 error<commit_after>// Copyright 2017 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package ratelimit

import (
	"errors"
	"net/http"
	"time"
)

// GenFunc 用于生成用户唯一 ID 的函数
type GenFunc func(*http.Request) (string, error)

// Store 存储 Bucket 的接口
type Store interface {
	// 设置或是添加指定名称的令牌桶
	Set(name string, val *Bucket) error

	// 删除指定的令牌桶
	Delete(name string) error

	// 获取指定的令牌桶，为空则返回 nil
	Get(name string) *Bucket
}

// Server 提供操作 Bucket 的一系列服务。
type Server struct {
	store    Store
	capacity int64
	rate     time.Duration
	genFunc  GenFunc
}

func genIP(r *http.Request) (string, error) {
	if len(r.RemoteAddr) == 0 {
		return "", errors.New("无法获取请求端的 IP 地址")
	}
	return r.RemoteAddr, nil
}

// NewServer 声明一个新的 Server。
// fn 为令牌桶名称的产生方法，默认为用户的访问 IP。
func NewServer(store Store, capacity int64, rate time.Duration, fn GenFunc) *Server {
	if fn == nil {
		fn = genIP
	}

	return &Server{
		store:    store,
		capacity: capacity,
		rate:     rate,
		genFunc:  fn,
	}
}

// 获取与前请求相对应的令牌桶。
func (srv *Server) bucket(r *http.Request) (*Bucket, error) {
	name, err := srv.genFunc(r)
	if err != nil {
		return nil, err
	}

	b := srv.store.Get(name)
	if b == nil {
		b = newBucket(srv.capacity, srv.rate)
		if err := srv.store.Set(name, b); err != nil {
			return nil, err
		}
	}

	return b, nil
}

// Transfer 将 oldName 的数据传送给 newName。
func (srv *Server) Transfer(oldName, newName string) error {
	b := srv.store.Get(oldName)
	if b != nil {
		if err := srv.store.Delete(oldName); err != nil {
			return err
		}
	}

	return srv.store.Set(newName, b)
}
<|endoftext|>


<commit_before>package client

import (
	"errors"

	"github.com/flynn/go-discover/discover"
	"github.com/flynn/rpcplus"
	"github.com/flynn/sampi/types"
)

func New() (*Client, error) {
	disc, err := discover.NewClient()
	if err != nil {
		return nil, err
	}
	services, err := disc.Services("flynn-sampi")
	if err != nil {
		return nil, err
	}
	addrs := services.OnlineAddrs()
	if len(addrs) == 0 {
		return nil, errors.New("sampi: no servers found")
	}
	c, err := rpcplus.DialHTTP("tcp", addrs[0])
	return &Client{c}, err
}

type Client struct {
	c *rpcplus.Client
}

func (c *Client) State() (map[string]sampi.Host, error) {
	var state map[string]sampi.Host
	err := c.c.Call("Scheduler.State", struct{}{}, &state)
	return state, err
}

func (c *Client) Schedule(req *sampi.ScheduleReq) (*sampi.ScheduleRes, error) {
	var res sampi.ScheduleRes
	err := c.c.Call("Scheduler.Schedule", req, &res)
	return &res, err
}

func (c *Client) RegisterHost(host *sampi.Host, stream chan *sampi.Job) *error {
	return &c.c.StreamGo("Scheduler.RegisterHost", host, stream).Error
}

func (c *Client) RemoveJobs(jobIDs []string) error {
	return c.c.Call("Scheduler.RemoveJobs", jobIDs, &struct{}{})
}
<commit_msg>pkg/sampi: Add random job ID generation<commit_after>package client

import (
	"bytes"
	"crypto/rand"
	"encoding/base64"
	"errors"
	"io"

	"github.com/flynn/go-discover/discover"
	"github.com/flynn/rpcplus"
	"github.com/flynn/sampi/types"
)

func New() (*Client, error) {
	disc, err := discover.NewClient()
	if err != nil {
		return nil, err
	}
	services, err := disc.Services("flynn-sampi")
	if err != nil {
		return nil, err
	}
	addrs := services.OnlineAddrs()
	if len(addrs) == 0 {
		return nil, errors.New("sampi: no servers found")
	}
	c, err := rpcplus.DialHTTP("tcp", addrs[0])
	return &Client{c}, err
}

type Client struct {
	c *rpcplus.Client
}

func (c *Client) State() (map[string]sampi.Host, error) {
	var state map[string]sampi.Host
	err := c.c.Call("Scheduler.State", struct{}{}, &state)
	return state, err
}

func (c *Client) Schedule(req *sampi.ScheduleReq) (*sampi.ScheduleRes, error) {
	var res sampi.ScheduleRes
	err := c.c.Call("Scheduler.Schedule", req, &res)
	return &res, err
}

func (c *Client) RegisterHost(host *sampi.Host, stream chan *sampi.Job) *error {
	return &c.c.StreamGo("Scheduler.RegisterHost", host, stream).Error
}

func (c *Client) RemoveJobs(jobIDs []string) error {
	return c.c.Call("Scheduler.RemoveJobs", jobIDs, &struct{}{})
}

func RandomJobID(prefix string) string { return prefix + randomID() }

func randomID() string {
	b := make([]byte, 16)
	enc := make([]byte, 24)
	_, err := io.ReadFull(rand.Reader, b)
	if err != nil {
		panic(err) // This shouldn't ever happen, right?
	}
	base64.URLEncoding.Encode(enc, b)
	return string(bytes.TrimRight(enc, "="))
}
<|endoftext|>


<commit_before>// package bit
package main

import (
	"bufio"
	"bytes"
	"crypto/aes"
	"crypto/cipher"
	"crypto/sha1"
	"encoding/binary"
	"flag"
	"fmt"
	"hash"
	"io"
	"log"
	"os"
	"time"

	chacha "github.com/codahale/chacha20poly1305"
	multihash "github.com/jbenet/go-multihash"
)

type File interface {
	// Path relative to the repo root
	Path() string

	// File size of the file in bytes
	Size() int

	// Modification timestamp (with timezone)
	Mtime() time.Time

	// Hash of the unencrypted file
	Hash() multihash.Multihash

	// Hash of the encrypted file from IPFS
	IpfsHash() multihash.Multihash
}

func NewFile(path string) (*File, error) {
	// TODO:
	return nil, nil
}

// Possible ciphers in Counter mode:
const (
	aeadCipherChaCha = iota
	aeadCipherAES
)

//
const (
	maxPackSize       = 1 * 1024 * 1024
	goodBufSize       = maxPackSize + 32
	padPackSize       = 4
	defaultCipherType = aeadCipherChaCha
)

func createAEADWorker(cipherType int, key []byte) (cipher.AEAD, error) {
	switch cipherType {
	case aeadCipherAES:
		block, err := aes.NewCipher(key)
		if err != nil {
			return nil, err
		}
		return cipher.NewGCM(block)
	case aeadCipherChaCha:
		return chacha.New(key)
	}

	return nil, fmt.Errorf("No such cipher type.")
}

type aeadCommon struct {
	// Hashing io.Writer for in-band hashing.
	Hasher hash.Hash

	// Nonce that form the first aead.NonceSize() bytes
	// of the output
	nonce []byte

	// Short temporary buffer for encoding the packSize
	sizeBuf []byte

	// For more information, see:
	// https://en.wikipedia.org/wiki/Authenticated_encryption
	aead cipher.AEAD
}

func (c *aeadCommon) initAeadCommon(key []byte) error {
	c.Hasher = sha1.New()

	aead, err := createAEADWorker(defaultCipherType, key)
	if err != nil {
		return err
	}

	c.nonce = make([]byte, aead.NonceSize())
	c.sizeBuf = make([]byte, padPackSize)
	c.aead = aead
	return nil
}

type AEADReader struct {
	aeadCommon

	Reader io.ReadSeeker

	packBuf []byte
	backlog *bytes.Buffer
}

// Read from source and decrypt + hash it.
//
// This method always decrypts one block to optimize for continous reads. If
// dest is too small to hold the block, the decrypted text is cached for the
// next read.
func (r *AEADReader) Read(dest []byte) (int, error) {
	if r.backlog.Len() == 0 {
		_, err := r.readBlock()
		if err != nil {
			return 0, err
		}
	}

	return r.backlog.Read(dest)
}

// Fill internal buffer with current block
func (r *AEADReader) readBlock() (int, error) {
	n, err := r.Reader.Read(r.sizeBuf)
	if err != nil {
		return 0, err
	}

	packSize := binary.BigEndian.Uint32(r.sizeBuf)
	if packSize > maxPackSize+uint32(r.aead.Overhead()) {
		return 0, fmt.Errorf("Pack size exceeded expectations: %d > %d",
			packSize, maxPackSize)
	}

	if n, err = r.Reader.Read(r.nonce); err != nil {
		return 0, err
	} else if n != r.aead.NonceSize() {
		return 0, fmt.Errorf("Nonce size mismatch. Should: %d. Have: %d",
			r.aead.NonceSize(), n)
	}

	n, err = r.Reader.Read(r.packBuf[:packSize])
	if err != nil {
		return 0, err
	}

	decrypted, err := r.aead.Open(nil, r.nonce, r.packBuf[:n], nil)
	if err != nil {
		return 0, err
	}

	if _, err = r.Hasher.Write(decrypted); err != nil {
		return 0, err
	}

	return r.backlog.Write(decrypted)
}

func convertEncToDec(n, blockSize, totalBlockSize, headerSize int64) int64 {
	encBlocks := n / totalBlockSize
	encOffset := n % totalBlockSize
	return (encBlocks * blockSize) + encOffset - headerSize
}

// Seek into the encrypted stream.
//
// Note that the seek offset is relative to the decrypted data,
// not to the underlying, encrypted stream.
//
// Mixing SEEK_CUR and SEEK_SET might not a good idea,
// since a seek might involve reading a whole encrypted block.
// Therefore relative seek offset
func (r *AEADReader) Seek(offset int64, whence int) (int64, error) {
	// Constants and assumption on the stream below:
	blockSize := int64(maxPackSize)
	headerSize := int64(padPackSize + r.aead.NonceSize())
	totalBlockSize := headerSize + blockSize + int64(r.aead.Overhead())

	// absolute Offset in the decrypted stream
	absOffsetDec := int64(0)

	// Find out where we are currently:
	absSeekOffsetEnc, err := r.Reader.Seek(0, os.SEEK_CUR)
	if err != nil {
		return 0, err
	}

	// Convert the encrypted offset to the decrypted offset:
	absSeekOffsetDec := convertEncToDec(absSeekOffsetEnc,
		blockSize, totalBlockSize, headerSize)

	// Convert possibly relative offset to absolute offset:
	switch whence {
	case os.SEEK_CUR:
		absOffsetDec = absSeekOffsetDec + offset
	case os.SEEK_SET:
		absOffsetDec = offset
	case os.SEEK_END:
		return 0, fmt.Errorf("There is no definite end, can't use SEEK_END")
	}

	// Caller wanted to know only the current stream pos:
	if absOffsetDec == absSeekOffsetDec {
		return absOffsetDec, nil
	}

	// Convert decrypted offset to encrypted offset
	absOffsetEnc := (absOffsetDec / blockSize) * totalBlockSize

	// Clear backlog, reading will cause it to be re-read
	r.backlog.Reset()

	// Seek to the beginning of the encrypted block:
	_, err = r.Reader.Seek(absOffsetEnc, os.SEEK_SET)
	if err != nil {
		return 0, err
	}

	// TODO: make this dummy buf persist during seeks
	dummy := make([]byte, absOffsetDec%blockSize)
	if _, err = r.Read(dummy); err != nil {
		return 0, err
	}

	return absOffsetDec, nil
}

func (r *AEADReader) Close() error {
	return nil
}

func NewAEADReader(r io.ReadSeeker, key []byte) (*AEADReader, error) {
	reader := &AEADReader{
		Reader:  r,
		backlog: new(bytes.Buffer),
	}

	if err := reader.initAeadCommon(key); err != nil {
		return nil, err
	}

	reader.packBuf = make([]byte, 0, maxPackSize+reader.aead.Overhead())
	return reader, nil
}

////////////

type AEADWriter struct {
	// Common fields with AEADReader
	aeadCommon

	// Internal Writer we would write to.
	Writer io.Writer

	// A buffer that is maxPackSize big.
	// Used for caching blocks
	packBuf *bytes.Buffer
}

func (w *AEADWriter) Write(p []byte) (int, error) {
	w.packBuf.Write(p)
	if w.packBuf.Len() < maxPackSize {
		return 0, nil
	}

	return w.flushPack(maxPackSize)
}

func (w *AEADWriter) flushPack(chunkSize int) (int, error) {
	// Try to update the checksum as we run:
	src := w.packBuf.Bytes()[:chunkSize]

	// Make sure to advance this many bytes
	// in case any leftovers are in the buffer.
	defer w.packBuf.Read(src[:chunkSize])

	if _, err := w.Hasher.Write(src); err != nil {
		return 0, err
	}

	// Create a new Nonce for this block:
	hash := w.Hasher.Sum(nil)
	copy(w.nonce, hash[w.aead.NonceSize():])

	// Encrypt the text:
	encrypted := w.aead.Seal(nil, w.nonce, src, nil)

	// Pass it to the underlying writer:
	binary.BigEndian.PutUint32(w.sizeBuf, uint32(len(encrypted)))

	w.Writer.Write(w.sizeBuf)
	w.Writer.Write(w.nonce)
	w.Writer.Write(encrypted)

	// len(encrypted) might be more than len(w.packBuf)
	return len(encrypted) + len(w.nonce) + len(w.sizeBuf), nil
}

func (w *AEADWriter) Close() error {
	_, err := w.flushPack(w.packBuf.Len())
	return err
}

func NewAEADWriter(w io.Writer, key []byte) (*AEADWriter, error) {
	writer := &AEADWriter{
		Writer:  w,
		packBuf: bytes.NewBuffer(make([]byte, 0, maxPackSize)),
	}

	if err := writer.initAeadCommon(key); err != nil {
		return nil, err
	}

	return writer, nil
}

func securedTransfer(reader io.Reader, writer io.Writer, encrypt bool) error {
	r := bufio.NewReader(reader)
	buf := make([]byte, 0, goodBufSize)

	for {
		n, err := r.Read(buf[:cap(buf)])
		buf = buf[:n]
		if n == 0 {
			if err == nil {
				continue
			}
			if err == io.EOF {
				break
			}

			return err
		}

		if err != nil && err != io.EOF {
			return err
		}

		_, err = writer.Write(buf)
		if err != nil {
			return err
		}
	}

	return nil
}

func Encrypt(key []byte, source io.ReadSeeker, dest io.Writer) error {
	layer, err := NewAEADWriter(dest, key)
	if err != nil {
		return err
	}

	defer layer.Close()
	return securedTransfer(source, layer, true)
}

func Decrypt(key []byte, source io.ReadSeeker, dest io.Writer) error {
	layer, err := NewAEADReader(source, key)
	if err != nil {
		return err
	}

	defer layer.Close()
	return securedTransfer(layer, dest, true)
}

func main() {
	decryptMode := flag.Bool("d", false, "Decrypt.")
	flag.Parse()

	key := []byte("01234567890ABCDE01234567890ABCDE")

	var err error
	if *decryptMode == false {
		err = Encrypt(key, os.Stdin, os.Stdout)
	} else {
		source, _ := os.Open("/tmp/dump")

		err = Decrypt(key, source, os.Stdout)
	}

	if err != nil {
		log.Fatal(err)
		return
	}
}
<commit_msg>Inc function added.<commit_after>// package bit
package main

import (
	"bufio"
	"bytes"
	"crypto/aes"
	"crypto/cipher"
	"crypto/sha1"
	"encoding/binary"
	"flag"
	"fmt"
	"hash"
	"io"
	"log"
	"os"
	"time"

	chacha "github.com/codahale/chacha20poly1305"
	multihash "github.com/jbenet/go-multihash"
)

type File interface {
	// Path relative to the repo root
	Path() string

	// File size of the file in bytes
	Size() int

	// Modification timestamp (with timezone)
	Mtime() time.Time

	// Hash of the unencrypted file
	Hash() multihash.Multihash

	// Hash of the encrypted file from IPFS
	IpfsHash() multihash.Multihash
}

func NewFile(path string) (*File, error) {
	// TODO:
	return nil, nil
}

// Possible ciphers in Counter mode:
const (
	aeadCipherChaCha = iota
	aeadCipherAES
)

//
const (
	maxPackSize       = 1 * 1024 * 1024
	goodBufSize       = maxPackSize + 32
	padPackSize       = 4
	defaultCipherType = aeadCipherChaCha
)

func createAEADWorker(cipherType int, key []byte) (cipher.AEAD, error) {
	switch cipherType {
	case aeadCipherAES:
		block, err := aes.NewCipher(key)
		if err != nil {
			return nil, err
		}
		return cipher.NewGCM(block)
	case aeadCipherChaCha:
		return chacha.New(key)
	}

	return nil, fmt.Errorf("No such cipher type.")
}

type aeadCommon struct {
	// Hashing io.Writer for in-band hashing.
	Hasher hash.Hash

	// Nonce that form the first aead.NonceSize() bytes
	// of the output
	nonce []byte

	// Short temporary buffer for encoding the packSize
	sizeBuf []byte

	// For more information, see:
	// https://en.wikipedia.org/wiki/Authenticated_encryption
	aead cipher.AEAD
}

func (c *aeadCommon) initAeadCommon(key []byte) error {
	c.Hasher = sha1.New()

	aead, err := createAEADWorker(defaultCipherType, key)
	if err != nil {
		return err
	}

	c.nonce = make([]byte, aead.NonceSize())
	c.sizeBuf = make([]byte, padPackSize)
	c.aead = aead
	return nil
}

type AEADReader struct {
	aeadCommon

	Reader io.ReadSeeker

	packBuf []byte
	backlog *bytes.Buffer
}

// Read from source and decrypt + hash it.
//
// This method always decrypts one block to optimize for continous reads. If
// dest is too small to hold the block, the decrypted text is cached for the
// next read.
func (r *AEADReader) Read(dest []byte) (int, error) {
	if r.backlog.Len() == 0 {
		_, err := r.readBlock()
		if err != nil {
			return 0, err
		}
	}

	return r.backlog.Read(dest)
}

// Fill internal buffer with current block
func (r *AEADReader) readBlock() (int, error) {
	n, err := r.Reader.Read(r.sizeBuf)
	if err != nil {
		return 0, err
	}

	packSize := binary.BigEndian.Uint32(r.sizeBuf)
	if packSize > maxPackSize+uint32(r.aead.Overhead()) {
		return 0, fmt.Errorf("Pack size exceeded expectations: %d > %d",
			packSize, maxPackSize)
	}

	if n, err = r.Reader.Read(r.nonce); err != nil {
		return 0, err
	} else if n != r.aead.NonceSize() {
		return 0, fmt.Errorf("Nonce size mismatch. Should: %d. Have: %d",
			r.aead.NonceSize(), n)
	}

	n, err = r.Reader.Read(r.packBuf[:packSize])
	if err != nil {
		return 0, err
	}

	decrypted, err := r.aead.Open(nil, r.nonce, r.packBuf[:n], nil)
	if err != nil {
		return 0, err
	}

	if _, err = r.Hasher.Write(decrypted); err != nil {
		return 0, err
	}

	return r.backlog.Write(decrypted)
}

func convertEncToDec(n, blockSize, totalBlockSize, headerSize int64) int64 {
	encBlocks := n / totalBlockSize
	encOffset := n % totalBlockSize
	return (encBlocks * blockSize) + encOffset - headerSize
}

// Seek into the encrypted stream.
//
// Note that the seek offset is relative to the decrypted data,
// not to the underlying, encrypted stream.
//
// Mixing SEEK_CUR and SEEK_SET might not a good idea,
// since a seek might involve reading a whole encrypted block.
// Therefore relative seek offset
func (r *AEADReader) Seek(offset int64, whence int) (int64, error) {
	// Constants and assumption on the stream below:
	blockSize := int64(maxPackSize)
	headerSize := int64(padPackSize + r.aead.NonceSize())
	totalBlockSize := headerSize + blockSize + int64(r.aead.Overhead())

	// absolute Offset in the decrypted stream
	absOffsetDec := int64(0)

	// Find out where we are currently:
	absSeekOffsetEnc, err := r.Reader.Seek(0, os.SEEK_CUR)
	if err != nil {
		return 0, err
	}

	// Convert the encrypted offset to the decrypted offset:
	absSeekOffsetDec := convertEncToDec(absSeekOffsetEnc,
		blockSize, totalBlockSize, headerSize)

	// Convert possibly relative offset to absolute offset:
	switch whence {
	case os.SEEK_CUR:
		absOffsetDec = absSeekOffsetDec + offset
	case os.SEEK_SET:
		absOffsetDec = offset
	case os.SEEK_END:
		return 0, fmt.Errorf("There is no definite end, can't use SEEK_END")
	}

	// Caller wanted to know only the current stream pos:
	if absOffsetDec == absSeekOffsetDec {
		return absOffsetDec, nil
	}

	// Convert decrypted offset to encrypted offset
	absOffsetEnc := (absOffsetDec / blockSize) * totalBlockSize

	// Clear backlog, reading will cause it to be re-read
	r.backlog.Reset()

	// Seek to the beginning of the encrypted block:
	_, err = r.Reader.Seek(absOffsetEnc, os.SEEK_SET)
	if err != nil {
		return 0, err
	}

	// TODO: make this dummy buf persist during seeks
	dummy := make([]byte, absOffsetDec%blockSize)
	if _, err = r.Read(dummy); err != nil {
		return 0, err
	}

	return absOffsetDec, nil
}

func (r *AEADReader) Close() error {
	return nil
}

func NewAEADReader(r io.ReadSeeker, key []byte) (*AEADReader, error) {
	reader := &AEADReader{
		Reader:  r,
		backlog: new(bytes.Buffer),
	}

	if err := reader.initAeadCommon(key); err != nil {
		return nil, err
	}

	reader.packBuf = make([]byte, 0, maxPackSize+reader.aead.Overhead())
	return reader, nil
}

////////////

type AEADWriter struct {
	// Common fields with AEADReader
	aeadCommon

	// Internal Writer we would write to.
	Writer io.Writer

	// A buffer that is maxPackSize big.
	// Used for caching blocks
	packBuf *bytes.Buffer
}

func inc(arr []byte) {
	i := uint64(0)
	for idx, val := range arr {
		i += uint64(val) << (8 * uint64(idx))
	}
	i++
	for idx, _ := range arr {
		arr[idx] = byte(i % 256)
		i = i >> 8
		if i == 0 {
			break
		}
	}
}

func (w *AEADWriter) Write(p []byte) (int, error) {
	w.packBuf.Write(p)
	if w.packBuf.Len() < maxPackSize {
		return 0, nil
	}

	return w.flushPack(maxPackSize)
}

func (w *AEADWriter) flushPack(chunkSize int) (int, error) {
	// Try to update the checksum as we run:
	src := w.packBuf.Bytes()[:chunkSize]

	// Make sure to advance this many bytes
	// in case any leftovers are in the buffer.
	defer w.packBuf.Read(src[:chunkSize])

	if _, err := w.Hasher.Write(src); err != nil {
		return 0, err
	}

	// Create a new Nonce for this block:
	inc(w.nonce)

	// Encrypt the text:
	encrypted := w.aead.Seal(nil, w.nonce, src, nil)

	// Pass it to the underlying writer:
	binary.BigEndian.PutUint32(w.sizeBuf, uint32(len(encrypted)))

	w.Writer.Write(w.sizeBuf)
	w.Writer.Write(w.nonce)
	w.Writer.Write(encrypted)

	// len(encrypted) might be more than len(w.packBuf)
	return len(encrypted) + len(w.nonce) + len(w.sizeBuf), nil
}

func (w *AEADWriter) Close() error {
	_, err := w.flushPack(w.packBuf.Len())
	return err
}

func NewAEADWriter(w io.Writer, key []byte) (*AEADWriter, error) {
	writer := &AEADWriter{
		Writer:  w,
		packBuf: bytes.NewBuffer(make([]byte, 0, maxPackSize)),
	}

	if err := writer.initAeadCommon(key); err != nil {
		return nil, err
	}

	return writer, nil
}

func securedTransfer(reader io.Reader, writer io.Writer, encrypt bool) error {
	r := bufio.NewReader(reader)
	buf := make([]byte, 0, goodBufSize)

	for {
		n, err := r.Read(buf[:cap(buf)])
		buf = buf[:n]
		if n == 0 {
			if err == nil {
				continue
			}
			if err == io.EOF {
				break
			}

			return err
		}

		if err != nil && err != io.EOF {
			return err
		}

		_, err = writer.Write(buf)
		if err != nil {
			return err
		}
	}

	return nil
}

func Encrypt(key []byte, source io.ReadSeeker, dest io.Writer) error {
	layer, err := NewAEADWriter(dest, key)
	if err != nil {
		return err
	}

	defer layer.Close()
	return securedTransfer(source, layer, true)
}

func Decrypt(key []byte, source io.ReadSeeker, dest io.Writer) error {
	layer, err := NewAEADReader(source, key)
	if err != nil {
		return err
	}

	defer layer.Close()
	return securedTransfer(layer, dest, true)
}

func main() {
	decryptMode := flag.Bool("d", false, "Decrypt.")
	flag.Parse()

	key := []byte("01234567890ABCDE01234567890ABCDE")

	var err error
	if *decryptMode == false {
		err = Encrypt(key, os.Stdin, os.Stdout)
	} else {
		source, _ := os.Open("/tmp/dump")

		err = Decrypt(key, source, os.Stdout)
	}

	if err != nil {
		log.Fatal(err)
		return
	}
}
<|endoftext|>


<commit_before>package peerstream_spdystream

import (
	"net"
	"net/http"

	pst "github.com/jbenet/go-peerstream/transport"
	ss "github.com/jbenet/spdystream"
)

// stream implements pst.Stream using a ss.Stream
type stream ss.Stream

func (s *stream) spdyStream() *ss.Stream {
	return (*ss.Stream)(s)
}

func (s *stream) Read(buf []byte) (int, error) {
	return s.spdyStream().Read(buf)
}

func (s *stream) Write(buf []byte) (int, error) {
	return s.spdyStream().Write(buf)
}

func (s *stream) Close() error {
	// Reset is spdystream's full bidirectional close.
	// We expose bidirectional close as our `Close`.
	// To close only half of the connection, and use other
	// spdystream options, just get the stream with:
	//  ssStream := (*ss.Stream)(stream)
	return s.spdyStream().Reset()
}

// Conn is a connection to a remote peer.
type conn struct {
	sc *ss.Connection

	closed chan struct{}
}

func (c *conn) spdyConn() *ss.Connection {
	return c.sc
}

func (c *conn) Close() error {
	err := c.spdyConn().CloseWait()
	if !c.IsClosed() {
		close(c.closed)
	}
	return err
}

func (c *conn) IsClosed() bool {
	select {
	case <-c.closed:
		return true
	default:
		return false
	}
}

// OpenStream creates a new stream.
func (c *conn) OpenStream() (pst.Stream, error) {
	s, err := c.spdyConn().CreateStream(http.Header{}, nil, false)
	if err != nil {
		return nil, err
	}

	// wait for a response before writing. for some reason
	// spdystream does not make forward progress unless you do this.
	s.Wait()
	return (*stream)(s), nil
}

// Serve starts listening for incoming requests and handles them
// using given StreamHandler
func (c *conn) Serve(handler pst.StreamHandler) {
	c.spdyConn().Serve(func(s *ss.Stream) {

		// Flow control and backpressure of Opening streams is broken.
		// I believe that spdystream has one set of workers that both send
		// data AND accept new streams (as it's just more data). there
		// is a problem where if the new stream handlers want to throttle,
		// they also eliminate the ability to read/write data, which makes
		// forward-progress impossible. Thus, throttling this function is
		// -- at this moment -- not the solution. Either spdystream must
		// change, or we must throttle another way. go-peerstream handles
		// every new stream in its own goroutine.
		s.SendReply(http.Header{}, false)
		go handler((*stream)(s))
	})
}

type transport struct{}

// Transport is a go-peerstream transport that constructs
// spdystream-backed connections.
var Transport = transport{}

func (t transport) NewConn(nc net.Conn, isServer bool) (pst.Conn, error) {
	sc, err := ss.NewConnection(nc, isServer)
	return &conn{sc: sc, closed: make(chan struct{})}, err
}
<commit_msg>add note about SendReply err<commit_after>package peerstream_spdystream

import (
	"net"
	"net/http"

	pst "github.com/jbenet/go-peerstream/transport"
	ss "github.com/jbenet/spdystream"
)

// stream implements pst.Stream using a ss.Stream
type stream ss.Stream

func (s *stream) spdyStream() *ss.Stream {
	return (*ss.Stream)(s)
}

func (s *stream) Read(buf []byte) (int, error) {
	return s.spdyStream().Read(buf)
}

func (s *stream) Write(buf []byte) (int, error) {
	return s.spdyStream().Write(buf)
}

func (s *stream) Close() error {
	// Reset is spdystream's full bidirectional close.
	// We expose bidirectional close as our `Close`.
	// To close only half of the connection, and use other
	// spdystream options, just get the stream with:
	//  ssStream := (*ss.Stream)(stream)
	return s.spdyStream().Reset()
}

// Conn is a connection to a remote peer.
type conn struct {
	sc *ss.Connection

	closed chan struct{}
}

func (c *conn) spdyConn() *ss.Connection {
	return c.sc
}

func (c *conn) Close() error {
	err := c.spdyConn().CloseWait()
	if !c.IsClosed() {
		close(c.closed)
	}
	return err
}

func (c *conn) IsClosed() bool {
	select {
	case <-c.closed:
		return true
	default:
		return false
	}
}

// OpenStream creates a new stream.
func (c *conn) OpenStream() (pst.Stream, error) {
	s, err := c.spdyConn().CreateStream(http.Header{}, nil, false)
	if err != nil {
		return nil, err
	}

	// wait for a response before writing. for some reason
	// spdystream does not make forward progress unless you do this.
	s.Wait()
	return (*stream)(s), nil
}

// Serve starts listening for incoming requests and handles them
// using given StreamHandler
func (c *conn) Serve(handler pst.StreamHandler) {
	c.spdyConn().Serve(func(s *ss.Stream) {

		// Flow control and backpressure of Opening streams is broken.
		// I believe that spdystream has one set of workers that both send
		// data AND accept new streams (as it's just more data). there
		// is a problem where if the new stream handlers want to throttle,
		// they also eliminate the ability to read/write data, which makes
		// forward-progress impossible. Thus, throttling this function is
		// -- at this moment -- not the solution. Either spdystream must
		// change, or we must throttle another way. go-peerstream handles
		// every new stream in its own goroutine.
		err := s.SendReply(http.Header{}, false)
		if err != nil {
			// this _could_ error out. not sure how to handle this failure.
			// don't return, and let the caller handle a broken stream.
			// better than _hiding_ an error.
			// return
		}
		go handler((*stream)(s))
	})
}

type transport struct{}

// Transport is a go-peerstream transport that constructs
// spdystream-backed connections.
var Transport = transport{}

func (t transport) NewConn(nc net.Conn, isServer bool) (pst.Conn, error) {
	sc, err := ss.NewConnection(nc, isServer)
	return &conn{sc: sc, closed: make(chan struct{})}, err
}
<|endoftext|>


<commit_before>package raftgorums

import (
	"sync"

	"github.com/relab/raft/commonpb"
)

// CacheStorage wraps a Storage adding a layer of caching. It uses the
// underlying storage as a fallback if the data is not cached.
type CacheStorage struct {
	s Storage

	l          sync.RWMutex
	stateCache map[uint64]uint64
	logCache   []*commonpb.Entry
}

// NewCacheStorage returns a new initialized CacheStorage.
func NewCacheStorage(s Storage, cacheSize int) *CacheStorage {
	return &CacheStorage{
		s:          s,
		stateCache: make(map[uint64]uint64),
		logCache:   make([]*commonpb.Entry, cacheSize),
	}
}

// Set implements the Storage interface.
func (cs *CacheStorage) Set(key uint64, value uint64) error {
	cs.l.Lock()
	cs.stateCache[key] = value
	cs.l.Unlock()
	return cs.s.Set(key, value)
}

// Get implements the Storage interface.
func (cs *CacheStorage) Get(key uint64) (uint64, error) {
	cs.l.Lock()
	defer cs.l.Unlock()

	value, ok := cs.stateCache[key]
	if !ok {
		return cs.s.Get(key)
	}
	return value, nil
}

// StoreEntries implements the Storage interface.
func (cs *CacheStorage) StoreEntries(entries []*commonpb.Entry) error {
	cs.l.Lock()
	for _, entry := range entries {
		cs.logCache[entry.Index%uint64(len(cs.logCache))] = entry
	}
	cs.stateCache[KeyNextIndex] = entries[len(entries)-1].Index + 1
	cs.l.Unlock()
	return cs.s.StoreEntries(entries)
}

// GetEntry implements the Storage interface.
func (cs *CacheStorage) GetEntry(index uint64) (*commonpb.Entry, error) {
	cs.l.RLock()
	entry := cs.logCache[index%uint64(len(cs.logCache))]
	cs.l.RUnlock()

	if entry != nil && entry.Index == index {
		return entry, nil
	}

	return cs.s.GetEntry(index)
}

// GetEntries implements the Storage interface.
func (cs *CacheStorage) GetEntries(first, last uint64) ([]*commonpb.Entry, error) {
	entries := make([]*commonpb.Entry, last-first+1)

	cs.l.RLock()
	var i uint64
	for index := last; index >= first; index-- {
		entry := cs.logCache[index%uint64(len(cs.logCache))]

		if entry != nil && entry.Index == index {
			entries[i] = entry
			i++
			continue
		}

		break
	}
	cs.l.RUnlock()

	if i == first {
		return entries, nil
	}

	prefix, err := cs.s.GetEntries(first, i)

	if err != nil {
		return nil, err
	}

	return append(prefix, entries...), nil
}

// RemoveEntries implements the Storage interface.
func (cs *CacheStorage) RemoveEntries(first, last uint64) error {
	cs.l.Lock()
	cs.stateCache = make(map[uint64]uint64)
	cs.logCache = make([]*commonpb.Entry, len(cs.logCache))
	cs.l.Unlock()

	return cs.s.RemoveEntries(first, last)
}

// FirstIndex implements the Storage interface.
func (cs *CacheStorage) FirstIndex() (uint64, error) {
	return cs.Get(KeyFirstIndex)
}

// NextIndex implements the Storage interface.
func (cs *CacheStorage) NextIndex() (uint64, error) {
	return cs.Get(KeyNextIndex)
}

// SetSnapshot implements the Storage interface.
func (cs *CacheStorage) SetSnapshot(snapshot *commonpb.Snapshot) error {
	return cs.s.SetSnapshot(snapshot)
}

// GetSnapshot implements the Storage interface.
func (cs *CacheStorage) GetSnapshot() (*commonpb.Snapshot, error) {
	return cs.s.GetSnapshot()
}
<commit_msg>raftgorums/cachestorage.go: Fix returning unset entries<commit_after>package raftgorums

import (
	"sync"

	"github.com/relab/raft/commonpb"
)

// CacheStorage wraps a Storage adding a layer of caching. It uses the
// underlying storage as a fallback if the data is not cached.
type CacheStorage struct {
	s Storage

	l          sync.RWMutex
	stateCache map[uint64]uint64
	logCache   []*commonpb.Entry
}

// NewCacheStorage returns a new initialized CacheStorage.
func NewCacheStorage(s Storage, cacheSize int) *CacheStorage {
	return &CacheStorage{
		s:          s,
		stateCache: make(map[uint64]uint64),
		logCache:   make([]*commonpb.Entry, cacheSize),
	}
}

// Set implements the Storage interface.
func (cs *CacheStorage) Set(key uint64, value uint64) error {
	cs.l.Lock()
	cs.stateCache[key] = value
	cs.l.Unlock()
	return cs.s.Set(key, value)
}

// Get implements the Storage interface.
func (cs *CacheStorage) Get(key uint64) (uint64, error) {
	cs.l.Lock()
	defer cs.l.Unlock()

	value, ok := cs.stateCache[key]
	if !ok {
		return cs.s.Get(key)
	}
	return value, nil
}

// StoreEntries implements the Storage interface.
func (cs *CacheStorage) StoreEntries(entries []*commonpb.Entry) error {
	cs.l.Lock()
	for _, entry := range entries {
		cs.logCache[entry.Index%uint64(len(cs.logCache))] = entry
	}
	cs.stateCache[KeyNextIndex] = entries[len(entries)-1].Index + 1
	cs.l.Unlock()
	return cs.s.StoreEntries(entries)
}

// GetEntry implements the Storage interface.
func (cs *CacheStorage) GetEntry(index uint64) (*commonpb.Entry, error) {
	cs.l.RLock()
	entry := cs.logCache[index%uint64(len(cs.logCache))]
	cs.l.RUnlock()

	if entry != nil && entry.Index == index {
		return entry, nil
	}

	return cs.s.GetEntry(index)
}

// GetEntries implements the Storage interface.
func (cs *CacheStorage) GetEntries(first, last uint64) ([]*commonpb.Entry, error) {
	entries := make([]*commonpb.Entry, last-first+1)

	cs.l.RLock()
	var i uint64
	for index := last; index >= first; index-- {
		entry := cs.logCache[index%uint64(len(cs.logCache))]

		if entry != nil && entry.Index == index {
			entries[i] = entry
			i++
			continue
		}

		break
	}
	cs.l.RUnlock()

	if i == first {
		return entries, nil
	}

	prefix, err := cs.s.GetEntries(first, i)

	if err != nil {
		return nil, err
	}

	return append(prefix, entries[:i]...), nil
}

// RemoveEntries implements the Storage interface.
func (cs *CacheStorage) RemoveEntries(first, last uint64) error {
	cs.l.Lock()
	cs.stateCache = make(map[uint64]uint64)
	cs.logCache = make([]*commonpb.Entry, len(cs.logCache))
	cs.l.Unlock()

	return cs.s.RemoveEntries(first, last)
}

// FirstIndex implements the Storage interface.
func (cs *CacheStorage) FirstIndex() (uint64, error) {
	return cs.Get(KeyFirstIndex)
}

// NextIndex implements the Storage interface.
func (cs *CacheStorage) NextIndex() (uint64, error) {
	return cs.Get(KeyNextIndex)
}

// SetSnapshot implements the Storage interface.
func (cs *CacheStorage) SetSnapshot(snapshot *commonpb.Snapshot) error {
	return cs.s.SetSnapshot(snapshot)
}

// GetSnapshot implements the Storage interface.
func (cs *CacheStorage) GetSnapshot() (*commonpb.Snapshot, error) {
	return cs.s.GetSnapshot()
}
<|endoftext|>


<commit_before>package network

import (
	"encoding/json"
	"errors"
	"net"

	"github.com/socketplane/socketplane/Godeps/_workspace/src/github.com/socketplane/ecc"
)

const networkStore = "network"
const vlanStore = "vlan"
const DefaultNetworkName = "default"

const vlanCount = 4096

type Network struct {
	ID     string `json:"id"`
	Subnet string `json:"subnet"`
	Vlan   uint   `json:"vlan"`
}

func GetNetwork(id string) (*Network, error) {
	netByteArray, _, ok := ecc.Get(networkStore, id)
	if ok {
		network := &Network{}
		err := json.Unmarshal(netByteArray, network)
		if err != nil {
			return nil, err
		}
		return network, nil
	}
	return nil, errors.New("Network unavailable")
}

func CreateNetwork(id string, subnet *net.IPNet) (*Network, error) {
	network, err := GetNetwork(id)
	if err == nil {
		return network, nil
	}
	vlan, err := allocateVlan()
	if err != nil {
		return nil, err
	}
	network = &Network{id, subnet.String(), vlan}
	data, err := json.Marshal(network)
	if err != nil {
		return nil, err
	}
	eccerr := ecc.Put(networkStore, id, data, nil)
	if eccerr == ecc.OUTDATED {
		releaseVlan(vlan)
		return CreateNetwork(id, subnet)
	}
	return network, nil
}

func DeleteNetwork(id string) error {
	network, err := GetNetwork(id)
	if err != nil {
		return err
	}
	eccerror := ecc.Delete(networkStore, id)
	if eccerror == ecc.OK {
		releaseVlan(network.Vlan)
		return nil
	}
	return errors.New("Error deleting network")
}

func CreateDefaultNetwork(subnet *net.IPNet) (*Network, error) {
	return CreateNetwork(DefaultNetworkName, subnet)
}

func GetDefaultNetwork() (*Network, error) {
	return GetNetwork(DefaultNetworkName)
}

func allocateVlan() (uint, error) {
	vlanArray, _, ok := ecc.Get(vlanStore, "vlan")
	currVal := make([]byte, vlanCount/8)
	copy(currVal, vlanArray)
	if !ok {
		vlanArray = make([]byte, vlanCount/8)
	}
	vlan := testAndSetBit(vlanArray)
	if vlan >= vlanCount {
		return vlanCount, errors.New("Vlan unavailable")
	}
	eccerr := ecc.Put(vlanStore, "vlan", vlanArray, currVal)
	if eccerr == ecc.OUTDATED {
		return allocateVlan()
	}
	return vlan, nil
}

func releaseVlan(vlan uint) {
	vlanArray, _, ok := ecc.Get(vlanStore, "vlan")
	currVal := make([]byte, vlanCount/8)
	copy(currVal, vlanArray)
	if !ok {
		vlanArray = make([]byte, vlanCount/8)
	}
	clearBit(vlanArray, vlan-1)
	eccerr := ecc.Put(vlanStore, "vlan", vlanArray, currVal)
	if eccerr == ecc.OUTDATED {
		releaseVlan(vlan)
	}
}
func setBit(a []byte, k uint) {
	a[k/8] |= 1 << (k % 8)
}

func clearBit(a []byte, k uint) {
	a[k/8] &= ^(1 << (k % 8))
}

func testBit(a []byte, k uint) bool {
	return ((a[k/8] & (1 << (k % 8))) != 0)
}

func testAndSetBit(a []byte) uint {
	var i uint
	for i = uint(0); i < uint(len(a)*8); i++ {
		if !testBit(a, i) {
			setBit(a, i)
			return i + 1
		}
	}
	return i
}
<commit_msg>GetNetworks Support<commit_after>package network

import (
	"encoding/json"
	"errors"
	"net"

	"github.com/socketplane/socketplane/Godeps/_workspace/src/github.com/socketplane/ecc"
)

const networkStore = "network"
const vlanStore = "vlan"
const DefaultNetworkName = "default"

const vlanCount = 4096

type Network struct {
	ID     string `json:"id"`
	Subnet string `json:"subnet"`
	Vlan   uint   `json:"vlan"`
}

func GetNetworks() ([]Network, error) {
	netByteArray, _, ok := ecc.GetAll(networkStore)
	if ok {
		var networks []Network
		for _, byteArray := range netByteArray {
			network := Network{}
			err := json.Unmarshal(byteArray, &network)
			if err != nil {
				return nil, err
			}
			networks = append(networks, network)
		}
		return networks, nil
	}
	return nil, errors.New("Network unavailable")
}

func GetNetwork(id string) (*Network, error) {
	netByteArray, _, ok := ecc.Get(networkStore, id)
	if ok {
		network := &Network{}
		err := json.Unmarshal(netByteArray, network)
		if err != nil {
			return nil, err
		}
		return network, nil
	}
	return nil, errors.New("Network unavailable")
}

func CreateNetwork(id string, subnet *net.IPNet) (*Network, error) {
	network, err := GetNetwork(id)
	if err == nil {
		return network, nil
	}
	vlan, err := allocateVlan()
	if err != nil {
		return nil, err
	}
	network = &Network{id, subnet.String(), vlan}
	data, err := json.Marshal(network)
	if err != nil {
		return nil, err
	}
	eccerr := ecc.Put(networkStore, id, data, nil)
	if eccerr == ecc.OUTDATED {
		releaseVlan(vlan)
		return CreateNetwork(id, subnet)
	}
	return network, nil
}

func DeleteNetwork(id string) error {
	network, err := GetNetwork(id)
	if err != nil {
		return err
	}
	eccerror := ecc.Delete(networkStore, id)
	if eccerror == ecc.OK {
		releaseVlan(network.Vlan)
		return nil
	}
	return errors.New("Error deleting network")
}

func CreateDefaultNetwork(subnet *net.IPNet) (*Network, error) {
	return CreateNetwork(DefaultNetworkName, subnet)
}

func GetDefaultNetwork() (*Network, error) {
	return GetNetwork(DefaultNetworkName)
}

func allocateVlan() (uint, error) {
	vlanArray, _, ok := ecc.Get(vlanStore, "vlan")
	currVal := make([]byte, vlanCount/8)
	copy(currVal, vlanArray)
	if !ok {
		vlanArray = make([]byte, vlanCount/8)
	}
	vlan := testAndSetBit(vlanArray)
	if vlan >= vlanCount {
		return vlanCount, errors.New("Vlan unavailable")
	}
	eccerr := ecc.Put(vlanStore, "vlan", vlanArray, currVal)
	if eccerr == ecc.OUTDATED {
		return allocateVlan()
	}
	return vlan, nil
}

func releaseVlan(vlan uint) {
	vlanArray, _, ok := ecc.Get(vlanStore, "vlan")
	currVal := make([]byte, vlanCount/8)
	copy(currVal, vlanArray)
	if !ok {
		vlanArray = make([]byte, vlanCount/8)
	}
	clearBit(vlanArray, vlan-1)
	eccerr := ecc.Put(vlanStore, "vlan", vlanArray, currVal)
	if eccerr == ecc.OUTDATED {
		releaseVlan(vlan)
	}
}
func setBit(a []byte, k uint) {
	a[k/8] |= 1 << (k % 8)
}

func clearBit(a []byte, k uint) {
	a[k/8] &= ^(1 << (k % 8))
}

func testBit(a []byte, k uint) bool {
	return ((a[k/8] & (1 << (k % 8))) != 0)
}

func testAndSetBit(a []byte) uint {
	var i uint
	for i = uint(0); i < uint(len(a)*8); i++ {
		if !testBit(a, i) {
			setBit(a, i)
			return i + 1
		}
	}
	return i
}
<|endoftext|>