<commit_before>package handler

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"strconv"

	"github.com/micro/go-micro/cmd"
	"github.com/micro/go-micro/registry"
)

func addService(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	defer r.Body.Close()

	var service *registry.Service
	err = json.Unmarshal(b, &service)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	err = (*cmd.DefaultOptions().Registry).Register(service)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
}

func delService(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	defer r.Body.Close()

	var service *registry.Service
	err = json.Unmarshal(b, &service)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	err = (*cmd.DefaultOptions().Registry).Deregister(service)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
}

func getService(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	service := r.Form.Get("service")
	if len(service) == 0 {
		http.Error(w, "Require service", 400)
		return
	}
	s, err := (*cmd.DefaultOptions().Registry).GetService(service)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	if s == nil || len(s) == 0 || len(s[0].Name) == 0 {
		http.Error(w, "Service not found", 404)
		return
	}
	b, err := json.Marshal(s)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("Content-Length", strconv.Itoa(len(b)))
	w.Write(b)
}

func Registry(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		getService(w, r)
	case "POST":
		addService(w, r)
	case "DELETE":
		delService(w, r)
	}
}
<commit_msg>Return all services when none is specified<commit_after>package handler

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"strconv"

	"github.com/micro/go-micro/cmd"
	"github.com/micro/go-micro/registry"
)

func addService(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	defer r.Body.Close()

	var service *registry.Service
	err = json.Unmarshal(b, &service)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	err = (*cmd.DefaultOptions().Registry).Register(service)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
}

func delService(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	defer r.Body.Close()

	var service *registry.Service
	err = json.Unmarshal(b, &service)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	err = (*cmd.DefaultOptions().Registry).Deregister(service)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
}

func getService(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	service := r.Form.Get("service")

	var s []*registry.Service
	var err error

	if len(service) == 0 {
		s, err = (*cmd.DefaultOptions().Registry).ListServices()
	} else {
		s, err = (*cmd.DefaultOptions().Registry).GetService(service)
	}

	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	if s == nil || (len(service) > 0 && (len(s) == 0 || len(s[0].Name) == 0)) {
		http.Error(w, "Service not found", 404)
		return
	}

	b, err := json.Marshal(s)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("Content-Length", strconv.Itoa(len(b)))
	w.Write(b)
}

func Registry(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		getService(w, r)
	case "POST":
		addService(w, r)
	case "DELETE":
		delService(w, r)
	}
}
<|endoftext|>


<commit_before>// Copyright 2020 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package shaderir

type Program struct {
	Uniforms     []Type
	Attributes   []Type
	Varyings     []Type
	Funcs        []Func
	VertexFunc   VertexFunc
	FragmentFunc FragmentFunc

	structNames map[string]string
	structTypes []Type
}

// TODO: How to avoid the name with existing functions?

type Func struct {
	Name        string
	InParams    []Type
	InOutParams []Type
	OutParams   []Type
	Return      Type
	Block       Block
}

// VertexFunc takes pseudo params, and the number if len(attributes) + len(varyings) + 1.
// If 0 <= index < len(attributes), the params are in-params and treated as attribute variables.
// If len(attributes) <= index < len(attributes) + len(varyings), the params are out-params and treated as varying
// variables.
// The last param represents the position in vec4 (gl_Position in GLSL).
type VertexFunc struct {
	Block Block
}

// FragmentFunc takes pseudo in-params, and the number is len(varyings) + 1.
// The last param represents the coordinate of the fragment (gl_FragCoord in GLSL)
type FragmentFunc struct {
	Block Block
}

type Block struct {
	LocalVars []Type
	Stmts     []Stmt
}

type Stmt struct {
	Type     StmtType
	Exprs    []Expr
	Blocks   []Block
	ForInit  int
	ForEnd   int
	ForOp    Op
	ForDelta int
}

type StmtType int

const (
	ExprStmt StmtType = iota
	BlockStmt
	Assign
	If
	For
	Continue
	Break
	Return
	Discard
)

type Expr struct {
	Type     ExprType
	Exprs    []Expr
	Variable Variable
	Int      int32
	Float    float32
	Ident    string
	Op       Op
}

type ExprType int

const (
	IntExpr ExprType = iota
	FloatExpr
	VarName
	Ident
	Unary
	Binary
	Selection
	Call
	FieldSelector
	Index
)

type Variable struct {
	Type  VariableType
	Index int
}

type VariableType int

const (
	Uniform VariableType = iota
	Local
)

type Op string

const (
	Add          Op = "+"
	Sub          Op = "-"
	Neg          Op = "!"
	Mul          Op = "*"
	Div          Op = "/"
	Mod          Op = "%"
	LeftShift    Op = "<<"
	RightShift   Op = ">>"
	LessThan     Op = "<"
	LessEqual    Op = "<="
	GreaterThan  Op = ">"
	GreaterEqual Op = ">="
	Equal        Op = "=="
	NotEqual     Op = "!="
	And          Op = "&"
	Xor          Op = "^"
	Or           Op = "|"
	AndAnd       Op = "&&"
	OrOr         Op = "||"
)
<commit_msg>shaderir: Add comments<commit_after>// Copyright 2020 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package shaderir offers intermediate representation for shader programs.
package shaderir

type Program struct {
	Uniforms     []Type
	Attributes   []Type
	Varyings     []Type
	Funcs        []Func
	VertexFunc   VertexFunc
	FragmentFunc FragmentFunc

	structNames map[string]string
	structTypes []Type
}

// TODO: How to avoid the name with existing functions?

type Func struct {
	Name        string
	InParams    []Type
	InOutParams []Type
	OutParams   []Type
	Return      Type
	Block       Block
}

// VertexFunc takes pseudo params, and the number if len(attributes) + len(varyings) + 1.
// If 0 <= index < len(attributes), the params are in-params and treated as attribute variables.
// If len(attributes) <= index < len(attributes) + len(varyings), the params are out-params and treated as varying
// variables.
// The last param represents the position in vec4 (gl_Position in GLSL).
type VertexFunc struct {
	Block Block
}

// FragmentFunc takes pseudo in-params, and the number is len(varyings) + 1.
// The last param represents the coordinate of the fragment (gl_FragCoord in GLSL)
type FragmentFunc struct {
	Block Block
}

type Block struct {
	LocalVars []Type
	Stmts     []Stmt
}

type Stmt struct {
	Type     StmtType
	Exprs    []Expr
	Blocks   []Block
	ForInit  int
	ForEnd   int
	ForOp    Op
	ForDelta int
}

type StmtType int

const (
	ExprStmt StmtType = iota
	BlockStmt
	Assign
	If
	For
	Continue
	Break
	Return
	Discard
)

type Expr struct {
	Type     ExprType
	Exprs    []Expr
	Variable Variable
	Int      int32
	Float    float32
	Ident    string
	Op       Op
}

type ExprType int

const (
	IntExpr ExprType = iota
	FloatExpr
	VarName
	Ident
	Unary
	Binary
	Selection
	Call
	FieldSelector
	Index
)

type Variable struct {
	Type  VariableType
	Index int
}

type VariableType int

const (
	Uniform VariableType = iota
	Local
)

type Op string

const (
	Add          Op = "+"
	Sub          Op = "-"
	Neg          Op = "!"
	Mul          Op = "*"
	Div          Op = "/"
	Mod          Op = "%"
	LeftShift    Op = "<<"
	RightShift   Op = ">>"
	LessThan     Op = "<"
	LessEqual    Op = "<="
	GreaterThan  Op = ">"
	GreaterEqual Op = ">="
	Equal        Op = "=="
	NotEqual     Op = "!="
	And          Op = "&"
	Xor          Op = "^"
	Or           Op = "|"
	AndAnd       Op = "&&"
	OrOr         Op = "||"
)
<|endoftext|>


<commit_before>// Package adhier provides an algorithm for adaptive hierarchical interpolation
// with local refinements.
package adhier

import (
	"runtime"
	"sync"
)

// Grid is a sparse grid in [0, 1]^n.
type Grid interface {
	Compute(indices []uint64) []float64
	ComputeChildren(indices []uint64, dimensions []bool) []uint64
}

// Basis is a functional basis in [0, 1]^n.
type Basis interface {
	Compute(index []uint64, point []float64) float64
	Integrate(index []uint64) float64
}

// Interpolator represents a particular instantiation of the algorithm.
type Interpolator struct {
	grid   Grid
	basis  Basis
	config Config
}

// New creates an instance of the algorithm for the given configuration.
func New(grid Grid, basis Basis, config *Config) *Interpolator {
	interpolator := &Interpolator{
		grid:   grid,
		basis:  basis,
		config: *config,
	}

	config = &interpolator.config
	if config.Workers == 0 {
		config.Workers = uint(runtime.GOMAXPROCS(0))
	}

	return interpolator
}

// Compute constructs an interpolant for a quantity of interest.
func (self *Interpolator) Compute(target Target) *Surrogate {
	config := &self.config

	ni, no := target.Dimensions()

	surrogate := new(Surrogate)
	surrogate.initialize(ni, no)

	// Level 0 is assumed to have only one node, and the order of that node is
	// assumed to be zero.
	level := uint(0)

	na := uint(1) // active
	np := uint(0) // passive

	indices := make([]uint64, na*ni)

	var i uint
	var nodes, values, approximations, surpluses []float64
	var refine []bool

	for {
		target.Monitor(level, np, na)

		surrogate.resize(np + na)
		copy(surrogate.Indices[np*ni:], indices)

		nodes = self.grid.Compute(indices)

		values = invoke(target.Compute, nodes, ni, no, config.Workers)
		approximations = approximate(self.basis, surrogate.Indices[:np*ni],
			surrogate.Surpluses[:np*no], nodes, ni, no, config.Workers)

		surpluses = surrogate.Surpluses[np*no : (np+na)*no]
		for i = 0; i < na*no; i++ {
			surpluses[i] = values[i] - approximations[i]
		}

		if level >= config.MaxLevel || (np+na) >= config.MaxNodes {
			break
		}

		refine = make([]bool, na*ni)
		if level < config.MinLevel {
			for i = 0; i < na*ni; i++ {
				refine[i] = true
			}
		} else {
			for i = 0; i < na; i++ {
				target.Refine(surpluses[i*no:(i+1)*no], refine[i*ni:(i+1)*ni])
			}
		}

		indices = self.grid.ComputeChildren(indices, refine)

		np += na
		na = uint(len(indices)) / ni

		// Trim if there are excessive nodes.
		if Δ := int32(np+na) - int32(config.MaxNodes); Δ > 0 {
			na -= uint(Δ)
			indices = indices[:na*ni]
		}

		if na == 0 {
			break
		}

		level++
	}

	surrogate.finalize(level, np+na)
	return surrogate
}

// Evaluate computes the values of a surrogate at a set of points.
func (self *Interpolator) Evaluate(surrogate *Surrogate, points []float64) []float64 {
	return approximate(self.basis, surrogate.Indices, surrogate.Surpluses, points,
		surrogate.Inputs, surrogate.Outputs, self.config.Workers)
}

// Integrate computes the integral of a surrogate over [0, 1]^n.
func (self *Interpolator) Integrate(surrogate *Surrogate) []float64 {
	basis, indices, surpluses := self.basis, surrogate.Indices, surrogate.Surpluses

	ni, no := surrogate.Inputs, surrogate.Outputs
	nn := uint(len(indices)) / ni

	value := make([]float64, no)

	for i := uint(0); i < nn; i++ {
		weight := basis.Integrate(indices[i*ni : (i+1)*ni])
		if weight == 0 {
			continue
		}
		for j := uint(0); j < no; j++ {
			value[j] += weight * surpluses[i*no+j]
		}
	}

	return value
}

func approximate(basis Basis, indices []uint64, surpluses, points []float64,
	ni, no, nw uint) []float64 {

	nn, np := uint(len(indices))/ni, uint(len(points))/ni

	values := make([]float64, np*no)

	jobs := make(chan uint, np)
	group := sync.WaitGroup{}
	group.Add(int(np))

	for i := uint(0); i < nw; i++ {
		go func() {
			for j := range jobs {
				point := points[j*ni : (j+1)*ni]
				value := values[j*no : (j+1)*no]

				for k := uint(0); k < nn; k++ {
					weight := basis.Compute(indices[k*ni:(k+1)*ni], point)
					if weight == 0 {
						continue
					}
					for l := uint(0); l < no; l++ {
						value[l] += weight * surpluses[k*no+l]
					}
				}

				group.Done()
			}
		}()
	}

	for i := uint(0); i < np; i++ {
		jobs <- i
	}

	group.Wait()
	close(jobs)

	return values
}

func invoke(compute func([]float64, []float64), nodes []float64, ni, no, nw uint) []float64 {
	nn := uint(len(nodes)) / ni

	values := make([]float64, nn*no)

	jobs := make(chan uint, nn)
	group := sync.WaitGroup{}
	group.Add(int(nn))

	for i := uint(0); i < nw; i++ {
		go func() {
			for j := range jobs {
				compute(nodes[j*ni:(j+1)*ni], values[j*no:(j+1)*no])
				group.Done()
			}
		}()
	}

	for i := uint(0); i < nn; i++ {
		jobs <- i
	}

	group.Wait()
	close(jobs)

	return values
}
<commit_msg>adhier: eliminate a couple of vars<commit_after>// Package adhier provides an algorithm for adaptive hierarchical interpolation
// with local refinements.
package adhier

import (
	"runtime"
	"sync"
)

// Grid is a sparse grid in [0, 1]^n.
type Grid interface {
	Compute(indices []uint64) []float64
	ComputeChildren(indices []uint64, dimensions []bool) []uint64
}

// Basis is a functional basis in [0, 1]^n.
type Basis interface {
	Compute(index []uint64, point []float64) float64
	Integrate(index []uint64) float64
}

// Interpolator represents a particular instantiation of the algorithm.
type Interpolator struct {
	grid   Grid
	basis  Basis
	config Config
}

// New creates an instance of the algorithm for the given configuration.
func New(grid Grid, basis Basis, config *Config) *Interpolator {
	interpolator := &Interpolator{
		grid:   grid,
		basis:  basis,
		config: *config,
	}

	config = &interpolator.config
	if config.Workers == 0 {
		config.Workers = uint(runtime.GOMAXPROCS(0))
	}

	return interpolator
}

// Compute constructs an interpolant for a quantity of interest.
func (self *Interpolator) Compute(target Target) *Surrogate {
	config := &self.config

	ni, no := target.Dimensions()

	surrogate := new(Surrogate)
	surrogate.initialize(ni, no)

	// Level 0 is assumed to have only one node, and the order of that node is
	// assumed to be zero.
	level := uint(0)

	na := uint(1) // active
	np := uint(0) // passive

	indices := make([]uint64, na*ni)

	var i uint

	for {
		target.Monitor(level, np, na)

		surrogate.resize(np + na)
		copy(surrogate.Indices[np*ni:], indices)

		nodes := self.grid.Compute(indices)

		values := invoke(target.Compute, nodes, ni, no, config.Workers)
		approximations := approximate(self.basis, surrogate.Indices[:np*ni],
			surrogate.Surpluses[:np*no], nodes, ni, no, config.Workers)

		surpluses := surrogate.Surpluses[np*no : (np+na)*no]
		for i = 0; i < na*no; i++ {
			surpluses[i] = values[i] - approximations[i]
		}

		if level >= config.MaxLevel || (np+na) >= config.MaxNodes {
			break
		}

		refine := make([]bool, na*ni)
		if level < config.MinLevel {
			for i = 0; i < na*ni; i++ {
				refine[i] = true
			}
		} else {
			for i = 0; i < na; i++ {
				target.Refine(surpluses[i*no:(i+1)*no], refine[i*ni:(i+1)*ni])
			}
		}

		indices = self.grid.ComputeChildren(indices, refine)

		np += na
		na = uint(len(indices)) / ni

		// Trim if there are excessive nodes.
		if Δ := int32(np+na) - int32(config.MaxNodes); Δ > 0 {
			na -= uint(Δ)
			indices = indices[:na*ni]
		}

		if na == 0 {
			break
		}

		level++
	}

	surrogate.finalize(level, np+na)
	return surrogate
}

// Evaluate computes the values of a surrogate at a set of points.
func (self *Interpolator) Evaluate(surrogate *Surrogate, points []float64) []float64 {
	return approximate(self.basis, surrogate.Indices, surrogate.Surpluses, points,
		surrogate.Inputs, surrogate.Outputs, self.config.Workers)
}

// Integrate computes the integral of a surrogate over [0, 1]^n.
func (self *Interpolator) Integrate(surrogate *Surrogate) []float64 {
	basis, indices, surpluses := self.basis, surrogate.Indices, surrogate.Surpluses

	ni, no := surrogate.Inputs, surrogate.Outputs
	nn := uint(len(indices)) / ni

	value := make([]float64, no)

	for i := uint(0); i < nn; i++ {
		weight := basis.Integrate(indices[i*ni : (i+1)*ni])
		if weight == 0 {
			continue
		}
		for j := uint(0); j < no; j++ {
			value[j] += weight * surpluses[i*no+j]
		}
	}

	return value
}

func approximate(basis Basis, indices []uint64, surpluses, points []float64,
	ni, no, nw uint) []float64 {

	nn, np := uint(len(indices))/ni, uint(len(points))/ni

	values := make([]float64, np*no)

	jobs := make(chan uint, np)
	group := sync.WaitGroup{}
	group.Add(int(np))

	for i := uint(0); i < nw; i++ {
		go func() {
			for j := range jobs {
				point := points[j*ni : (j+1)*ni]
				value := values[j*no : (j+1)*no]

				for k := uint(0); k < nn; k++ {
					weight := basis.Compute(indices[k*ni:(k+1)*ni], point)
					if weight == 0 {
						continue
					}
					for l := uint(0); l < no; l++ {
						value[l] += weight * surpluses[k*no+l]
					}
				}

				group.Done()
			}
		}()
	}

	for i := uint(0); i < np; i++ {
		jobs <- i
	}

	group.Wait()
	close(jobs)

	return values
}

func invoke(compute func([]float64, []float64), nodes []float64, ni, no, nw uint) []float64 {
	nn := uint(len(nodes)) / ni

	values := make([]float64, nn*no)

	jobs := make(chan uint, nn)
	group := sync.WaitGroup{}
	group.Add(int(nn))

	for i := uint(0); i < nw; i++ {
		go func() {
			for j := range jobs {
				compute(nodes[j*ni:(j+1)*ni], values[j*no:(j+1)*no])
				group.Done()
			}
		}()
	}

	for i := uint(0); i < nn; i++ {
		jobs <- i
	}

	group.Wait()
	close(jobs)

	return values
}
<|endoftext|>


<commit_before>// Copyright 2016 Politecnico di Torino
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package l2switch

var SwitchSecurityPolicy = `
#define BPF_TRACE

#define IP_SECURITY_INGRESS
#define MAC_SECURITY_INGRESS
#undef MAC_SECURITY_EGRESS

#define MAX_PORTS 32

struct mac_t {
  u64 mac;
};

struct interface {
  u32 ifindex;
};

struct ifindex{
  u32 ifindex;
};

struct ip_leaf{
  u32 ip;
};

/*
  The Forwarding Table (fwdtable) contains the association between mac Addresses
  and	ports learned by the switch in the learning phase.
  This table is used also in the forwarding phase when the switch has to decide
  the port to use for forwarding the packet.
  The interface number uses the convention of hover, so is an incremental number
  given by hover daemon each time a port is attached to the IOModule (1, 2,..).
*/
BPF_TABLE("hash", struct mac_t, struct interface, fwdtable, 10240);

/*
  The Ports Table (ports) is a fixed length array that identifies the fd (file
  descriptors) of the network interfaces attached to the switch.
  This is a workaround for broadcast implementation, in order to be able to call
  bpf_clone_redirect that accepts as parameter the fd of the network interface.
  This array is not ordered. The index of the array does NOT represent the
  interface number.
*/
BPF_TABLE("array", u32, u32, ports, MAX_PORTS);

/*
  The Security Mac Table (securitymac) associate to each port the allowed mac
  address. If no entry is associated with the port, the port security is not
  applied to the port.
*/
BPF_TABLE("hash", struct ifindex, struct mac_t, securitymac, MAX_PORTS + 1);

/*
  The Security Ip Table (securityip) associate to each port the allowed ip
  address. If no entry is associated with the port, the port security is not
  applied to the port.
*/
BPF_TABLE("hash", struct ifindex, struct ip_leaf, securityip, MAX_PORTS + 1);

static int handle_rx(void *skb, struct metadata *md) {
  u8 *cursor = 0;
  struct ethernet_t *ethernet = cursor_advance(cursor, sizeof(*ethernet));

  #ifdef BPF_TRACE
    bpf_trace_printk("[switch-%d]: in_ifc=%d\n", md->module_id, md->in_ifc);
  #endif

  //set in-interface for lookup ports security
  struct ifindex in_iface = {};
  in_iface.ifindex = md->in_ifc;

  //port security on source mac
  #ifdef MAC_SECURITY_INGRESS
  struct mac_t * mac_lookup;
  mac_lookup = securitymac.lookup(&in_iface);
  if (mac_lookup)
    if (ethernet->src != mac_lookup->mac) {
      #ifdef BPF_TRACE
        bpf_trace_printk("[switch-%d]: mac INGRESS %lx mismatch %lx -> DROP\n",
          md->module_id, ethernet->src, mac_lookup->mac);
      #endif
      return RX_DROP;
    }
  #endif

  //port security on source ip
  #ifdef IP_SECURITY_INGRESS
  if (ethernet->type == 0x0800) {
    struct ip_leaf *ip_lookup;
    ip_lookup = securityip.lookup(&in_iface);
    if (ip_lookup) {
      struct ip_t *ip = cursor_advance(cursor, sizeof(*ip));
      if (ip->src != ip_lookup->ip) {
        #ifdef BPF_TRACE
          bpf_trace_printk("[switch-%d]: IP INGRESS %x mismatch %x -> DROP\n", md->module_id, ip->src, ip_lookup->ip);
        #endif
        return RX_DROP;
      }
    }
  }
  #endif

  #ifdef BPF_TRACE
    bpf_trace_printk("[switch-%d]: mac src:%lx dst:%lx\n", md->module_id, ethernet->src, ethernet->dst);
  #endif

  //LEARNING PHASE: mapping in_iface with src_interface
  struct mac_t src_key = {};
  struct interface interface = {};

  //set in_iface as key
  src_key.mac = ethernet->src;

  //set in_ifc, and 0 counters as leaf
  interface.ifindex = md->in_ifc;

  //lookup in fwdtable. if no key present initialize with interface
  struct interface *interface_lookup = fwdtable.lookup_or_init(&src_key, &interface);

  //if the same mac has changed interface, update it
  if (interface_lookup->ifindex != md->in_ifc)
    interface_lookup->ifindex = md->in_ifc;

  //FORWARDING PHASE: select interface(s) to send the packet
  struct mac_t dst_mac = {ethernet->dst};

  //lookup in forwarding table fwdtable
  struct interface *dst_interface = fwdtable.lookup(&dst_mac);

  if (dst_interface) {
    //HIT in forwarding table
    //redirect packet to dst_interface

    #ifdef MAC_SECURITY_EGRESS
    struct mac_t * mac_lookup;
    struct ifindex out_iface = {};
    out_iface.ifindex = dst_interface->ifindex;
    mac_lookup = securitymac.lookup(&out_iface);
    if (mac_lookup)
      if (ethernet->dst != mac_lookup->mac){
        #ifdef BPF_TRACE
          bpf_trace_printk("[switch-%d]: mac EGRESS %lx mismatch %lx -> DROP\n",
            md->module_id, ethernet->dst, mac_lookup->mac);
        #endif
        return RX_DROP;
      }
    #endif

    pkt_redirect(skb, md, dst_interface->ifindex);

    #ifdef BPF_TRACE
      bpf_trace_printk("[switch-%d]: redirect out_ifc=%d\n", md->module_id, dst_interface->ifindex);
    #endif

    return RX_REDIRECT;

  } else {
    //MISS in forwarding table
    #ifdef BPF_TRACE
      bpf_trace_printk("[switch-%d]s: broadcast\n", md->module_id);
    #endif

    /* this loop broadcasts the packet to the standard network interfaces, the
     * code that is after the loop broadcast the packet to a single iomodule
     */
    u32 i = 0;
    u32 t;
    #pragma unroll
    for (i = 0; i < MAX_PORTS - 1; i++) {
      u32 *iface_p;
      // For some reason the compiler does not unroll the loop if the 'i'
      // variable is used in the lookup function
      t = i;
      iface_p = ports.lookup(&t);

      if (iface_p)
        if (*iface_p != 0 && *iface_p != md->in_ifc)
          bpf_clone_redirect(skb, *iface_p, 0);
    }

    /* the last slot in the ports array is reserved for connections to other
     * iomodules.  Due to the hover architecture in order to send a packet to
     * other iomodule the pkt_redirect() function has to be used, this function
     * internally produces a tail call for the iomodule.
     */
    u32 last = MAX_PORTS - 1;
    u32 *iface_p = ports.lookup(&last);
    if (iface_p)
      if (*iface_p != 0 && *iface_p != md->in_ifc) {
        bpf_trace_printk("[switch-%d]: broadcast to IOModule\n", md->module_id);
        pkt_redirect(skb, md, *iface_p);
        return RX_REDIRECT;
      }

    return RX_DROP;
  }
}
`
<commit_msg>iomodules/switch: do not send packt back on ingress interface<commit_after>// Copyright 2016 Politecnico di Torino
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package l2switch

var SwitchSecurityPolicy = `
#define BPF_TRACE

#define IP_SECURITY_INGRESS
#define MAC_SECURITY_INGRESS
#undef MAC_SECURITY_EGRESS

#define MAX_PORTS 32

struct mac_t {
  u64 mac;
};

struct interface {
  u32 ifindex;
};

struct ifindex{
  u32 ifindex;
};

struct ip_leaf{
  u32 ip;
};

/*
  The Forwarding Table (fwdtable) contains the association between mac Addresses
  and	ports learned by the switch in the learning phase.
  This table is used also in the forwarding phase when the switch has to decide
  the port to use for forwarding the packet.
  The interface number uses the convention of hover, so is an incremental number
  given by hover daemon each time a port is attached to the IOModule (1, 2,..).
*/
BPF_TABLE("hash", struct mac_t, struct interface, fwdtable, 10240);

/*
  The Ports Table (ports) is a fixed length array that identifies the fd (file
  descriptors) of the network interfaces attached to the switch.
  This is a workaround for broadcast implementation, in order to be able to call
  bpf_clone_redirect that accepts as parameter the fd of the network interface.
  This array is not ordered. The index of the array does NOT represent the
  interface number.
*/
BPF_TABLE("array", u32, u32, ports, MAX_PORTS);

/*
  The Security Mac Table (securitymac) associate to each port the allowed mac
  address. If no entry is associated with the port, the port security is not
  applied to the port.
*/
BPF_TABLE("hash", struct ifindex, struct mac_t, securitymac, MAX_PORTS + 1);

/*
  The Security Ip Table (securityip) associate to each port the allowed ip
  address. If no entry is associated with the port, the port security is not
  applied to the port.
*/
BPF_TABLE("hash", struct ifindex, struct ip_leaf, securityip, MAX_PORTS + 1);

static int handle_rx(void *skb, struct metadata *md) {
  u8 *cursor = 0;
  struct ethernet_t *ethernet = cursor_advance(cursor, sizeof(*ethernet));

  #ifdef BPF_TRACE
    bpf_trace_printk("[switch-%d]: in_ifc=%d\n", md->module_id, md->in_ifc);
  #endif

  //set in-interface for lookup ports security
  struct ifindex in_iface = {};
  in_iface.ifindex = md->in_ifc;

  //port security on source mac
  #ifdef MAC_SECURITY_INGRESS
  struct mac_t * mac_lookup;
  mac_lookup = securitymac.lookup(&in_iface);
  if (mac_lookup)
    if (ethernet->src != mac_lookup->mac) {
      #ifdef BPF_TRACE
        bpf_trace_printk("[switch-%d]: mac INGRESS %lx mismatch %lx -> DROP\n",
          md->module_id, ethernet->src, mac_lookup->mac);
      #endif
      return RX_DROP;
    }
  #endif

  //port security on source ip
  #ifdef IP_SECURITY_INGRESS
  if (ethernet->type == 0x0800) {
    struct ip_leaf *ip_lookup;
    ip_lookup = securityip.lookup(&in_iface);
    if (ip_lookup) {
      struct ip_t *ip = cursor_advance(cursor, sizeof(*ip));
      if (ip->src != ip_lookup->ip) {
        #ifdef BPF_TRACE
          bpf_trace_printk("[switch-%d]: IP INGRESS %x mismatch %x -> DROP\n", md->module_id, ip->src, ip_lookup->ip);
        #endif
        return RX_DROP;
      }
    }
  }
  #endif

  #ifdef BPF_TRACE
    bpf_trace_printk("[switch-%d]: mac src:%lx dst:%lx\n", md->module_id, ethernet->src, ethernet->dst);
  #endif

  //LEARNING PHASE: mapping in_iface with src_interface
  struct mac_t src_key = {};
  struct interface interface = {};

  //set in_iface as key
  src_key.mac = ethernet->src;

  //set in_ifc, and 0 counters as leaf
  interface.ifindex = md->in_ifc;

  //lookup in fwdtable. if no key present initialize with interface
  struct interface *interface_lookup = fwdtable.lookup_or_init(&src_key, &interface);

  //if the same mac has changed interface, update it
  if (interface_lookup->ifindex != md->in_ifc)
    interface_lookup->ifindex = md->in_ifc;

  //FORWARDING PHASE: select interface(s) to send the packet
  struct mac_t dst_mac = {ethernet->dst};

  //lookup in forwarding table fwdtable
  struct interface *dst_interface = fwdtable.lookup(&dst_mac);

  if (dst_interface) {
    //HIT in forwarding table
    //redirect packet to dst_interface

    #ifdef MAC_SECURITY_EGRESS
    struct mac_t * mac_lookup;
    struct ifindex out_iface = {};
    out_iface.ifindex = dst_interface->ifindex;
    mac_lookup = securitymac.lookup(&out_iface);
    if (mac_lookup)
      if (ethernet->dst != mac_lookup->mac){
        #ifdef BPF_TRACE
          bpf_trace_printk("[switch-%d]: mac EGRESS %lx mismatch %lx -> DROP\n",
            md->module_id, ethernet->dst, mac_lookup->mac);
        #endif
        return RX_DROP;
      }
    #endif

    /* do not send packet back on the ingress interface */
    if (dst_interface->ifindex == md->in_ifc)
      return RX_DROP;

    pkt_redirect(skb, md, dst_interface->ifindex);

    #ifdef BPF_TRACE
      bpf_trace_printk("[switch-%d]: redirect out_ifc=%d\n", md->module_id, dst_interface->ifindex);
    #endif

    return RX_REDIRECT;

  } else {
    //MISS in forwarding table
    #ifdef BPF_TRACE
      bpf_trace_printk("[switch-%d]s: broadcast\n", md->module_id);
    #endif

    /* this loop broadcasts the packet to the standard network interfaces, the
     * code that is after the loop broadcast the packet to a single iomodule
     */
    u32 i = 0;
    u32 t;
    #pragma unroll
    for (i = 0; i < MAX_PORTS - 1; i++) {
      u32 *iface_p;
      // For some reason the compiler does not unroll the loop if the 'i'
      // variable is used in the lookup function
      t = i;
      iface_p = ports.lookup(&t);

      if (iface_p)
        if (*iface_p != 0 && *iface_p != md->in_ifc)
          bpf_clone_redirect(skb, *iface_p, 0);
    }

    /* the last slot in the ports array is reserved for connections to other
     * iomodules.  Due to the hover architecture in order to send a packet to
     * other iomodule the pkt_redirect() function has to be used, this function
     * internally produces a tail call for the iomodule.
     */
    u32 last = MAX_PORTS - 1;
    u32 *iface_p = ports.lookup(&last);
    if (iface_p)
      if (*iface_p != 0 && *iface_p != md->in_ifc) {
        bpf_trace_printk("[switch-%d]: broadcast to IOModule\n", md->module_id);
        pkt_redirect(skb, md, *iface_p);
        return RX_REDIRECT;
      }

    return RX_DROP;
  }
}
`
<|endoftext|>


<commit_before>package wikifier

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
)

var (
	keyNormalizer = regexp.MustCompile(`\W`)
	keySplitter   = regexp.MustCompile(`(.+)_(\d+)`)
)

// Map represents a Key-value dictionary.
// It is a quiki data type as well as the base of many block types.
type Map struct {
	noFormatValues bool
	didParse       bool
	mapList        []*mapListEntry
	*parserBlock
	*variableScope
}

type mapListEntry struct {
	keyTitle string          // displayed key text
	key      string          // actual underlying key
	value    interface{}     // string, block, or mixed []interface{}
	typ      valueType       // value type
	pos      position        // position where the item started
	metas    map[string]bool // metadata
}

func (entry *mapListEntry) setMeta(key string, val bool) {
	if val == false {
		delete(entry.metas, key)
		return
	}
	entry.metas[key] = val
}

func (entry *mapListEntry) meta(key string) bool {
	return entry.metas[key]
}

type mapParser struct {
	key    interface{}
	values []interface{}

	escape        bool
	inValue       bool
	startPos      position
	pos           position
	overwroteKey  interface{}
	overwroteWith interface{}
	appendedKey   interface{}
}

// NewMap creates a new map, given the main block of the page it is to be associated with.
func NewMap(mb block) *Map {
	underlying := &parserBlock{
		openPos:      position{0, 0}, // FIXME
		parentB:      mb,
		parentC:      mb,
		typ:          "map",
		element:      newElement("div", "map"),
		genericCatch: &genericCatch{},
	}
	return &Map{false, false, nil, underlying, newVariableScope()}
}

func newMapBlock(name string, b *parserBlock) block {
	return &Map{false, false, nil, b, newVariableScope()}
}

func (m *Map) parse(page *Page) {

	// already parsed
	if m.didParse {
		return
	}
	m.didParse = true

	p := new(mapParser)
	for _, pc := range m.posContent() {
		p.pos = pc.position

		// infer start position to this one
		if p.startPos.none() {
			p.startPos = pc.position
		}

		switch item := pc.content.(type) {

		// block
		case block:

			if p.inValue {

				// first item
				if len(p.values) == 0 {
					p.startPos = p.pos
				}

				// add item
				p.values = append(p.values, item)

			} else {
				// overwrote a key
				p.overwroteKey = p.key
				p.overwroteWith = item
				p.key = item
			}
			m.warnMaybe(p)

			// parse the block
			item.parse(page)

		case string:
			item = strings.TrimSpace(item)
			if item == "" {
				continue
			}
			for i, c := range item {
				m.handleChar(page, i, p, c)
			}
		}
	}

	// positional warnings
	m.warnMaybe(p)
	keyHR, valueHR := humanReadableValue(p.key), humanReadableValue(p.values)

	// end of map warnings
	if valueHR != "" || p.inValue {
		// looks like we were in the middle of a value
		m.warn(p.pos, "Value "+valueHR+" for key "+keyHR+" not terminated")
	} else if keyHR != "" {
		// we were in the middle of a key
		m.warn(p.pos, "Stray key "+keyHR+" ignored")
	}

}

func (m *Map) handleChar(page *Page, i int, p *mapParser, c rune) {

	if c == ':' && !p.inValue && !p.escape {
		// first colon indicates we're entering a value

		p.inValue = true

	} else if c == '\\' && !p.escape {
		// escape

		p.escape = true

	} else if c == ';' && !p.escape {
		// semicolon indicates termination of a pair

		strKey, isStrKey := p.key.(string)
		keyTitle := ""

		// determine key
		if (isStrKey && strKey == "") || p.key == nil {
			// this is something like
			//		: value; (can be text or block though)

			strKey = "anon_" + strconv.Itoa(i)
			p.key = strKey
			// no keyTitle

		} else if !p.inValue {
			// if there is a key but we aren't in the value,
			// it is something like
			//		value; (can be text or block though)

			// better to prefix text with : for less ambiguity
			if isStrKey && strKey[0] != '-' {
				m.warn(p.pos, "Standalone text should be prefixed with ':")
			}

			p.values = append(p.values, p.key)
			strKey = "anon_" + strconv.Itoa(i)
			p.key = strKey

			// no keyTitle

		} else {
			// otherwise it's a normal key-value pair
			// (can be text or block though)

			// we have to convert this to a string key somehow, so use the address
			if !isStrKey {
				strKey = fmt.Sprintf("%p", p.key)
			}

			// normalize the key for internal use
			strKey = strings.TrimSpace(strKey)
			keyTitle = strKey
			strKey = keyNormalizer.ReplaceAllString(strKey, "_")
			p.key = strKey

		}

		// fix the value
		// this returns either a string, block, HTML, or []interface{} combination
		// strings next to each other are merged; empty strings are removed
		valueToStore := fixValuesForStorage(p.values, page, !m.noFormatValues)

		// if this key exists, rename it to the next available <key>_key_<n>
		for exist, err := m.Get(strKey); exist != nil && err != nil; {
			matches := keySplitter.FindStringSubmatch(strKey)
			keyName, keyNumber := matches[1], matches[2]

			// first one, so make it _2
			if matches == nil {
				strKey += "_2"
				p.key = strKey
				continue
			}

			// it has _n, so increment that
			newKeyNumber, _ := strconv.Atoi(keyNumber)
			newKeyNumber++
			strKey = keyName + "_" + strconv.Itoa(newKeyNumber)
			p.key = strKey

		}

		// store the value in the underlying variableScope
		m.Set(strKey, valueToStore)

		// store the value in the map list
		m.mapList = append(m.mapList, &mapListEntry{
			keyTitle: keyTitle,                   // displayed key
			value:    valueToStore,               // string, block, or mixed []interface{}
			typ:      getValueType(valueToStore), // type of value
			key:      strKey,                     // actual underlying key
			pos:      p.startPos,                 // position where the item started
		})

		// check for warnings once more
		m.warnMaybe(p)

		// reset status
		p.inValue = false
		p.key = nil
		p.values = nil

	} else {
		// any other character; add to key or value

		// if it was escaped but not a parser char, add the \
		add := string(c)
		if p.escape && c != ';' && c != ':' && c != '\\' {
			add = "\\" + add
		}
		p.escape = false

		// this is part of the value
		if p.inValue {

			// first item
			if len(p.values) == 0 {
				p.startPos = p.pos
				p.values = append(p.values, add)
				return
			}

			// check previous item
			last := p.values[len(p.values)-1]
			if lastStr, ok := last.(string); ok {
				// previous item was a string, so append it

				p.values[len(p.values)-1] = lastStr + add
			} else {
				// previous item was not a string,
				// so start a new string item

				p.values = append(p.values, add)
			}

			return
		}

		// this is part of the key

		// starting a new key
		if p.key == nil {
			p.startPos = p.pos
			p.key = add
			return
		}

		// check current key
		if lastStr, ok := p.key.(string); ok {
			// already working on a string key, so append it

			p.key = lastStr + add
		} else if strings.TrimSpace(add) != "" {
			// previous item was not a string
			// trying to add text to a non-text key...
			// (above ignores whitespace chars)

			p.appendedKey = p.key
		}
	}
}

// getEntry fetches the MapListEntry for a key.
func (m *Map) getEntry(key string) *mapListEntry {
	for _, entry := range m.mapList {
		if entry.key == key {
			return entry
		}
	}
	return nil
}

// getKeyPos returns the position where a key started.
// If the key doesn't exist, it returns the position where the map started.
func (m *Map) getKeyPos(key string) position {
	if entry := m.getEntry(key); entry != nil {
		return entry.pos
	}
	return m.openPos
}

// produce warnings as needed at current parser state
func (m *Map) warnMaybe(p *mapParser) {
	hrKey := humanReadableValue(p.key)

	// string keys spanning multiple lines are fishy
	if strKey, ok := p.key.(string); ok && strings.ContainsRune(strKey, '\n') {
		m.warn(p.pos, "Suspicious key "+hrKey)
	}

	// tried to append an object key
	if p.appendedKey != nil {
		appendText := humanReadableValue(p.appendedKey)
		m.warn(p.pos, "Stray text after "+appendText+" ignored")
		p.appendedKey = nil
	}

	// overwrote a key
	if p.overwroteKey != nil {
		old := humanReadableValue(p.overwroteKey)
		new := humanReadableValue(p.overwroteWith)
		m.warn(p.pos, "Overwrote "+old+" with "+new)
		p.overwroteKey = nil
		p.overwroteWith = nil
	}
}

// default behavior for maps is to run html() on all children
// and replace the block value in the map with the generated element
func (m *Map) html(page *Page, el element) {
	if m.noFormatValues {
		return
	}
	for i, entry := range m.mapList {
		m.mapList[i].value = prepareForHTML(entry.value, page, entry.pos)
		m.setOwn(entry.key, m.mapList[i].value)
	}
}

// since maps can be stored in variables and are generated on the fly,
// we sometimes need the main block to associate them with
func (m *Map) mainBlock() block {
	var b block = m
	for b.parentBlock() != nil {
		b = b.parentBlock()
	}
	return b
}

// Map returns the actual underlying Go map.
func (m *Map) Map() map[string]interface{} {
	return m.vars
}

// Keys returns a string of actual underlying map keys.
func (m *Map) Keys() []string {
	keys := make([]string, len(m.vars))
	i := 0
	for key := range m.vars {
		keys[i] = key
		i++
	}
	return keys
}

// OrderedKeys returns a string of map keys in the order
// provided in the source. Keys that were set internally
// (and not from quiki source code) are omitted.
func (m *Map) OrderedKeys() []string {
	keys := make([]string, len(m.mapList))
	i := 0
	for _, entry := range m.mapList {
		keys[i] = entry.key
		i++
	}
	return keys
}
<commit_msg>Update block-map.go<commit_after>package wikifier

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
)

var (
	keyNormalizer = regexp.MustCompile(`\W`)
	keySplitter   = regexp.MustCompile(`(.+)_(\d+)`)
)

// Map represents a Key-value dictionary.
// It is a quiki data type as well as the base of many block types.
type Map struct {
	noFormatValues bool
	didParse       bool
	mapList        []*mapListEntry
	*parserBlock
	*variableScope
}

type mapListEntry struct {
	keyTitle string          // displayed key text
	key      string          // actual underlying key
	value    interface{}     // string, block, or mixed []interface{}
	typ      valueType       // value type
	pos      position        // position where the item started
	metas    map[string]bool // metadata
}

func (entry *mapListEntry) setMeta(key string, val bool) {
	if val == false {
		delete(entry.metas, key)
		return
	}
	entry.metas[key] = val
}

func (entry *mapListEntry) meta(key string) bool {
	return entry.metas[key]
}

type mapParser struct {
	key    interface{}
	values []interface{}

	escape        bool
	inValue       bool
	startPos      position
	pos           position
	overwroteKey  interface{}
	overwroteWith interface{}
	appendedKey   interface{}
}

// NewMap creates a new map, given the main block of the page it is to be associated with.
func NewMap(mb block) *Map {
	underlying := &parserBlock{
		openPos:      position{0, 0}, // FIXME
		parentB:      mb,
		parentC:      mb,
		typ:          "map",
		element:      newElement("div", "map"),
		genericCatch: &genericCatch{},
	}
	return &Map{false, false, nil, underlying, newVariableScope()}
}

func newMapBlock(name string, b *parserBlock) block {
	return &Map{false, false, nil, b, newVariableScope()}
}

func (m *Map) parse(page *Page) {

	// already parsed
	if m.didParse {
		return
	}
	m.didParse = true

	p := new(mapParser)
	for i, pc := range m.posContent() {
		p.pos = pc.position

		// infer start position to this one
		if p.startPos.none() {
			p.startPos = pc.position
		}

		switch item := pc.content.(type) {

		// block
		case block:

			if p.inValue {

				// first item
				if len(p.values) == 0 {
					p.startPos = p.pos
				}

				// add item
				p.values = append(p.values, item)

			} else {
				// overwrote a key
				p.overwroteKey = p.key
				p.overwroteWith = item
				p.key = item
			}
			m.warnMaybe(p)

			// parse the block
			item.parse(page)

		case string:
			item = strings.TrimSpace(item)
			if item == "" {
				continue
			}
			for _, c := range item {
				m.handleChar(page, i, p, c)
			}
		}
	}

	// positional warnings
	m.warnMaybe(p)
	keyHR, valueHR := humanReadableValue(p.key), humanReadableValue(p.values)

	// end of map warnings
	if valueHR != "" || p.inValue {
		// looks like we were in the middle of a value
		m.warn(p.pos, "Value "+valueHR+" for key "+keyHR+" not terminated")
	} else if keyHR != "" {
		// we were in the middle of a key
		m.warn(p.pos, "Stray key "+keyHR+" ignored")
	}

}

func (m *Map) handleChar(page *Page, i int, p *mapParser, c rune) {

	if c == ':' && !p.inValue && !p.escape {
		// first colon indicates we're entering a value

		p.inValue = true

	} else if c == '\\' && !p.escape {
		// escape

		p.escape = true

	} else if c == ';' && !p.escape {
		// semicolon indicates termination of a pair

		strKey, isStrKey := p.key.(string)
		keyTitle := ""

		// determine key
		if (isStrKey && strKey == "") || p.key == nil {
			// this is something like
			//		: value; (can be text or block though)

			strKey = "anon_" + strconv.Itoa(i)
			p.key = strKey
			// no keyTitle

		} else if !p.inValue {
			// if there is a key but we aren't in the value,
			// it is something like
			//		value; (can be text or block though)

			// better to prefix text with : for less ambiguity
			if isStrKey && strKey[0] != '-' {
				m.warn(p.pos, "Standalone text should be prefixed with ':")
			}

			p.values = append(p.values, p.key)
			strKey = "anon_" + strconv.Itoa(i)
			p.key = strKey

			// no keyTitle

		} else {
			// otherwise it's a normal key-value pair
			// (can be text or block though)

			// we have to convert this to a string key somehow, so use the address
			if !isStrKey {
				strKey = fmt.Sprintf("%p", p.key)
			}

			// normalize the key for internal use
			strKey = strings.TrimSpace(strKey)
			keyTitle = strKey
			strKey = keyNormalizer.ReplaceAllString(strKey, "_")
			p.key = strKey

		}

		// fix the value
		// this returns either a string, block, HTML, or []interface{} combination
		// strings next to each other are merged; empty strings are removed
		valueToStore := fixValuesForStorage(p.values, page, !m.noFormatValues)

		// if this key exists, rename it to the next available <key>_key_<n>
		for exist, err := m.Get(strKey); exist != nil && err != nil; {
			matches := keySplitter.FindStringSubmatch(strKey)
			keyName, keyNumber := matches[1], matches[2]

			// first one, so make it _2
			if matches == nil {
				strKey += "_2"
				p.key = strKey
				continue
			}

			// it has _n, so increment that
			newKeyNumber, _ := strconv.Atoi(keyNumber)
			newKeyNumber++
			strKey = keyName + "_" + strconv.Itoa(newKeyNumber)
			p.key = strKey

		}

		// store the value in the underlying variableScope
		m.Set(strKey, valueToStore)

		// store the value in the map list
		m.mapList = append(m.mapList, &mapListEntry{
			keyTitle: keyTitle,                   // displayed key
			value:    valueToStore,               // string, block, or mixed []interface{}
			typ:      getValueType(valueToStore), // type of value
			key:      strKey,                     // actual underlying key
			pos:      p.startPos,                 // position where the item started
		})

		// check for warnings once more
		m.warnMaybe(p)

		// reset status
		p.inValue = false
		p.key = nil
		p.values = nil

	} else {
		// any other character; add to key or value

		// if it was escaped but not a parser char, add the \
		add := string(c)
		if p.escape && c != ';' && c != ':' && c != '\\' {
			add = "\\" + add
		}
		p.escape = false

		// this is part of the value
		if p.inValue {

			// first item
			if len(p.values) == 0 {
				p.startPos = p.pos
				p.values = append(p.values, add)
				return
			}

			// check previous item
			last := p.values[len(p.values)-1]
			if lastStr, ok := last.(string); ok {
				// previous item was a string, so append it

				p.values[len(p.values)-1] = lastStr + add
			} else {
				// previous item was not a string,
				// so start a new string item

				p.values = append(p.values, add)
			}

			return
		}

		// this is part of the key

		// starting a new key
		if p.key == nil {
			p.startPos = p.pos
			p.key = add
			return
		}

		// check current key
		if lastStr, ok := p.key.(string); ok {
			// already working on a string key, so append it

			p.key = lastStr + add
		} else if strings.TrimSpace(add) != "" {
			// previous item was not a string
			// trying to add text to a non-text key...
			// (above ignores whitespace chars)

			p.appendedKey = p.key
		}
	}
}

// getEntry fetches the MapListEntry for a key.
func (m *Map) getEntry(key string) *mapListEntry {
	for _, entry := range m.mapList {
		if entry.key == key {
			return entry
		}
	}
	return nil
}

// getKeyPos returns the position where a key started.
// If the key doesn't exist, it returns the position where the map started.
func (m *Map) getKeyPos(key string) position {
	if entry := m.getEntry(key); entry != nil {
		return entry.pos
	}
	return m.openPos
}

// produce warnings as needed at current parser state
func (m *Map) warnMaybe(p *mapParser) {
	hrKey := humanReadableValue(p.key)

	// string keys spanning multiple lines are fishy
	if strKey, ok := p.key.(string); ok && strings.ContainsRune(strKey, '\n') {
		m.warn(p.pos, "Suspicious key "+hrKey)
	}

	// tried to append an object key
	if p.appendedKey != nil {
		appendText := humanReadableValue(p.appendedKey)
		m.warn(p.pos, "Stray text after "+appendText+" ignored")
		p.appendedKey = nil
	}

	// overwrote a key
	if p.overwroteKey != nil {
		old := humanReadableValue(p.overwroteKey)
		new := humanReadableValue(p.overwroteWith)
		m.warn(p.pos, "Overwrote "+old+" with "+new)
		p.overwroteKey = nil
		p.overwroteWith = nil
	}
}

// default behavior for maps is to run html() on all children
// and replace the block value in the map with the generated element
func (m *Map) html(page *Page, el element) {
	if m.noFormatValues {
		return
	}
	for i, entry := range m.mapList {
		m.mapList[i].value = prepareForHTML(entry.value, page, entry.pos)
		m.setOwn(entry.key, m.mapList[i].value)
	}
}

// since maps can be stored in variables and are generated on the fly,
// we sometimes need the main block to associate them with
func (m *Map) mainBlock() block {
	var b block = m
	for b.parentBlock() != nil {
		b = b.parentBlock()
	}
	return b
}

// Map returns the actual underlying Go map.
func (m *Map) Map() map[string]interface{} {
	return m.vars
}

// Keys returns a string of actual underlying map keys.
func (m *Map) Keys() []string {
	keys := make([]string, len(m.vars))
	i := 0
	for key := range m.vars {
		keys[i] = key
		i++
	}
	return keys
}

// OrderedKeys returns a string of map keys in the order
// provided in the source. Keys that were set internally
// (and not from quiki source code) are omitted.
func (m *Map) OrderedKeys() []string {
	keys := make([]string, len(m.mapList))
	i := 0
	for _, entry := range m.mapList {
		keys[i] = entry.key
		i++
	}
	return keys
}
<|endoftext|>


<commit_before>package bootstrap

import (
	"fmt"
	"io/ioutil"
	"os"

	"html/template"

	"github.com/tomogoma/authms/api"
	"github.com/tomogoma/authms/config"
	"github.com/tomogoma/authms/db"
	"github.com/tomogoma/authms/facebook"
	"github.com/tomogoma/authms/logging"
	"github.com/tomogoma/authms/model"
	"github.com/tomogoma/authms/sms/africas_talking"
	"github.com/tomogoma/authms/sms/messagebird"
	"github.com/tomogoma/authms/sms/twilio"
	"github.com/tomogoma/authms/smtp"
	"github.com/tomogoma/crdb"
	token "github.com/tomogoma/jwt"
)

func InstantiateRoach(lg logging.Logger, conf crdb.Config) *db.Roach {
	var opts []db.Option
	if dsn := conf.FormatDSN(); dsn != "" {
		opts = append(opts, db.WithDSN(dsn))
	}
	if dbn := conf.DBName; dbn != "" {
		opts = append(opts, db.WithDBName(dbn))
	}
	rdb := db.NewRoach(opts...)
	err := rdb.InitDBIfNot()
	logging.LogWarnOnError(lg, err, "Initiate Cockroach DB connection")
	return rdb
}

func InstantiateJWTHandler(lg logging.Logger, tknKyF string) *token.Handler {
	JWTKey, err := ioutil.ReadFile(tknKyF)
	logging.LogFatalOnError(lg, err, "Read JWT key file")
	jwter, err := token.NewHandler(JWTKey)
	logging.LogFatalOnError(lg, err, "Instantiate JWT handler")
	return jwter
}

func InstantiateFacebook(conf config.Facebook) (*facebook.FacebookOAuth, error) {
	if conf.ID < 1 {
		return nil, nil
	}
	fbSecret, err := readFile(conf.SecretFilePath)
	if err != nil {
		return nil, fmt.Errorf("read facebook secret from file: %v", err)
	}
	fb, err := facebook.New(conf.ID, fbSecret)
	if err != nil {
		return nil, err
	}
	return fb, nil
}

func InstantiateSMSer(conf config.SMS) (model.SMSer, error) {
	if conf.ActiveAPI == "" {
		return nil, nil
	}
	var s model.SMSer
	switch conf.ActiveAPI {
	case config.SMSAPIAfricasTalking:
		apiKey, err := readFile(conf.AfricasTalking.APIKeyFile)
		if err != nil {
			return nil, fmt.Errorf("read africa's talking API key: %v", err)
		}
		s, err = africas_talking.NewSMSCl(conf.AfricasTalking.UserName, apiKey)
		if err != nil {
			return nil, fmt.Errorf("new africasTalking client: %v", err)
		}
	case config.SMSAPITwilio:
		tkn, err := readFile(conf.Twilio.TokenKeyFile)
		if err != nil {
			return nil, fmt.Errorf("read twilio token: %v", err)
		}
		s, err = twilio.NewSMSCl(conf.Twilio.ID, tkn, conf.Twilio.SenderPhone)
		if err != nil {
			return nil, fmt.Errorf("new twilio client: %v", err)
		}
	case config.SMSAPIMessageBird:
		apiKey, err := readFile(conf.MessageBird.APIKeyFile)
		if err != nil {
			return nil, fmt.Errorf("read messageBird API key: %v", err)
		}
		s, err = messagebird.NewClient(conf.MessageBird.AccountName, apiKey)
		if err != nil {
			return nil, fmt.Errorf("new messageBird client: %v", err)
		}
	default:
		return nil, fmt.Errorf("invalid API selected can be %s or %s",
			config.SMSAPIAfricasTalking, config.SMSAPITwilio)
	}
	var testMessage string
	host := hostname()
	testMessage = fmt.Sprintf("The SMS API is being used on %s", host)
	if err := s.SMS(conf.TestNumber, testMessage); err != nil {
		return s, fmt.Errorf("test SMS: %v", err)
	}
	return s, nil
}

func InstantiateSMTP(rdb *db.Roach, lg logging.Logger, conf config.SMTP) *smtp.Mailer {

	emailCl, err := smtp.New(rdb)
	logging.LogFatalOnError(lg, err, "Instantiate email API")

	err = emailCl.Configured()
	if err == nil {
		return emailCl
	}

	if !emailCl.IsNotFoundError(err) {
		logging.LogFatalOnError(lg, err, "Check email API configured")
	}

	pass, err := readFile(conf.PasswordFile)
	logging.LogWarnOnError(lg, err, "Read SMTP password file")

	host := hostname()
	err = emailCl.SetConfig(
		smtp.Config{
			ServerAddress: conf.ServerAddress,
			TLSPort:       conf.TLSPort,
			SSLPort:       conf.SSLPort,
			Username:      conf.Username,
			FromEmail:     conf.FromEmail,
			Password:      pass,
		},
		model.SendMail{
			ToEmails: []string{conf.TestEmail},
			Subject:  "Authentication Micro-Service Started on " + host,
			Body:     template.HTML("The authentication micro-service is being used on " + host),
		},
	)
	logging.LogWarnOnError(lg, err, "Set default SMTP config")

	return emailCl
}

func Instantiate(confFile string, lg logging.Logger) (config.General, *model.Authentication, *api.Guard, *db.Roach, model.JWTEr, model.SMSer, *smtp.Mailer) {

	conf, err := config.ReadFile(confFile)
	logging.LogFatalOnError(lg, err, "Read config file")

	rdb := InstantiateRoach(lg, conf.Database)
	tg := InstantiateJWTHandler(lg, conf.Token.TokenKeyFile)

	var authOpts []model.Option
	fb, err := InstantiateFacebook(conf.Authentication.Facebook)
	logging.LogWarnOnError(lg, err, "Set up OAuth options")
	if fb != nil {
		authOpts = append(authOpts, model.WithFacebookCl(fb))
	}

	sms, err := InstantiateSMSer(conf.SMS)
	logging.LogWarnOnError(lg, err, "Instantiate SMS API")
	if sms != nil {
		authOpts = append(authOpts, model.WithSMSCl(sms))
	}

	emailCl := InstantiateSMTP(rdb, lg, conf.SMTP)
	authOpts = append(authOpts, model.WithEmailCl(emailCl))

	authOpts = append(
		authOpts,
		model.WithAppName(conf.Service.AppName),
		model.WithDevLockedToUser(conf.Authentication.LockDevsToUsers),
		model.WithSelfRegAllowed(conf.Authentication.AllowSelfReg),
	)

	a, err := model.NewAuthentication(rdb, tg, authOpts...)
	logging.LogFatalOnError(lg, err, "Instantiate Auth Model")

	g, err := api.NewGuard(rdb, api.WithMasterKey(conf.Service.MasterAPIKey))
	logging.LogFatalOnError(lg, err, "Instantate API access guard")

	return *conf, a, g, rdb, tg, sms, emailCl
}

func hostname() string {
	hostName, err := os.Hostname()
	if err != nil {
		return "an unknown host"
	}
	return hostName
}

func readFile(path string) (string, error) {
	contentB, err := ioutil.ReadFile(path)
	if err != nil {
		return "", fmt.Errorf("read: %v", err)
	}
	return string(contentB), nil
}
<commit_msg>don't quit on email configuration check error<commit_after>package bootstrap

import (
	"fmt"
	"io/ioutil"
	"os"

	"html/template"

	"github.com/tomogoma/authms/api"
	"github.com/tomogoma/authms/config"
	"github.com/tomogoma/authms/db"
	"github.com/tomogoma/authms/facebook"
	"github.com/tomogoma/authms/logging"
	"github.com/tomogoma/authms/model"
	"github.com/tomogoma/authms/sms/africas_talking"
	"github.com/tomogoma/authms/sms/messagebird"
	"github.com/tomogoma/authms/sms/twilio"
	"github.com/tomogoma/authms/smtp"
	"github.com/tomogoma/crdb"
	token "github.com/tomogoma/jwt"
)

func InstantiateRoach(lg logging.Logger, conf crdb.Config) *db.Roach {
	var opts []db.Option
	if dsn := conf.FormatDSN(); dsn != "" {
		opts = append(opts, db.WithDSN(dsn))
	}
	if dbn := conf.DBName; dbn != "" {
		opts = append(opts, db.WithDBName(dbn))
	}
	rdb := db.NewRoach(opts...)
	err := rdb.InitDBIfNot()
	logging.LogWarnOnError(lg, err, "Initiate Cockroach DB connection")
	return rdb
}

func InstantiateJWTHandler(lg logging.Logger, tknKyF string) *token.Handler {
	JWTKey, err := ioutil.ReadFile(tknKyF)
	logging.LogFatalOnError(lg, err, "Read JWT key file")
	jwter, err := token.NewHandler(JWTKey)
	logging.LogFatalOnError(lg, err, "Instantiate JWT handler")
	return jwter
}

func InstantiateFacebook(conf config.Facebook) (*facebook.FacebookOAuth, error) {
	if conf.ID < 1 {
		return nil, nil
	}
	fbSecret, err := readFile(conf.SecretFilePath)
	if err != nil {
		return nil, fmt.Errorf("read facebook secret from file: %v", err)
	}
	fb, err := facebook.New(conf.ID, fbSecret)
	if err != nil {
		return nil, err
	}
	return fb, nil
}

func InstantiateSMSer(conf config.SMS) (model.SMSer, error) {
	if conf.ActiveAPI == "" {
		return nil, nil
	}
	var s model.SMSer
	switch conf.ActiveAPI {
	case config.SMSAPIAfricasTalking:
		apiKey, err := readFile(conf.AfricasTalking.APIKeyFile)
		if err != nil {
			return nil, fmt.Errorf("read africa's talking API key: %v", err)
		}
		s, err = africas_talking.NewSMSCl(conf.AfricasTalking.UserName, apiKey)
		if err != nil {
			return nil, fmt.Errorf("new africasTalking client: %v", err)
		}
	case config.SMSAPITwilio:
		tkn, err := readFile(conf.Twilio.TokenKeyFile)
		if err != nil {
			return nil, fmt.Errorf("read twilio token: %v", err)
		}
		s, err = twilio.NewSMSCl(conf.Twilio.ID, tkn, conf.Twilio.SenderPhone)
		if err != nil {
			return nil, fmt.Errorf("new twilio client: %v", err)
		}
	case config.SMSAPIMessageBird:
		apiKey, err := readFile(conf.MessageBird.APIKeyFile)
		if err != nil {
			return nil, fmt.Errorf("read messageBird API key: %v", err)
		}
		s, err = messagebird.NewClient(conf.MessageBird.AccountName, apiKey)
		if err != nil {
			return nil, fmt.Errorf("new messageBird client: %v", err)
		}
	default:
		return nil, fmt.Errorf("invalid API selected can be %s or %s",
			config.SMSAPIAfricasTalking, config.SMSAPITwilio)
	}
	var testMessage string
	host := hostname()
	testMessage = fmt.Sprintf("The SMS API is being used on %s", host)
	if err := s.SMS(conf.TestNumber, testMessage); err != nil {
		return s, fmt.Errorf("test SMS: %v", err)
	}
	return s, nil
}

func InstantiateSMTP(rdb *db.Roach, lg logging.Logger, conf config.SMTP) *smtp.Mailer {

	emailCl, err := smtp.New(rdb)
	logging.LogFatalOnError(lg, err, "Instantiate email API")

	err = emailCl.Configured()
	if err == nil {
		return emailCl
	}

	if !emailCl.IsNotFoundError(err) {
		logging.LogWarnOnError(lg, err, "Check email API configured")
		return emailCl
	}

	pass, err := readFile(conf.PasswordFile)
	logging.LogWarnOnError(lg, err, "Read SMTP password file")

	host := hostname()
	err = emailCl.SetConfig(
		smtp.Config{
			ServerAddress: conf.ServerAddress,
			TLSPort:       conf.TLSPort,
			SSLPort:       conf.SSLPort,
			Username:      conf.Username,
			FromEmail:     conf.FromEmail,
			Password:      pass,
		},
		model.SendMail{
			ToEmails: []string{conf.TestEmail},
			Subject:  "Authentication Micro-Service Started on " + host,
			Body:     template.HTML("The authentication micro-service is being used on " + host),
		},
	)
	logging.LogWarnOnError(lg, err, "Set default SMTP config")

	return emailCl
}

func Instantiate(confFile string, lg logging.Logger) (config.General, *model.Authentication, *api.Guard, *db.Roach, model.JWTEr, model.SMSer, *smtp.Mailer) {

	conf, err := config.ReadFile(confFile)
	logging.LogFatalOnError(lg, err, "Read config file")

	rdb := InstantiateRoach(lg, conf.Database)
	tg := InstantiateJWTHandler(lg, conf.Token.TokenKeyFile)

	var authOpts []model.Option
	fb, err := InstantiateFacebook(conf.Authentication.Facebook)
	logging.LogWarnOnError(lg, err, "Set up OAuth options")
	if fb != nil {
		authOpts = append(authOpts, model.WithFacebookCl(fb))
	}

	sms, err := InstantiateSMSer(conf.SMS)
	logging.LogWarnOnError(lg, err, "Instantiate SMS API")
	if sms != nil {
		authOpts = append(authOpts, model.WithSMSCl(sms))
	}

	emailCl := InstantiateSMTP(rdb, lg, conf.SMTP)
	authOpts = append(authOpts, model.WithEmailCl(emailCl))

	authOpts = append(
		authOpts,
		model.WithAppName(conf.Service.AppName),
		model.WithDevLockedToUser(conf.Authentication.LockDevsToUsers),
		model.WithSelfRegAllowed(conf.Authentication.AllowSelfReg),
	)

	a, err := model.NewAuthentication(rdb, tg, authOpts...)
	logging.LogFatalOnError(lg, err, "Instantiate Auth Model")

	g, err := api.NewGuard(rdb, api.WithMasterKey(conf.Service.MasterAPIKey))
	logging.LogFatalOnError(lg, err, "Instantate API access guard")

	return *conf, a, g, rdb, tg, sms, emailCl
}

func hostname() string {
	hostName, err := os.Hostname()
	if err != nil {
		return "an unknown host"
	}
	return hostName
}

func readFile(path string) (string, error) {
	contentB, err := ioutil.ReadFile(path)
	if err != nil {
		return "", fmt.Errorf("read: %v", err)
	}
	return string(contentB), nil
}
<|endoftext|>


<commit_before>package fetch

import (
	"context"
	"math/rand"
	"sync"
	"time"

	"chain/database/sql"
	"chain/errors"
	"chain/log"
	"chain/net/rpc"
	"chain/protocol"
	"chain/protocol/bc"
	"chain/protocol/state"
)

const getBlocksTimeout = 3 * time.Second
const getSnapshotTimeout = 10 * time.Second
const heightPollingPeriod = 3 * time.Second

var (
	generatorHeight          uint64
	generatorHeightFetchedAt time.Time
	generatorLock            sync.Mutex
)

func GeneratorHeight() (uint64, time.Time) {
	generatorLock.Lock()
	h := generatorHeight
	t := generatorHeightFetchedAt
	generatorLock.Unlock()
	return h, t
}

// Fetch runs in a loop, fetching blocks from the configured
// peer (e.g. the generator) and applying them to the local
// Chain.
//
// It returns when its context is canceled.
func Fetch(ctx context.Context, c *protocol.Chain, peer *rpc.Client) {
	// This process just became leader, so it's responsible
	// for recovering after the previous leader's exit.
	prevBlock, prevSnapshot, err := c.Recover(ctx)
	if err != nil {
		log.Fatal(ctx, log.KeyError, err)
	}

	// Fetch the generator height periodically.
	go pollGeneratorHeight(ctx, peer)

	var nfailures uint // for backoff
	for {
		select {
		case <-ctx.Done():
			log.Messagef(ctx, "Deposed, Fetch exiting")
			return
		default:
			var height uint64
			if prevBlock != nil {
				height = prevBlock.Height
			}

			blocks, err := getBlocks(ctx, peer, height)
			if err != nil {
				log.Error(ctx, err)
				nfailures++
				time.Sleep(backoffDur(nfailures))
				continue
			}

			prevSnapshot, prevBlock, err = applyBlocks(ctx, c, prevSnapshot, prevBlock, blocks)
			if err != nil {
				log.Error(ctx, err)
				nfailures++
				time.Sleep(backoffDur(nfailures))
				continue
			}
			nfailures = 0
		}
	}
}

func pollGeneratorHeight(ctx context.Context, peer *rpc.Client) {
	updateGeneratorHeight(ctx, peer)

	ticker := time.NewTicker(heightPollingPeriod)
	for {
		select {
		case <-ctx.Done():
			log.Messagef(ctx, "Deposed, fetchGeneratorHeight exiting")
			ticker.Stop()
			return
		case <-ticker.C:
			updateGeneratorHeight(ctx, peer)
		}
	}
}

func updateGeneratorHeight(ctx context.Context, peer *rpc.Client) {
	gh, err := getHeight(ctx, peer)
	if err != nil {
		log.Error(ctx, err)
		return
	}

	generatorLock.Lock()
	defer generatorLock.Unlock()
	generatorHeight = gh
	generatorHeightFetchedAt = time.Now()
}

func applyBlocks(ctx context.Context, c *protocol.Chain, snap *state.Snapshot, block *bc.Block, blocks []*bc.Block) (*state.Snapshot, *bc.Block, error) {
	for _, b := range blocks {
		ss, err := c.ValidateBlock(ctx, snap, block, b)
		if err != nil {
			// TODO(kr): this is a validation failure.
			// It's either a serious bug or an attack.
			// Do something better than just log the error
			// (in the caller above). Alert a human,
			// the security team, the legal team, the A-team,
			// somebody.
			return snap, block, err
		}

		err = c.CommitBlock(ctx, b, ss)
		if err != nil {
			return snap, block, err
		}

		snap, block = ss, b
	}
	return snap, block, nil
}

func backoffDur(n uint) time.Duration {
	if n > 33 {
		n = 33 // cap to about 10s
	}
	d := rand.Int63n(1 << n)
	return time.Duration(d)
}

// getBlocks sends a get-blocks RPC request to another Core
// for all blocks since the highest-known one.
func getBlocks(ctx context.Context, peer *rpc.Client, height uint64) ([]*bc.Block, error) {
	ctx, cancel := context.WithTimeout(ctx, getBlocksTimeout)
	defer cancel()

	var blocks []*bc.Block
	err := peer.Call(ctx, "/rpc/get-blocks", height, &blocks)
	if ctx.Err() == context.DeadlineExceeded {
		return nil, nil
	}
	return blocks, errors.Wrap(err, "get blocks rpc")
}

// getHeight sends a get-height RPC request to another Core for
// the latest height that that peer knows about.
func getHeight(ctx context.Context, peer *rpc.Client) (uint64, error) {
	var resp map[string]uint64
	err := peer.Call(ctx, "/rpc/block-height", nil, &resp)
	if err != nil {
		return 0, errors.Wrap(err, "could not get remote block height")
	}
	h, ok := resp["block_height"]
	if !ok {
		return 0, errors.New("unexpected response from generator")
	}

	return h, nil
}

// Snapshot fetches the latest snapshot from the generator and applies it to this
// core's snapshot set. It should only be called on freshly configured cores--
// cores that have been operating should replay all transactions so that they can
// index them properly.
func Snapshot(ctx context.Context, peer *rpc.Client, s protocol.Store, db *sql.DB) error {
	ctx, cancel := context.WithTimeout(ctx, getSnapshotTimeout)
	defer cancel()

	var snapResp struct {
		Data   []byte
		Height uint64
	}
	err := peer.Call(ctx, "/rpc/get-snapshot", nil, &snapResp)
	if err != nil {
		return err
	}

	const snapQ = `
		INSERT INTO snapshots (height, data) VALUES ($1, $2)
		ON CONFLICT DO NOTHING
	`
	_, err = db.Exec(ctx, snapQ, snapResp.Height, snapResp.Data)
	if err != nil {
		return err
	}

	// Next, get the genesis block.
	blocks, err := getBlocks(ctx, peer, 0)
	if err != nil {
		return err
	}

	if len(blocks) < 1 {
		// Something seriously funny is afoot.
		return errors.New("could not get initial block from generator")
	}

	err = s.SaveBlock(ctx, blocks[0])

	// Also get the corresponding block.
	blocks, err = getBlocks(ctx, peer, snapResp.Height-1) // because we get the NEXT block
	if err != nil {
		return err
	}

	if len(blocks) < 1 {
		// Something seriously funny is still afoot.
		return errors.New("generator provided snapshot but could not provide block")
	}

	return s.SaveBlock(ctx, blocks[0])
}
<commit_msg>core/fetch: use backoff on get-blocks RPC timeout<commit_after>package fetch

import (
	"context"
	"math/rand"
	"sync"
	"time"

	"chain/database/sql"
	"chain/errors"
	"chain/log"
	"chain/net/rpc"
	"chain/protocol"
	"chain/protocol/bc"
	"chain/protocol/state"
)

const getSnapshotTimeout = 10 * time.Second
const heightPollingPeriod = 3 * time.Second

var (
	generatorHeight          uint64
	generatorHeightFetchedAt time.Time
	generatorLock            sync.Mutex
)

func GeneratorHeight() (uint64, time.Time) {
	generatorLock.Lock()
	h := generatorHeight
	t := generatorHeightFetchedAt
	generatorLock.Unlock()
	return h, t
}

// Fetch runs in a loop, fetching blocks from the configured
// peer (e.g. the generator) and applying them to the local
// Chain.
//
// It returns when its context is canceled.
func Fetch(ctx context.Context, c *protocol.Chain, peer *rpc.Client) {
	// This process just became leader, so it's responsible
	// for recovering after the previous leader's exit.
	prevBlock, prevSnapshot, err := c.Recover(ctx)
	if err != nil {
		log.Fatal(ctx, log.KeyError, err)
	}

	// Fetch the generator height periodically.
	go pollGeneratorHeight(ctx, peer)

	var ntimeouts uint // for backoff
	var nfailures uint // for backoff
	for {
		select {
		case <-ctx.Done():
			log.Messagef(ctx, "Deposed, Fetch exiting")
			return
		default:
			var height uint64
			if prevBlock != nil {
				height = prevBlock.Height
			}

			blocks, err := getBlocks(ctx, peer, height, timeoutBackoffDur(ntimeouts))
			if err != nil {
				log.Error(ctx, err)
				nfailures++
				time.Sleep(backoffDur(nfailures))
				continue
			}
			if len(blocks) == 0 {
				// Request time out. There might not have been any blocks published,
				// or there was a network error or it just took too long to process the
				// request.
				ntimeouts++
				continue
			}

			prevSnapshot, prevBlock, err = applyBlocks(ctx, c, prevSnapshot, prevBlock, blocks)
			if err != nil {
				log.Error(ctx, err)
				nfailures++
				time.Sleep(backoffDur(nfailures))
				continue
			}
			nfailures, ntimeouts = 0, 0
		}
	}
}

func pollGeneratorHeight(ctx context.Context, peer *rpc.Client) {
	updateGeneratorHeight(ctx, peer)

	ticker := time.NewTicker(heightPollingPeriod)
	for {
		select {
		case <-ctx.Done():
			log.Messagef(ctx, "Deposed, fetchGeneratorHeight exiting")
			ticker.Stop()
			return
		case <-ticker.C:
			updateGeneratorHeight(ctx, peer)
		}
	}
}

func updateGeneratorHeight(ctx context.Context, peer *rpc.Client) {
	gh, err := getHeight(ctx, peer)
	if err != nil {
		log.Error(ctx, err)
		return
	}

	generatorLock.Lock()
	defer generatorLock.Unlock()
	generatorHeight = gh
	generatorHeightFetchedAt = time.Now()
}

func applyBlocks(ctx context.Context, c *protocol.Chain, snap *state.Snapshot, block *bc.Block, blocks []*bc.Block) (*state.Snapshot, *bc.Block, error) {
	for _, b := range blocks {
		ss, err := c.ValidateBlock(ctx, snap, block, b)
		if err != nil {
			// TODO(kr): this is a validation failure.
			// It's either a serious bug or an attack.
			// Do something better than just log the error
			// (in the caller above). Alert a human,
			// the security team, the legal team, the A-team,
			// somebody.
			return snap, block, err
		}

		err = c.CommitBlock(ctx, b, ss)
		if err != nil {
			return snap, block, err
		}

		snap, block = ss, b
	}
	return snap, block, nil
}

func backoffDur(n uint) time.Duration {
	if n > 33 {
		n = 33 // cap to about 10s
	}
	d := rand.Int63n(1 << n)
	return time.Duration(d)
}

func timeoutBackoffDur(n uint) time.Duration {
	const baseTimeout = 3 * time.Second
	if n > 4 {
		n = 4 // cap to extra 16s
	}
	d := rand.Int63n(int64(time.Second) * (1 << n))
	return baseTimeout + time.Duration(d)
}

// getBlocks sends a get-blocks RPC request to another Core
// for all blocks since the highest-known one.
func getBlocks(ctx context.Context, peer *rpc.Client, height uint64, timeout time.Duration) ([]*bc.Block, error) {
	ctx, cancel := context.WithTimeout(ctx, timeout)
	defer cancel()

	var blocks []*bc.Block
	err := peer.Call(ctx, "/rpc/get-blocks", height, &blocks)
	if ctx.Err() == context.DeadlineExceeded {
		return nil, nil
	}
	return blocks, errors.Wrap(err, "get blocks rpc")
}

// getHeight sends a get-height RPC request to another Core for
// the latest height that that peer knows about.
func getHeight(ctx context.Context, peer *rpc.Client) (uint64, error) {
	var resp map[string]uint64
	err := peer.Call(ctx, "/rpc/block-height", nil, &resp)
	if err != nil {
		return 0, errors.Wrap(err, "could not get remote block height")
	}
	h, ok := resp["block_height"]
	if !ok {
		return 0, errors.New("unexpected response from generator")
	}

	return h, nil
}

// Snapshot fetches the latest snapshot from the generator and applies it to this
// core's snapshot set. It should only be called on freshly configured cores--
// cores that have been operating should replay all transactions so that they can
// index them properly.
func Snapshot(ctx context.Context, peer *rpc.Client, s protocol.Store, db *sql.DB) error {
	ctx, cancel := context.WithTimeout(ctx, getSnapshotTimeout)
	defer cancel()

	var snapResp struct {
		Data   []byte
		Height uint64
	}
	err := peer.Call(ctx, "/rpc/get-snapshot", nil, &snapResp)
	if err != nil {
		return err
	}

	const snapQ = `
		INSERT INTO snapshots (height, data) VALUES ($1, $2)
		ON CONFLICT DO NOTHING
	`
	_, err = db.Exec(ctx, snapQ, snapResp.Height, snapResp.Data)
	if err != nil {
		return err
	}

	// Next, get the genesis block.
	blocks, err := getBlocks(ctx, peer, 0, getSnapshotTimeout)
	if err != nil {
		return err
	}

	if len(blocks) < 1 {
		// Something seriously funny is afoot.
		return errors.New("could not get initial block from generator")
	}

	err = s.SaveBlock(ctx, blocks[0])

	// Also get the corresponding block.
	blocks, err = getBlocks(ctx, peer, snapResp.Height-1, getSnapshotTimeout) // because we get the NEXT block
	if err != nil {
		return err
	}

	if len(blocks) < 1 {
		// Something seriously funny is still afoot.
		return errors.New("generator provided snapshot but could not provide block")
	}

	return s.SaveBlock(ctx, blocks[0])
}
<|endoftext|>


<commit_before>package command

var (
	Commands = map[string]Command{}
)

// Command is the interface for specific named
// commands executed via plugins or the bot.
type Command interface {
	// Executes the command with args passed in
	Exec(args ...string) ([]byte, error)
	// Usage of the command
	Usage() string
	// Description of the command
	Description() string
	// Name of the command; used to match the command
	String() string
}

type cmd struct {
	name        string
	usage       string
	description string
	exec        func(args ...string) ([]byte, error)
}

func (c *cmd) Description() string {
	return c.description
}

func (c *cmd) Exec(args ...string) ([]byte, error) {
	return c.exec(args...)
}

func (c *cmd) Usage() string {
	return c.usage
}

func (c *cmd) String() string {
	return c.name
}

// NewCommand helps quickly create a new command
func NewCommand(name, usage, description string, exec func(args ...string) ([]byte, error)) Command {
	return &cmd{
		name:        name,
		usage:       usage,
		description: description,
		exec:        exec,
	}
}
<commit_msg>Name not used to match anymore<commit_after>package command

var (
	Commands = map[string]Command{}
)

// Command is the interface for specific named
// commands executed via plugins or the bot.
type Command interface {
	// Executes the command with args passed in
	Exec(args ...string) ([]byte, error)
	// Usage of the command
	Usage() string
	// Description of the command
	Description() string
	// Name of the command
	String() string
}

type cmd struct {
	name        string
	usage       string
	description string
	exec        func(args ...string) ([]byte, error)
}

func (c *cmd) Description() string {
	return c.description
}

func (c *cmd) Exec(args ...string) ([]byte, error) {
	return c.exec(args...)
}

func (c *cmd) Usage() string {
	return c.usage
}

func (c *cmd) String() string {
	return c.name
}

// NewCommand helps quickly create a new command
func NewCommand(name, usage, description string, exec func(args ...string) ([]byte, error)) Command {
	return &cmd{
		name:        name,
		usage:       usage,
		description: description,
		exec:        exec,
	}
}
<|endoftext|>