<commit_before>/*
 * Copyright 2014 Jason Woods.
 *
 * This file is a modification of code from Logstash Forwarder.
 * Copyright 2012-2013 Jordan Sissel and contributors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package transports

import (
  "bytes"
  "crypto/tls"
  "crypto/x509"
  "encoding/binary"
  "encoding/pem"
  "errors"
  "fmt"
  "io/ioutil"
  "lc-lib/core"
  "math/rand"
  "net"
  "regexp"
  "sync"
  "time"
)

// Support for newer SSL signature algorithms
import _ "crypto/sha256"
import _ "crypto/sha512"

const (
  // Essentially, this is how often we should check for disconnect/shutdown during socket reads
  socket_interval_seconds = 1
)

type TransportTcpRegistrar struct {
}

type TransportTcpFactory struct {
  transport string

  SSLCertificate string `config:"ssl certificate"`
  SSLKey         string `config:"ssl key"`
  SSLCA          string `config:"ssl ca"`

  hostport_re *regexp.Regexp
  tls_config  tls.Config
}

type TransportTcp struct {
  config     *TransportTcpFactory
  net_config *core.NetworkConfig
  socket     net.Conn
  tlssocket  *tls.Conn

  wait     sync.WaitGroup
  shutdown chan interface{}

  send_chan chan []byte
  recv_chan chan interface{}

  can_send chan int

  roundrobin int
  host_is_ip bool
  host       string
  port       string
  addresses  []net.IP
}

func NewTcpTransportFactory(config *core.Config, config_path string, unused map[string]interface{}, name string) (core.TransportFactory, error) {
  var err error

  ret := &TransportTcpFactory{
    transport:   name,
    hostport_re: regexp.MustCompile(`^\[?([^]]+)\]?:([0-9]+)$`),
  }

  // Only allow SSL configurations if this is "tls"
  if name == "tls" {
    if err = config.PopulateConfig(ret, config_path, unused); err != nil {
      return nil, err
    }

    if len(ret.SSLCertificate) > 0 && len(ret.SSLKey) > 0 {
      cert, err := tls.LoadX509KeyPair(ret.SSLCertificate, ret.SSLKey)
      if err != nil {
        return nil, fmt.Errorf("Failed loading client ssl certificate: %s", err)
      }

      ret.tls_config.Certificates = []tls.Certificate{cert}
    }

    if len(ret.SSLCA) > 0 {
      ret.tls_config.RootCAs = x509.NewCertPool()

      pemdata, err := ioutil.ReadFile(ret.SSLCA)
      if err != nil {
        return nil, fmt.Errorf("Failure reading CA certificate: %s", err)
      }

      block, _ := pem.Decode(pemdata)
      if block == nil {
        return nil, errors.New("Failed to decode CA certificate data")
      }
      if block.Type != "CERTIFICATE" {
        return nil, fmt.Errorf("Specified CA certificate is not a certificate: %s", ret.SSLCA)
      }

      cert, err := x509.ParseCertificate(block.Bytes)
      if err != nil {
        return nil, fmt.Errorf("Failed to parse CA certificate: %s", err)
      }

      ret.tls_config.RootCAs.AddCert(cert)
    }
  } else {
    if err := config.ReportUnusedConfig(config_path, unused); err != nil {
      return nil, err
    }
  }

  return ret, nil
}

func (f *TransportTcpFactory) NewTransport(config *core.NetworkConfig) (core.Transport, error) {
  ret := &TransportTcp{
    config: f,
    net_config: config,
  }

  // Randomise the initial host - after this it will round robin
  // Round robin after initial attempt ensures we don't retry same host twice,
  // and also ensures we try all hosts one by one
  ret.roundrobin = rand.Intn(len(config.Servers))

  return ret, nil
}

func (t *TransportTcp) ReloadConfig(new_net_config *core.NetworkConfig) int {
  // Check we can grab new TCP config to compare, if not force transport reinit
  new_config, ok := new_net_config.TransportFactory.(*TransportTcpFactory)
  if !ok {
    return core.Reload_Transport
  }

  // TODO - This does not catch changes to the underlying certificate file!
  if new_config.SSLCertificate != t.config.SSLCertificate || new_config.SSLKey != t.config.SSLKey || new_config.SSLCA != t.config.SSLCA {
    return core.Reload_Transport
  }

  // Publisher handles changes to net_config, but ensure we store the latest in case it asks for a reconnect
  t.net_config = new_net_config

  return core.Reload_None
}

func (t *TransportTcp) Init() error {
  if t.shutdown != nil {
    t.disconnect()
  }

  // Have we exhausted the address list we had?
  if t.addresses == nil {
    var err error

    // Round robin to the next server
    selected := t.net_config.Servers[t.roundrobin%len(t.net_config.Servers)]
    t.roundrobin++

    t.host, t.port, err = net.SplitHostPort(selected)
    if err != nil {
      return fmt.Errorf("Invalid hostport given: %s", selected)
    }

    // Are we an IP?
    if ip := net.ParseIP(t.host); ip != nil {
      t.host_is_ip = true
      t.addresses = []net.IP{ip}
    } else {
      // Lookup the server in DNS
      t.host_is_ip = false
      t.addresses, err = net.LookupIP(t.host)
      if err != nil {
        return fmt.Errorf("DNS lookup failure \"%s\": %s", t.host, err)
      }
    }
  }

  // Try next address and drop it from our list
  addressport := net.JoinHostPort(t.addresses[0].String(), t.port)
  if len(t.addresses) > 1 {
    t.addresses = t.addresses[1:]
  } else {
    t.addresses = nil
  }

  var desc string
  if t.host_is_ip {
    desc = fmt.Sprintf("%s", addressport)
  } else {
    desc = fmt.Sprintf("%s (%s)", addressport, t.host)
  }

  log.Info("Attempting to connect to %s", desc)

  tcpsocket, err := net.DialTimeout("tcp", addressport, t.net_config.Timeout)
  if err != nil {
    return fmt.Errorf("Failed to connect to %s: %s", desc, err)
  }

  // Now wrap in TLS if this is the "tls" transport
  if t.config.transport == "tls" {
    // Set the tlsconfig server name for server validation (required since Go 1.3)
    t.config.tls_config.ServerName = t.host

    t.tlssocket = tls.Client(&transportTcpWrap{transport: t, tcpsocket: tcpsocket}, &t.config.tls_config)
    t.tlssocket.SetDeadline(time.Now().Add(t.net_config.Timeout))
    err = t.tlssocket.Handshake()
    if err != nil {
      t.tlssocket.Close()
      tcpsocket.Close()
      return fmt.Errorf("TLS Handshake failure with %s: %s", desc, err)
    }

    t.socket = t.tlssocket
  } else {
    t.socket = tcpsocket
  }

  log.Info("Connected to %s", desc)

  // Signal channels
  t.shutdown = make(chan interface{}, 1)
  t.send_chan = make(chan []byte, 1)
  // Buffer of two for recv_chan since both routines may send an error to it
  // First error we get back initiates disconnect, thus we must not block routines
  t.recv_chan = make(chan interface{}, 2)
  t.can_send = make(chan int, 1)

  // Start with a send
  t.can_send <- 1

  t.wait.Add(2)

  // Start separate sender and receiver so we can asynchronously send and receive for max performance
  // They have to be different routines too because we don't have cross-platform poll, so they will need to block
  // Of course, we'll time out and check shutdown on occasion
  go t.sender()
  go t.receiver()

  return nil
}

func (t *TransportTcp) disconnect() {
  if t.shutdown == nil {
    return
  }

  // Send shutdown request
  close(t.shutdown)
  t.wait.Wait()
  t.shutdown = nil

  // If tls, shutdown tls socket first
  if t.config.transport == "tls" {
    t.tlssocket.Close()
  }

  t.socket.Close()
}

func (t *TransportTcp) sender() {
SendLoop:
  for {
    select {
    case <-t.shutdown:
      // Shutdown
      break SendLoop
    case msg := <-t.send_chan:
      // Ask for more while we send this
      t.setChan(t.can_send)
      // Write deadline is managed by our net.Conn wrapper that tls will call into
      _, err := t.socket.Write(msg)
      if err != nil {
        if net_err, ok := err.(net.Error); ok && net_err.Timeout() {
          // Shutdown will have been received by the wrapper
          break SendLoop
        } else {
          // Pass error back
          t.recv_chan <- err
        }
      }
    }
  }

  t.wait.Done()
}

func (t *TransportTcp) receiver() {
  var err error
  var shutdown bool
  header := make([]byte, 8)

  for {
    if err, shutdown = t.receiverRead(header); err != nil || shutdown {
      break
    }

    // Grab length of message
    length := binary.BigEndian.Uint32(header[4:8])

    // Sanity
    if length > 1048576 {
      err = fmt.Errorf("Data too large (%d)", length)
      break
    }

    // Allocate for full message
    message := make([]byte, length)

    if err, shutdown = t.receiverRead(message); err != nil || shutdown {
      break
    }

    // Pass back the message
    select {
    case <-t.shutdown:
      break
    case t.recv_chan <- [][]byte{header[0:4], message}:
    }
  } /* loop until shutdown */

  if err != nil {
    // Pass the error back and abort
    select {
    case <-t.shutdown:
    case t.recv_chan <- err:
    }
  }

  t.wait.Done()
}

func (t *TransportTcp) receiverRead(data []byte) (error, bool) {
  received := 0

RecvLoop:
  for {
    select {
    case <-t.shutdown:
      // Shutdown
      break RecvLoop
    default:
      // Timeout after socket_interval_seconds, check for shutdown, and try again
      t.socket.SetReadDeadline(time.Now().Add(socket_interval_seconds * time.Second))

      length, err := t.socket.Read(data[received:])
      received += length
      if err == nil || received >= len(data) {
        // Success
        return nil, false
      } else if net_err, ok := err.(net.Error); ok && net_err.Timeout() {
        // Keep trying
        continue
      } else {
        // Pass an error back
        return err, false
      }
    } /* select */
  } /* loop until required amount receive or shutdown */

  return nil, true
}

func (t *TransportTcp) setChan(set chan<- int) {
  select {
  case set <- 1:
  default:
  }
}

func (t *TransportTcp) CanSend() <-chan int {
  return t.can_send
}

func (t *TransportTcp) Write(signature string, message []byte) (err error) {
  var write_buffer *bytes.Buffer
  write_buffer = bytes.NewBuffer(make([]byte, 0, len(signature)+4+len(message)))

  if _, err = write_buffer.Write([]byte(signature)); err != nil {
    return
  }
  if err = binary.Write(write_buffer, binary.BigEndian, uint32(len(message))); err != nil {
    return
  }
  if len(message) != 0 {
    if _, err = write_buffer.Write(message); err != nil {
      return
    }
  }

  t.send_chan <- write_buffer.Bytes()
  return nil
}

func (t *TransportTcp) Read() <-chan interface{} {
  return t.recv_chan
}

func (t *TransportTcp) Shutdown() {
  t.disconnect()
}

// Register the transports
func init() {
  rand.Seed(time.Now().UnixNano())

  core.RegisterTransport("tcp", NewTcpTransportFactory)
  core.RegisterTransport("tls", NewTcpTransportFactory)
}
<commit_msg>Allow certificate authorities with intermediates.<commit_after>/*
 * Copyright 2014 Jason Woods.
 *
 * This file is a modification of code from Logstash Forwarder.
 * Copyright 2012-2013 Jordan Sissel and contributors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package transports

import (
  "bytes"
  "crypto/tls"
  "crypto/x509"
  "encoding/binary"
  "encoding/pem"
  "fmt"
  "io/ioutil"
  "lc-lib/core"
  "math/rand"
  "net"
  "regexp"
  "sync"
  "time"
)

// Support for newer SSL signature algorithms
import _ "crypto/sha256"
import _ "crypto/sha512"

const (
  // Essentially, this is how often we should check for disconnect/shutdown during socket reads
  socket_interval_seconds = 1
)

type TransportTcpRegistrar struct {
}

type TransportTcpFactory struct {
  transport string

  SSLCertificate string `config:"ssl certificate"`
  SSLKey         string `config:"ssl key"`
  SSLCA          string `config:"ssl ca"`

  hostport_re *regexp.Regexp
  tls_config  tls.Config
}

type TransportTcp struct {
  config     *TransportTcpFactory
  net_config *core.NetworkConfig
  socket     net.Conn
  tlssocket  *tls.Conn

  wait     sync.WaitGroup
  shutdown chan interface{}

  send_chan chan []byte
  recv_chan chan interface{}

  can_send chan int

  roundrobin int
  host_is_ip bool
  host       string
  port       string
  addresses  []net.IP
}

func NewTcpTransportFactory(config *core.Config, config_path string, unused map[string]interface{}, name string) (core.TransportFactory, error) {
  var err error

  ret := &TransportTcpFactory{
    transport:   name,
    hostport_re: regexp.MustCompile(`^\[?([^]]+)\]?:([0-9]+)$`),
  }

  // Only allow SSL configurations if this is "tls"
  if name == "tls" {
    if err = config.PopulateConfig(ret, config_path, unused); err != nil {
      return nil, err
    }

    if len(ret.SSLCertificate) > 0 && len(ret.SSLKey) > 0 {
      cert, err := tls.LoadX509KeyPair(ret.SSLCertificate, ret.SSLKey)
      if err != nil {
        return nil, fmt.Errorf("Failed loading client ssl certificate: %s", err)
      }

      ret.tls_config.Certificates = []tls.Certificate{cert}
    }

    if len(ret.SSLCA) > 0 {
      ret.tls_config.RootCAs = x509.NewCertPool()
      pemdata, err := ioutil.ReadFile(ret.SSLCA)
      if err != nil {
        return nil, fmt.Errorf("Failure reading CA certificate: %s\n", err)
      }
      rest := pemdata
      var block *pem.Block
      var pemBlockNum = 1
      for {
        block, rest = pem.Decode(rest)
        if block != nil {
          if block.Type != "CERTIFICATE" {
            return nil, fmt.Errorf("Block %d does not contain a certificate: %s\n", pemBlockNum, ret.SSLCA)
          }
          cert, err := x509.ParseCertificate(block.Bytes)
          if err != nil {
            return nil, fmt.Errorf("Failed to parse CA certificate in block %d: %s\n", pemBlockNum, ret.SSLCA)
          }
          ret.tls_config.RootCAs.AddCert(cert)
          pemBlockNum += 1
        } else {
          break
        }
      }
    }
  } else {
    if err := config.ReportUnusedConfig(config_path, unused); err != nil {
      return nil, err
    }
  }

  return ret, nil
}

func (f *TransportTcpFactory) NewTransport(config *core.NetworkConfig) (core.Transport, error) {
  ret := &TransportTcp{
    config: f,
    net_config: config,
  }

  // Randomise the initial host - after this it will round robin
  // Round robin after initial attempt ensures we don't retry same host twice,
  // and also ensures we try all hosts one by one
  ret.roundrobin = rand.Intn(len(config.Servers))

  return ret, nil
}

func (t *TransportTcp) ReloadConfig(new_net_config *core.NetworkConfig) int {
  // Check we can grab new TCP config to compare, if not force transport reinit
  new_config, ok := new_net_config.TransportFactory.(*TransportTcpFactory)
  if !ok {
    return core.Reload_Transport
  }

  // TODO - This does not catch changes to the underlying certificate file!
  if new_config.SSLCertificate != t.config.SSLCertificate || new_config.SSLKey != t.config.SSLKey || new_config.SSLCA != t.config.SSLCA {
    return core.Reload_Transport
  }

  // Publisher handles changes to net_config, but ensure we store the latest in case it asks for a reconnect
  t.net_config = new_net_config

  return core.Reload_None
}

func (t *TransportTcp) Init() error {
  if t.shutdown != nil {
    t.disconnect()
  }

  // Have we exhausted the address list we had?
  if t.addresses == nil {
    var err error

    // Round robin to the next server
    selected := t.net_config.Servers[t.roundrobin%len(t.net_config.Servers)]
    t.roundrobin++

    t.host, t.port, err = net.SplitHostPort(selected)
    if err != nil {
      return fmt.Errorf("Invalid hostport given: %s", selected)
    }

    // Are we an IP?
    if ip := net.ParseIP(t.host); ip != nil {
      t.host_is_ip = true
      t.addresses = []net.IP{ip}
    } else {
      // Lookup the server in DNS
      t.host_is_ip = false
      t.addresses, err = net.LookupIP(t.host)
      if err != nil {
        return fmt.Errorf("DNS lookup failure \"%s\": %s", t.host, err)
      }
    }
  }

  // Try next address and drop it from our list
  addressport := net.JoinHostPort(t.addresses[0].String(), t.port)
  if len(t.addresses) > 1 {
    t.addresses = t.addresses[1:]
  } else {
    t.addresses = nil
  }

  var desc string
  if t.host_is_ip {
    desc = fmt.Sprintf("%s", addressport)
  } else {
    desc = fmt.Sprintf("%s (%s)", addressport, t.host)
  }

  log.Info("Attempting to connect to %s", desc)

  tcpsocket, err := net.DialTimeout("tcp", addressport, t.net_config.Timeout)
  if err != nil {
    return fmt.Errorf("Failed to connect to %s: %s", desc, err)
  }

  // Now wrap in TLS if this is the "tls" transport
  if t.config.transport == "tls" {
    // Set the tlsconfig server name for server validation (required since Go 1.3)
    t.config.tls_config.ServerName = t.host

    t.tlssocket = tls.Client(&transportTcpWrap{transport: t, tcpsocket: tcpsocket}, &t.config.tls_config)
    t.tlssocket.SetDeadline(time.Now().Add(t.net_config.Timeout))
    err = t.tlssocket.Handshake()
    if err != nil {
      t.tlssocket.Close()
      tcpsocket.Close()
      return fmt.Errorf("TLS Handshake failure with %s: %s", desc, err)
    }

    t.socket = t.tlssocket
  } else {
    t.socket = tcpsocket
  }

  log.Info("Connected to %s", desc)

  // Signal channels
  t.shutdown = make(chan interface{}, 1)
  t.send_chan = make(chan []byte, 1)
  // Buffer of two for recv_chan since both routines may send an error to it
  // First error we get back initiates disconnect, thus we must not block routines
  t.recv_chan = make(chan interface{}, 2)
  t.can_send = make(chan int, 1)

  // Start with a send
  t.can_send <- 1

  t.wait.Add(2)

  // Start separate sender and receiver so we can asynchronously send and receive for max performance
  // They have to be different routines too because we don't have cross-platform poll, so they will need to block
  // Of course, we'll time out and check shutdown on occasion
  go t.sender()
  go t.receiver()

  return nil
}

func (t *TransportTcp) disconnect() {
  if t.shutdown == nil {
    return
  }

  // Send shutdown request
  close(t.shutdown)
  t.wait.Wait()
  t.shutdown = nil

  // If tls, shutdown tls socket first
  if t.config.transport == "tls" {
    t.tlssocket.Close()
  }

  t.socket.Close()
}

func (t *TransportTcp) sender() {
SendLoop:
  for {
    select {
    case <-t.shutdown:
      // Shutdown
      break SendLoop
    case msg := <-t.send_chan:
      // Ask for more while we send this
      t.setChan(t.can_send)
      // Write deadline is managed by our net.Conn wrapper that tls will call into
      _, err := t.socket.Write(msg)
      if err != nil {
        if net_err, ok := err.(net.Error); ok && net_err.Timeout() {
          // Shutdown will have been received by the wrapper
          break SendLoop
        } else {
          // Pass error back
          t.recv_chan <- err
        }
      }
    }
  }

  t.wait.Done()
}

func (t *TransportTcp) receiver() {
  var err error
  var shutdown bool
  header := make([]byte, 8)

  for {
    if err, shutdown = t.receiverRead(header); err != nil || shutdown {
      break
    }

    // Grab length of message
    length := binary.BigEndian.Uint32(header[4:8])

    // Sanity
    if length > 1048576 {
      err = fmt.Errorf("Data too large (%d)", length)
      break
    }

    // Allocate for full message
    message := make([]byte, length)

    if err, shutdown = t.receiverRead(message); err != nil || shutdown {
      break
    }

    // Pass back the message
    select {
    case <-t.shutdown:
      break
    case t.recv_chan <- [][]byte{header[0:4], message}:
    }
  } /* loop until shutdown */

  if err != nil {
    // Pass the error back and abort
    select {
    case <-t.shutdown:
    case t.recv_chan <- err:
    }
  }

  t.wait.Done()
}

func (t *TransportTcp) receiverRead(data []byte) (error, bool) {
  received := 0

RecvLoop:
  for {
    select {
    case <-t.shutdown:
      // Shutdown
      break RecvLoop
    default:
      // Timeout after socket_interval_seconds, check for shutdown, and try again
      t.socket.SetReadDeadline(time.Now().Add(socket_interval_seconds * time.Second))

      length, err := t.socket.Read(data[received:])
      received += length
      if err == nil || received >= len(data) {
        // Success
        return nil, false
      } else if net_err, ok := err.(net.Error); ok && net_err.Timeout() {
        // Keep trying
        continue
      } else {
        // Pass an error back
        return err, false
      }
    } /* select */
  } /* loop until required amount receive or shutdown */

  return nil, true
}

func (t *TransportTcp) setChan(set chan<- int) {
  select {
  case set <- 1:
  default:
  }
}

func (t *TransportTcp) CanSend() <-chan int {
  return t.can_send
}

func (t *TransportTcp) Write(signature string, message []byte) (err error) {
  var write_buffer *bytes.Buffer
  write_buffer = bytes.NewBuffer(make([]byte, 0, len(signature)+4+len(message)))

  if _, err = write_buffer.Write([]byte(signature)); err != nil {
    return
  }
  if err = binary.Write(write_buffer, binary.BigEndian, uint32(len(message))); err != nil {
    return
  }
  if len(message) != 0 {
    if _, err = write_buffer.Write(message); err != nil {
      return
    }
  }

  t.send_chan <- write_buffer.Bytes()
  return nil
}

func (t *TransportTcp) Read() <-chan interface{} {
  return t.recv_chan
}

func (t *TransportTcp) Shutdown() {
  t.disconnect()
}

// Register the transports
func init() {
  rand.Seed(time.Now().UnixNano())

  core.RegisterTransport("tcp", NewTcpTransportFactory)
  core.RegisterTransport("tls", NewTcpTransportFactory)
}
<|endoftext|>


<commit_before>package network

import (
	"fmt"
	"bytes"
	"errors"
)

// A NETWORK is a LIST of input NODEs and a LIST of output NODEs.
// The point of the network is to define a single entity which can evolve
// or learn on its own, even though it may be part of a larger framework.
type Network struct {
	// A network id
	Id            int
	// Is a name of this network */
	Name          string

	// The number of links in the net (-1 means not yet counted)
	numlinks      int

	// A list of all the nodes in the network except MIMO control ones
	all_nodes     []*NNode
	// NNodes that input into the network
	inputs        []*NNode
	// NNodes that output from the network
	Outputs       []*NNode

	// NNodes that connect network modules
	control_nodes []*NNode
}

// Creates new network
func NewNetwork(in, out, all []*NNode, net_id int) *Network {
	n := Network{
		Id:net_id,
		inputs:in,
		Outputs:out,
		all_nodes:all,
		numlinks:-1,
	}
	return &n
}

// Creates new modular network with control nodes
func NewModularNetwork(in, out, all, control []*NNode, net_id int) *Network {
	n := NewNetwork(in, out, all, net_id)
	n.control_nodes = control
	return n
}

// Creates fast network solver based on the architecture of this network. It's primarily aimed for big networks to improve
// processing speed.
func (n *Network) FastNetworkSolver() (NetworkSolver, error) {
	// calculate neurons per layer
	outputNeuronCount := len(n.Outputs)
	// build bias, input and hidden neurons lists
	biasNeuronCount := 0
	in_list := make([]*NNode, 0)
	bias_list := make([]*NNode, 0)
	hidn_list := make([]*NNode, 0)
	for _, ne := range n.all_nodes {
		switch ne.NeuronType {
		case BiasNeuron:
			biasNeuronCount += 1
			bias_list = append(bias_list, ne)
		case InputNeuron:
			in_list = append(in_list, ne)
		case HiddenNeuron:
			hidn_list = append(hidn_list, ne)
		}
	}
	inputNeuronCount := len(in_list)
	totalNeuronCount := len(n.all_nodes)

	// create activation functions array

	activations := make([]NodeActivationType, totalNeuronCount)
	neuronLookup := make(map[int]int)// id:index
	neuronIndex := 0
	// walk through neuron nodes in order: bias, input, output, hidden
	neuronIndex = processList(neuronIndex, bias_list, activations, neuronLookup)
	neuronIndex = processList(neuronIndex, in_list, activations, neuronLookup)
	neuronIndex = processList(neuronIndex, n.Outputs, activations, neuronLookup)
	neuronIndex = processList(neuronIndex, hidn_list, activations, neuronLookup)

	// walk through neurons in order: input, output, hidden and create bias and connections lists
	biases := make([]float64, totalNeuronCount)
	connections := make([]*FastNetworkLink, 0)

	if in_connects, err := processIncomingConnections(in_list, biases, neuronLookup); err == nil {
		connections = append(connections, in_connects...)
	} else {
		return nil, err
	}
	if in_connects, err := processIncomingConnections(hidn_list, biases, neuronLookup); err == nil {
		connections = append(connections, in_connects...)
	} else {
		return nil, err
	}
	if in_connects, err := processIncomingConnections(n.Outputs, biases, neuronLookup); err == nil {
		connections = append(connections, in_connects...)
	} else {
		return nil, err
	}

	// walk through control neurons
	modules := make([]*FastControlNode, len(n.control_nodes))
	for i, cn := range n.control_nodes {
		// collect inputs
		inputs := make([]int, len(cn.Incoming))
		for j, in := range cn.Incoming {
			if in_index, ok := neuronLookup[in.InNode.Id]; ok {
				inputs[j] = in_index
			} else {
				return nil, errors.New(
					fmt.Sprintf("Failed to lookup for input neuron with id: %d at control neuron: %d",
						in.InNode.Id, cn.Id))
			}
		}
		// collect outputs
		outputs := make([]int, len(cn.Outgoing))
		for j, out := range cn.Outgoing {
			if out_index, ok := neuronLookup[out.OutNode.Id]; ok {
				outputs[j] = out_index
			} else {
				return nil, errors.New(
					fmt.Sprintf("Failed to lookup for output neuron with id: %d at control neuron: %d",
						out.InNode.Id, cn.Id))
			}
		}
		// build control node
		modules[i] = &FastControlNode{InputIndxs:inputs, OutputIndxs:outputs, ActivationType:cn.ActivationType}
	}

	return NewFastModularNetworkSolver(biasNeuronCount, inputNeuronCount, outputNeuronCount, totalNeuronCount,
		activations, connections, biases, modules), nil
}

func processList(startIndex int, nList []*NNode, activations[]NodeActivationType, neuronLookup map[int]int) int {
	for _, ne := range nList {
		activations[startIndex] = ne.ActivationType
		neuronLookup[ne.Id] = startIndex
		startIndex += 1
	}
	return startIndex
}

func processIncomingConnections(nList []*NNode, biases []float64, neuronLookup map[int]int) (connections []*FastNetworkLink, err error) {
	connections = make([]*FastNetworkLink, 0)
	for _, ne := range nList {
		if targetIndex, ok := neuronLookup[ne.Id]; ok {
			for _, in := range ne.Incoming {
				if sourceIndex, ok := neuronLookup[in.InNode.Id]; ok {
					if in.InNode.NeuronType == BiasNeuron {
						// store bias for target neuron
						biases[targetIndex] += in.Weight
					}
					// save connection
					conn := FastNetworkLink{
						SourceIndx:sourceIndex,
						TargetIndx:targetIndex,
						Weight:in.Weight,
					}
					connections = append(connections, &conn)
				} else {
					err = errors.New(
						fmt.Sprintf("Failed to lookup for source neuron with id: %d", in.InNode.Id))
					break
				}
			}
		} else {
			err = errors.New(fmt.Sprintf("Failed to lookup for target neuron with id: %d", ne.Id))
			break
		}
	}
	if err != nil {
		return nil, err
	}
	return connections, err
}

// Puts the network back into an initial state
func (n *Network) Flush() (res bool, err error) {
	res = true
	// Flush back recursively
	for _, node := range n.all_nodes {
		node.Flushback()
		err = node.FlushbackCheck()
		if err != nil {
			// failed - no need to continue
			res = false
			break
		}
	}
	return res, err
}

// Prints the values of network outputs to the console
func (n *Network) PrintActivation() string {
	out := bytes.NewBufferString(fmt.Sprintf("Network %s with id %d outputs: (", n.Name, n.Id))
	for i, node := range n.Outputs {
		fmt.Fprintf(out, "[Output #%d: %s] ", i, node)
	}
	fmt.Fprint(out, ")")
	return out.String()
}

// Print the values of network inputs to the console
func (n *Network) PrintInput() string {
	out := bytes.NewBufferString(fmt.Sprintf("Network %s with id %d inputs: (", n.Name, n.Id))
	for i, node := range n.inputs {
		fmt.Fprintf(out, "[Input #%d: %s] ", i, node)
	}
	fmt.Fprint(out, ")")
	return out.String()
}

// If at least one output is not active then return true
func (n *Network) OutputIsOff() bool {
	for _, node := range n.Outputs {
		if node.ActivationsCount == 0 {
			return true
		}

	}
	return false
}

// Attempts to activate the network given number of steps before returning error.
func (n *Network) ActivateSteps(max_steps int) (bool, error) {
	// For adding to the activesum
	add_amount := 0.0
	// Make sure we at least activate once
	one_time := false
	// Used in case the output is somehow truncated from the network
	abort_count := 0

	// Keep activating until all the outputs have become active
	// (This only happens on the first activation, because after that they are always active)
	for n.OutputIsOff() || !one_time {

		if abort_count >= max_steps {
			return false, NetErrExceededMaxActivationAttempts
		}

		// For each neuron node, compute the sum of its incoming activation
		for _, np := range n.all_nodes {
			if np.IsNeuron() {
				np.ActivationSum = 0.0 // reset activation value

				// For each node's incoming connection, add the activity from the connection to the activesum
				for _, link := range np.Incoming {
					// Handle possible time delays
					if !link.IsTimeDelayed {
						add_amount = link.Weight * link.InNode.GetActiveOut()
						if link.InNode.isActive || link.InNode.IsSensor() {
							np.isActive = true
						}
					} else {
						add_amount = link.Weight * link.InNode.GetActiveOutTd()
					}
					np.ActivationSum += add_amount
				} // End {for} over incoming links
			} // End if != SENSOR
		}  // End {for} over all nodes

		// Now activate all the neuron nodes off their incoming activation
		for _, np := range n.all_nodes {
			if np.IsNeuron() {
				// Only activate if some active input came in
				if np.isActive {
					// Now run the net activation through an activation function
					err := NodeActivators.ActivateNode(np)
					if err != nil {
						return false, err
					}
				}
			}
		}

		// Now activate all MIMO control genes to propagate activation through genome modules
		for _, cn := range n.control_nodes {
			cn.isActive = false
			// Activate control MIMO node as control module
			err := NodeActivators.ActivateModule(cn)
			if err != nil {
				return false, err
			}
			// mark control node as active
			cn.isActive = true
		}

		one_time = true
		abort_count += 1
	}
	return true, nil
}

// Activates the net such that all outputs are active
func (n *Network) Activate() (bool, error) {
	return n.ActivateSteps(20)
}

// Propagates activation wave through all network nodes provided number of steps in forward direction.
// Returns true if activation wave passed from all inputs to outputs.
func (n *Network) ForwardSteps(steps int) (res bool, err error) {
	for i := 0; i < steps; i++ {
		res, err = n.Activate()
		if err != nil {
			// failure - no need to continue
			break
		}
	}
	return res, err
}

// Propagates activation wave through all network nodes provided number of steps by recursion from output nodes
// Returns true if activation wave passed from all inputs to outputs.
func (n *Network) RecursiveSteps() (bool, error) {
	return false, errors.New("RecursiveSteps Not Implemented")
}

// Attempts to relax network given amount of steps until giving up. The network considered relaxed when absolute
// value of the change at any given point is less than maxAllowedSignalDelta during activation waves propagation.
// If maxAllowedSignalDelta value is less than or equal to 0, the method will return true without checking for relaxation.
func (n *Network) Relax(maxSteps int, maxAllowedSignalDelta float64) (bool, error) {
	return false, errors.New("Relax Not Implemented")
}

// Takes an array of sensor values and loads it into SENSOR inputs ONLY
func (n *Network) LoadSensors(sensors []float64) error {
	if len(sensors) != len(n.inputs) {
		return NetErrUnsupportedSensorsArraySize
	}

	counter := 0
	for _, node := range n.inputs {
		if node.IsSensor() {
			node.SensorLoad(sensors[counter])
			counter += 1
		}
	}
	return nil
}

// Read output values from the output nodes of the network
func (n *Network) ReadOutputs() []float64 {
	outs := make([]float64, len(n.Outputs))
	for i, o := range n.Outputs {
		outs[i] = o.Activation
	}
	return outs
}

// Counts the number of nodes in the net
func (n *Network) NodeCount() int {
	if len(n.control_nodes) == 0 {
		return len(n.all_nodes)
	} else {
		return len(n.all_nodes) + len(n.control_nodes)
	}
}

// Counts the number of links in the net
func (n *Network) LinkCount() int {
	n.numlinks = 0
	for _, node := range n.all_nodes {
		n.numlinks += len(node.Incoming)
	}
	if len(n.control_nodes) != 0 {
		for _, node := range n.control_nodes {
			n.numlinks += len(node.Incoming)
			n.numlinks += len(node.Outgoing)
		}
	}
	return n.numlinks
}

// Returns complexity of this network which is sum of nodes count and links count
func (n *Network) Complexity() int {
	return n.NodeCount() + n.LinkCount()
}

// This checks a POTENTIAL link between a potential in_node
// and potential out_node to see if it must be recurrent.
// Use count and thresh to jump out in the case of an infinite loop.
func (n *Network) IsRecurrent(in_node, out_node *NNode, count *int, thresh int) bool {
	// Count the node as visited
	*count++

	if *count > thresh {
		return false // Short out the whole thing - loop detected
	}

	if in_node == out_node {
		return true
	} else {
		// Check back on all links ...
		for _, link := range in_node.Incoming {
			// But skip links that are already recurrent -
			// We want to check back through the forward flow of signals only
			if !link.IsRecurrent {
				if n.IsRecurrent(link.InNode, out_node, count, thresh) {
					return true
				}
			}
		}
	}
	return false
}

// Find the maximum number of neurons between an output and an input
func (n *Network) MaxDepth() (int, error) {
	if len(n.control_nodes) > 0 {
		return -1, errors.New("unsupported for modular networks")
	}
	// The quick case when there are no hidden nodes
	if len(n.all_nodes) == len(n.inputs) + len(n.Outputs) && len(n.control_nodes) == 0 {
		return 1, nil // just one layer depth
	}

	max := 0 // The max depth
	for _, node := range n.Outputs {
		curr_depth, err := node.Depth(0)
		if err != nil {
			return curr_depth, err
		}
		if curr_depth > max {
			max = curr_depth
		}
	}

	return max, nil
}

// Returns all nodes in the network
func (n *Network) AllNodes() []*NNode {
	return n.all_nodes
}
<commit_msg>Fixed bug when BIAS connections was added resulting in BIAS values duplication<commit_after>package network

import (
	"fmt"
	"bytes"
	"errors"
)

// A NETWORK is a LIST of input NODEs and a LIST of output NODEs.
// The point of the network is to define a single entity which can evolve
// or learn on its own, even though it may be part of a larger framework.
type Network struct {
	// A network id
	Id            int
	// Is a name of this network */
	Name          string

	// The number of links in the net (-1 means not yet counted)
	numlinks      int

	// A list of all the nodes in the network except MIMO control ones
	all_nodes     []*NNode
	// NNodes that input into the network
	inputs        []*NNode
	// NNodes that output from the network
	Outputs       []*NNode

	// NNodes that connect network modules
	control_nodes []*NNode
}

// Creates new network
func NewNetwork(in, out, all []*NNode, net_id int) *Network {
	n := Network{
		Id:net_id,
		inputs:in,
		Outputs:out,
		all_nodes:all,
		numlinks:-1,
	}
	return &n
}

// Creates new modular network with control nodes
func NewModularNetwork(in, out, all, control []*NNode, net_id int) *Network {
	n := NewNetwork(in, out, all, net_id)
	n.control_nodes = control
	return n
}

// Creates fast network solver based on the architecture of this network. It's primarily aimed for big networks to improve
// processing speed.
func (n *Network) FastNetworkSolver() (NetworkSolver, error) {
	// calculate neurons per layer
	outputNeuronCount := len(n.Outputs)
	// build bias, input and hidden neurons lists
	biasNeuronCount := 0
	in_list := make([]*NNode, 0)
	bias_list := make([]*NNode, 0)
	hidn_list := make([]*NNode, 0)
	for _, ne := range n.all_nodes {
		switch ne.NeuronType {
		case BiasNeuron:
			biasNeuronCount += 1
			bias_list = append(bias_list, ne)
		case InputNeuron:
			in_list = append(in_list, ne)
		case HiddenNeuron:
			hidn_list = append(hidn_list, ne)
		}
	}
	inputNeuronCount := len(in_list)
	totalNeuronCount := len(n.all_nodes)

	// create activation functions array

	activations := make([]NodeActivationType, totalNeuronCount)
	neuronLookup := make(map[int]int)// id:index
	neuronIndex := 0
	// walk through neuron nodes in order: bias, input, output, hidden
	neuronIndex = processList(neuronIndex, bias_list, activations, neuronLookup)
	neuronIndex = processList(neuronIndex, in_list, activations, neuronLookup)
	neuronIndex = processList(neuronIndex, n.Outputs, activations, neuronLookup)
	neuronIndex = processList(neuronIndex, hidn_list, activations, neuronLookup)

	// walk through neurons in order: input, output, hidden and create bias and connections lists
	biases := make([]float64, totalNeuronCount)
	connections := make([]*FastNetworkLink, 0)

	if in_connects, err := processIncomingConnections(in_list, biases, neuronLookup); err == nil {
		connections = append(connections, in_connects...)
	} else {
		return nil, err
	}
	if in_connects, err := processIncomingConnections(hidn_list, biases, neuronLookup); err == nil {
		connections = append(connections, in_connects...)
	} else {
		return nil, err
	}
	if in_connects, err := processIncomingConnections(n.Outputs, biases, neuronLookup); err == nil {
		connections = append(connections, in_connects...)
	} else {
		return nil, err
	}

	// walk through control neurons
	modules := make([]*FastControlNode, len(n.control_nodes))
	for i, cn := range n.control_nodes {
		// collect inputs
		inputs := make([]int, len(cn.Incoming))
		for j, in := range cn.Incoming {
			if in_index, ok := neuronLookup[in.InNode.Id]; ok {
				inputs[j] = in_index
			} else {
				return nil, errors.New(
					fmt.Sprintf("Failed to lookup for input neuron with id: %d at control neuron: %d",
						in.InNode.Id, cn.Id))
			}
		}
		// collect outputs
		outputs := make([]int, len(cn.Outgoing))
		for j, out := range cn.Outgoing {
			if out_index, ok := neuronLookup[out.OutNode.Id]; ok {
				outputs[j] = out_index
			} else {
				return nil, errors.New(
					fmt.Sprintf("Failed to lookup for output neuron with id: %d at control neuron: %d",
						out.InNode.Id, cn.Id))
			}
		}
		// build control node
		modules[i] = &FastControlNode{InputIndxs:inputs, OutputIndxs:outputs, ActivationType:cn.ActivationType}
	}

	return NewFastModularNetworkSolver(biasNeuronCount, inputNeuronCount, outputNeuronCount, totalNeuronCount,
		activations, connections, biases, modules), nil
}

func processList(startIndex int, nList []*NNode, activations[]NodeActivationType, neuronLookup map[int]int) int {
	for _, ne := range nList {
		activations[startIndex] = ne.ActivationType
		neuronLookup[ne.Id] = startIndex
		startIndex += 1
	}
	return startIndex
}

func processIncomingConnections(nList []*NNode, biases []float64, neuronLookup map[int]int) (connections []*FastNetworkLink, err error) {
	connections = make([]*FastNetworkLink, 0)
	for _, ne := range nList {
		if targetIndex, ok := neuronLookup[ne.Id]; ok {
			for _, in := range ne.Incoming {
				if sourceIndex, ok := neuronLookup[in.InNode.Id]; ok {
					if in.InNode.NeuronType == BiasNeuron {
						// store bias for target neuron
						biases[targetIndex] += in.Weight
					} else {
						// save connection
						conn := FastNetworkLink{
							SourceIndx:sourceIndex,
							TargetIndx:targetIndex,
							Weight:in.Weight,
						}
						connections = append(connections, &conn)
					}
				} else {
					err = errors.New(
						fmt.Sprintf("Failed to lookup for source neuron with id: %d", in.InNode.Id))
					break
				}
			}
		} else {
			err = errors.New(fmt.Sprintf("Failed to lookup for target neuron with id: %d", ne.Id))
			break
		}
	}
	if err != nil {
		return nil, err
	}
	return connections, err
}

// Puts the network back into an initial state
func (n *Network) Flush() (res bool, err error) {
	res = true
	// Flush back recursively
	for _, node := range n.all_nodes {
		node.Flushback()
		err = node.FlushbackCheck()
		if err != nil {
			// failed - no need to continue
			res = false
			break
		}
	}
	return res, err
}

// Prints the values of network outputs to the console
func (n *Network) PrintActivation() string {
	out := bytes.NewBufferString(fmt.Sprintf("Network %s with id %d outputs: (", n.Name, n.Id))
	for i, node := range n.Outputs {
		fmt.Fprintf(out, "[Output #%d: %s] ", i, node)
	}
	fmt.Fprint(out, ")")
	return out.String()
}

// Print the values of network inputs to the console
func (n *Network) PrintInput() string {
	out := bytes.NewBufferString(fmt.Sprintf("Network %s with id %d inputs: (", n.Name, n.Id))
	for i, node := range n.inputs {
		fmt.Fprintf(out, "[Input #%d: %s] ", i, node)
	}
	fmt.Fprint(out, ")")
	return out.String()
}

// If at least one output is not active then return true
func (n *Network) OutputIsOff() bool {
	for _, node := range n.Outputs {
		if node.ActivationsCount == 0 {
			return true
		}

	}
	return false
}

// Attempts to activate the network given number of steps before returning error.
func (n *Network) ActivateSteps(max_steps int) (bool, error) {
	// For adding to the activesum
	add_amount := 0.0
	// Make sure we at least activate once
	one_time := false
	// Used in case the output is somehow truncated from the network
	abort_count := 0

	// Keep activating until all the outputs have become active
	// (This only happens on the first activation, because after that they are always active)
	for n.OutputIsOff() || !one_time {

		if abort_count >= max_steps {
			return false, NetErrExceededMaxActivationAttempts
		}

		// For each neuron node, compute the sum of its incoming activation
		for _, np := range n.all_nodes {
			if np.IsNeuron() {
				np.ActivationSum = 0.0 // reset activation value

				// For each node's incoming connection, add the activity from the connection to the activesum
				for _, link := range np.Incoming {
					// Handle possible time delays
					if !link.IsTimeDelayed {
						add_amount = link.Weight * link.InNode.GetActiveOut()
						if link.InNode.isActive || link.InNode.IsSensor() {
							np.isActive = true
						}
					} else {
						add_amount = link.Weight * link.InNode.GetActiveOutTd()
					}
					np.ActivationSum += add_amount
				} // End {for} over incoming links
			} // End if != SENSOR
		}  // End {for} over all nodes

		// Now activate all the neuron nodes off their incoming activation
		for _, np := range n.all_nodes {
			if np.IsNeuron() {
				// Only activate if some active input came in
				if np.isActive {
					// Now run the net activation through an activation function
					err := NodeActivators.ActivateNode(np)
					if err != nil {
						return false, err
					}
				}
			}
		}

		// Now activate all MIMO control genes to propagate activation through genome modules
		for _, cn := range n.control_nodes {
			cn.isActive = false
			// Activate control MIMO node as control module
			err := NodeActivators.ActivateModule(cn)
			if err != nil {
				return false, err
			}
			// mark control node as active
			cn.isActive = true
		}

		one_time = true
		abort_count += 1
	}
	return true, nil
}

// Activates the net such that all outputs are active
func (n *Network) Activate() (bool, error) {
	return n.ActivateSteps(20)
}

// Propagates activation wave through all network nodes provided number of steps in forward direction.
// Returns true if activation wave passed from all inputs to outputs.
func (n *Network) ForwardSteps(steps int) (res bool, err error) {
	for i := 0; i < steps; i++ {
		res, err = n.Activate()
		if err != nil {
			// failure - no need to continue
			break
		}
	}
	return res, err
}

// Propagates activation wave through all network nodes provided number of steps by recursion from output nodes
// Returns true if activation wave passed from all inputs to outputs.
func (n *Network) RecursiveSteps() (bool, error) {
	return false, errors.New("RecursiveSteps Not Implemented")
}

// Attempts to relax network given amount of steps until giving up. The network considered relaxed when absolute
// value of the change at any given point is less than maxAllowedSignalDelta during activation waves propagation.
// If maxAllowedSignalDelta value is less than or equal to 0, the method will return true without checking for relaxation.
func (n *Network) Relax(maxSteps int, maxAllowedSignalDelta float64) (bool, error) {
	return false, errors.New("Relax Not Implemented")
}

// Takes an array of sensor values and loads it into SENSOR inputs ONLY
func (n *Network) LoadSensors(sensors []float64) error {
	if len(sensors) != len(n.inputs) {
		return NetErrUnsupportedSensorsArraySize
	}

	counter := 0
	for _, node := range n.inputs {
		if node.IsSensor() {
			node.SensorLoad(sensors[counter])
			counter += 1
		}
	}
	return nil
}

// Read output values from the output nodes of the network
func (n *Network) ReadOutputs() []float64 {
	outs := make([]float64, len(n.Outputs))
	for i, o := range n.Outputs {
		outs[i] = o.Activation
	}
	return outs
}

// Counts the number of nodes in the net
func (n *Network) NodeCount() int {
	if len(n.control_nodes) == 0 {
		return len(n.all_nodes)
	} else {
		return len(n.all_nodes) + len(n.control_nodes)
	}
}

// Counts the number of links in the net
func (n *Network) LinkCount() int {
	n.numlinks = 0
	for _, node := range n.all_nodes {
		n.numlinks += len(node.Incoming)
	}
	if len(n.control_nodes) != 0 {
		for _, node := range n.control_nodes {
			n.numlinks += len(node.Incoming)
			n.numlinks += len(node.Outgoing)
		}
	}
	return n.numlinks
}

// Returns complexity of this network which is sum of nodes count and links count
func (n *Network) Complexity() int {
	return n.NodeCount() + n.LinkCount()
}

// This checks a POTENTIAL link between a potential in_node
// and potential out_node to see if it must be recurrent.
// Use count and thresh to jump out in the case of an infinite loop.
func (n *Network) IsRecurrent(in_node, out_node *NNode, count *int, thresh int) bool {
	// Count the node as visited
	*count++

	if *count > thresh {
		return false // Short out the whole thing - loop detected
	}

	if in_node == out_node {
		return true
	} else {
		// Check back on all links ...
		for _, link := range in_node.Incoming {
			// But skip links that are already recurrent -
			// We want to check back through the forward flow of signals only
			if !link.IsRecurrent {
				if n.IsRecurrent(link.InNode, out_node, count, thresh) {
					return true
				}
			}
		}
	}
	return false
}

// Find the maximum number of neurons between an output and an input
func (n *Network) MaxDepth() (int, error) {
	if len(n.control_nodes) > 0 {
		return -1, errors.New("unsupported for modular networks")
	}
	// The quick case when there are no hidden nodes
	if len(n.all_nodes) == len(n.inputs) + len(n.Outputs) && len(n.control_nodes) == 0 {
		return 1, nil // just one layer depth
	}

	max := 0 // The max depth
	for _, node := range n.Outputs {
		curr_depth, err := node.Depth(0)
		if err != nil {
			return curr_depth, err
		}
		if curr_depth > max {
			max = curr_depth
		}
	}

	return max, nil
}

// Returns all nodes in the network
func (n *Network) AllNodes() []*NNode {
	return n.all_nodes
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

// runtime·duffzero is a Duff's device for zeroing memory.
// The compiler jumps to computed addresses within
// the routine to zero chunks of memory.
// Do not change duffzero without also
// changing clearfat in cmd/?g/ggen.go.

// runtime·duffcopy is a Duff's device for copying memory.
// The compiler jumps to computed addresses within
// the routine to copy chunks of memory.
// Source and destination must not overlap.
// Do not change duffcopy without also
// changing blockcopy in cmd/?g/cgen.go.

// See the zero* and copy* generators below
// for architecture-specific comments.

// mkduff generates duff_*.s.
package main

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
)

func main() {
	gen("amd64", notags, zeroAMD64, copyAMD64)
	gen("386", notags, zero386, copy386)
	gen("arm", notags, zeroARM, copyARM)
	gen("arm64", notags, zeroARM64, copyARM64)
	gen("ppc64x", tagsPPC64x, zeroPPC64x, copyPPC64x)
	gen("mips64x", tagsMIPS64x, zeroMIPS64x, copyMIPS64x)
}

func gen(arch string, tags, zero, copy func(io.Writer)) {
	var buf bytes.Buffer

	fmt.Fprintln(&buf, "// AUTO-GENERATED by mkduff.go")
	fmt.Fprintln(&buf, "// Run go generate from src/runtime to update.")
	fmt.Fprintln(&buf, "// See mkduff.go for comments.")
	tags(&buf)
	fmt.Fprintln(&buf, "#include \"textflag.h\"")
	fmt.Fprintln(&buf)
	zero(&buf)
	fmt.Fprintln(&buf)
	copy(&buf)

	if err := ioutil.WriteFile("duff_"+arch+".s", buf.Bytes(), 0644); err != nil {
		log.Fatalln(err)
	}
}

func notags(w io.Writer) { fmt.Fprintln(w) }

func zeroAMD64(w io.Writer) {
	// X0: zero
	// DI: ptr to memory to be zeroed
	// DI is updated as a side effect.
	fmt.Fprintln(w, "TEXT runtime·duffzero(SB), NOSPLIT, $0-0")
	for i := 0; i < 16; i++ {
		fmt.Fprintln(w, "\tMOVUPS\tX0,(DI)")
		fmt.Fprintln(w, "\tMOVUPS\tX0,16(DI)")
		fmt.Fprintln(w, "\tMOVUPS\tX0,32(DI)")
		fmt.Fprintln(w, "\tMOVUPS\tX0,48(DI)")
		fmt.Fprintln(w, "\tADDQ\t$64,DI")
		fmt.Fprintln(w)
	}
	fmt.Fprintln(w, "\tRET")
}

func copyAMD64(w io.Writer) {
	// SI: ptr to source memory
	// DI: ptr to destination memory
	// SI and DI are updated as a side effect.
	//
	// This is equivalent to a sequence of MOVSQ but
	// for some reason that is 3.5x slower than this code.
	// The STOSQ in duffzero seem fine, though.
	fmt.Fprintln(w, "TEXT runtime·duffcopy(SB), NOSPLIT, $0-0")
	for i := 0; i < 64; i++ {
		fmt.Fprintln(w, "\tMOVUPS\t(SI), X0")
		fmt.Fprintln(w, "\tADDQ\t$16, SI")
		fmt.Fprintln(w, "\tMOVUPS\tX0, (DI)")
		fmt.Fprintln(w, "\tADDQ\t$16, DI")
		fmt.Fprintln(w)
	}
	fmt.Fprintln(w, "\tRET")
}

func zero386(w io.Writer) {
	// AX: zero
	// DI: ptr to memory to be zeroed
	// DI is updated as a side effect.
	fmt.Fprintln(w, "TEXT runtime·duffzero(SB), NOSPLIT, $0-0")
	for i := 0; i < 128; i++ {
		fmt.Fprintln(w, "\tSTOSL")
	}
	fmt.Fprintln(w, "\tRET")
}

func copy386(w io.Writer) {
	// SI: ptr to source memory
	// DI: ptr to destination memory
	// SI and DI are updated as a side effect.
	//
	// This is equivalent to a sequence of MOVSL but
	// for some reason MOVSL is really slow.
	fmt.Fprintln(w, "TEXT runtime·duffcopy(SB), NOSPLIT, $0-0")
	for i := 0; i < 128; i++ {
		fmt.Fprintln(w, "\tMOVL\t(SI), CX")
		fmt.Fprintln(w, "\tADDL\t$4, SI")
		fmt.Fprintln(w, "\tMOVL\tCX, (DI)")
		fmt.Fprintln(w, "\tADDL\t$4, DI")
		fmt.Fprintln(w)
	}
	fmt.Fprintln(w, "\tRET")
}

func zeroARM(w io.Writer) {
	// R0: zero
	// R1: ptr to memory to be zeroed
	// R1 is updated as a side effect.
	fmt.Fprintln(w, "TEXT runtime·duffzero(SB), NOSPLIT, $0-0")
	for i := 0; i < 128; i++ {
		fmt.Fprintln(w, "\tMOVW.P\tR0, 4(R1)")
	}
	fmt.Fprintln(w, "\tRET")
}

func copyARM(w io.Writer) {
	// R0: scratch space
	// R1: ptr to source memory
	// R2: ptr to destination memory
	// R1 and R2 are updated as a side effect
	fmt.Fprintln(w, "TEXT runtime·duffcopy(SB), NOSPLIT, $0-0")
	for i := 0; i < 128; i++ {
		fmt.Fprintln(w, "\tMOVW.P\t4(R1), R0")
		fmt.Fprintln(w, "\tMOVW.P\tR0, 4(R2)")
		fmt.Fprintln(w)
	}
	fmt.Fprintln(w, "\tRET")
}

func zeroARM64(w io.Writer) {
	// ZR: always zero
	// R16 (aka REGRT1): ptr to memory to be zeroed - 8
	// On return, R16 points to the last zeroed dword.
	fmt.Fprintln(w, "TEXT runtime·duffzero(SB), NOSPLIT, $-8-0")
	for i := 0; i < 128; i++ {
		fmt.Fprintln(w, "\tMOVD.W\tZR, 8(R16)")
	}
	fmt.Fprintln(w, "\tRET")
}

func copyARM64(w io.Writer) {
	// R16 (aka REGRT1): ptr to source memory
	// R17 (aka REGRT2): ptr to destination memory
	// R27 (aka REGTMP): scratch space
	// R16 and R17 are updated as a side effect
	fmt.Fprintln(w, "TEXT runtime·duffcopy(SB), NOSPLIT, $0-0")
	for i := 0; i < 128; i++ {
		fmt.Fprintln(w, "\tMOVD.P\t8(R16), R27")
		fmt.Fprintln(w, "\tMOVD.P\tR27, 8(R17)")
		fmt.Fprintln(w)
	}
	fmt.Fprintln(w, "\tRET")
}

func tagsPPC64x(w io.Writer) {
	fmt.Fprintln(w)
	fmt.Fprintln(w, "// +build ppc64 ppc64le")
	fmt.Fprintln(w)
}

func zeroPPC64x(w io.Writer) {
	// R0: always zero
	// R3 (aka REGRT1): ptr to memory to be zeroed - 8
	// On return, R3 points to the last zeroed dword.
	fmt.Fprintln(w, "TEXT runtime·duffzero(SB), NOSPLIT|NOFRAME, $0-0")
	for i := 0; i < 128; i++ {
		fmt.Fprintln(w, "\tMOVDU\tR0, 8(R3)")
	}
	fmt.Fprintln(w, "\tRET")
}

func copyPPC64x(w io.Writer) {
	fmt.Fprintln(w, "// TODO: Implement runtime·duffcopy.")
}

func tagsMIPS64x(w io.Writer) {
	fmt.Fprintln(w)
	fmt.Fprintln(w, "// +build mips64 mips64le")
	fmt.Fprintln(w)
}

func zeroMIPS64x(w io.Writer) {
	// R0: always zero
	// R1 (aka REGRT1): ptr to memory to be zeroed - 8
	// On return, R1 points to the last zeroed dword.
	fmt.Fprintln(w, "TEXT runtime·duffzero(SB), NOSPLIT, $-8-0")
	for i := 0; i < 128; i++ {
		fmt.Fprintln(w, "\tMOVV\tR0, 8(R1)")
		fmt.Fprintln(w, "\tADDV\t$8, R1")
	}
	fmt.Fprintln(w, "\tRET")
}

func copyMIPS64x(w io.Writer) {
	fmt.Fprintln(w, "// TODO: Implement runtime·duffcopy.")
}
<commit_msg>runtime: update mkduff legacy comments<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

// runtime·duffzero is a Duff's device for zeroing memory.
// The compiler jumps to computed addresses within
// the routine to zero chunks of memory.
// Do not change duffzero without also
// changing the uses in cmd/compile/internal/*/*.go.

// runtime·duffcopy is a Duff's device for copying memory.
// The compiler jumps to computed addresses within
// the routine to copy chunks of memory.
// Source and destination must not overlap.
// Do not change duffcopy without also
// changing the uses in cmd/compile/internal/*/*.go.

// See the zero* and copy* generators below
// for architecture-specific comments.

// mkduff generates duff_*.s.
package main

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
)

func main() {
	gen("amd64", notags, zeroAMD64, copyAMD64)
	gen("386", notags, zero386, copy386)
	gen("arm", notags, zeroARM, copyARM)
	gen("arm64", notags, zeroARM64, copyARM64)
	gen("ppc64x", tagsPPC64x, zeroPPC64x, copyPPC64x)
	gen("mips64x", tagsMIPS64x, zeroMIPS64x, copyMIPS64x)
}

func gen(arch string, tags, zero, copy func(io.Writer)) {
	var buf bytes.Buffer

	fmt.Fprintln(&buf, "// AUTO-GENERATED by mkduff.go")
	fmt.Fprintln(&buf, "// Run go generate from src/runtime to update.")
	fmt.Fprintln(&buf, "// See mkduff.go for comments.")
	tags(&buf)
	fmt.Fprintln(&buf, "#include \"textflag.h\"")
	fmt.Fprintln(&buf)
	zero(&buf)
	fmt.Fprintln(&buf)
	copy(&buf)

	if err := ioutil.WriteFile("duff_"+arch+".s", buf.Bytes(), 0644); err != nil {
		log.Fatalln(err)
	}
}

func notags(w io.Writer) { fmt.Fprintln(w) }

func zeroAMD64(w io.Writer) {
	// X0: zero
	// DI: ptr to memory to be zeroed
	// DI is updated as a side effect.
	fmt.Fprintln(w, "TEXT runtime·duffzero(SB), NOSPLIT, $0-0")
	for i := 0; i < 16; i++ {
		fmt.Fprintln(w, "\tMOVUPS\tX0,(DI)")
		fmt.Fprintln(w, "\tMOVUPS\tX0,16(DI)")
		fmt.Fprintln(w, "\tMOVUPS\tX0,32(DI)")
		fmt.Fprintln(w, "\tMOVUPS\tX0,48(DI)")
		fmt.Fprintln(w, "\tADDQ\t$64,DI")
		fmt.Fprintln(w)
	}
	fmt.Fprintln(w, "\tRET")
}

func copyAMD64(w io.Writer) {
	// SI: ptr to source memory
	// DI: ptr to destination memory
	// SI and DI are updated as a side effect.
	//
	// This is equivalent to a sequence of MOVSQ but
	// for some reason that is 3.5x slower than this code.
	// The STOSQ in duffzero seem fine, though.
	fmt.Fprintln(w, "TEXT runtime·duffcopy(SB), NOSPLIT, $0-0")
	for i := 0; i < 64; i++ {
		fmt.Fprintln(w, "\tMOVUPS\t(SI), X0")
		fmt.Fprintln(w, "\tADDQ\t$16, SI")
		fmt.Fprintln(w, "\tMOVUPS\tX0, (DI)")
		fmt.Fprintln(w, "\tADDQ\t$16, DI")
		fmt.Fprintln(w)
	}
	fmt.Fprintln(w, "\tRET")
}

func zero386(w io.Writer) {
	// AX: zero
	// DI: ptr to memory to be zeroed
	// DI is updated as a side effect.
	fmt.Fprintln(w, "TEXT runtime·duffzero(SB), NOSPLIT, $0-0")
	for i := 0; i < 128; i++ {
		fmt.Fprintln(w, "\tSTOSL")
	}
	fmt.Fprintln(w, "\tRET")
}

func copy386(w io.Writer) {
	// SI: ptr to source memory
	// DI: ptr to destination memory
	// SI and DI are updated as a side effect.
	//
	// This is equivalent to a sequence of MOVSL but
	// for some reason MOVSL is really slow.
	fmt.Fprintln(w, "TEXT runtime·duffcopy(SB), NOSPLIT, $0-0")
	for i := 0; i < 128; i++ {
		fmt.Fprintln(w, "\tMOVL\t(SI), CX")
		fmt.Fprintln(w, "\tADDL\t$4, SI")
		fmt.Fprintln(w, "\tMOVL\tCX, (DI)")
		fmt.Fprintln(w, "\tADDL\t$4, DI")
		fmt.Fprintln(w)
	}
	fmt.Fprintln(w, "\tRET")
}

func zeroARM(w io.Writer) {
	// R0: zero
	// R1: ptr to memory to be zeroed
	// R1 is updated as a side effect.
	fmt.Fprintln(w, "TEXT runtime·duffzero(SB), NOSPLIT, $0-0")
	for i := 0; i < 128; i++ {
		fmt.Fprintln(w, "\tMOVW.P\tR0, 4(R1)")
	}
	fmt.Fprintln(w, "\tRET")
}

func copyARM(w io.Writer) {
	// R0: scratch space
	// R1: ptr to source memory
	// R2: ptr to destination memory
	// R1 and R2 are updated as a side effect
	fmt.Fprintln(w, "TEXT runtime·duffcopy(SB), NOSPLIT, $0-0")
	for i := 0; i < 128; i++ {
		fmt.Fprintln(w, "\tMOVW.P\t4(R1), R0")
		fmt.Fprintln(w, "\tMOVW.P\tR0, 4(R2)")
		fmt.Fprintln(w)
	}
	fmt.Fprintln(w, "\tRET")
}

func zeroARM64(w io.Writer) {
	// ZR: always zero
	// R16 (aka REGRT1): ptr to memory to be zeroed - 8
	// On return, R16 points to the last zeroed dword.
	fmt.Fprintln(w, "TEXT runtime·duffzero(SB), NOSPLIT, $-8-0")
	for i := 0; i < 128; i++ {
		fmt.Fprintln(w, "\tMOVD.W\tZR, 8(R16)")
	}
	fmt.Fprintln(w, "\tRET")
}

func copyARM64(w io.Writer) {
	// R16 (aka REGRT1): ptr to source memory
	// R17 (aka REGRT2): ptr to destination memory
	// R27 (aka REGTMP): scratch space
	// R16 and R17 are updated as a side effect
	fmt.Fprintln(w, "TEXT runtime·duffcopy(SB), NOSPLIT, $0-0")
	for i := 0; i < 128; i++ {
		fmt.Fprintln(w, "\tMOVD.P\t8(R16), R27")
		fmt.Fprintln(w, "\tMOVD.P\tR27, 8(R17)")
		fmt.Fprintln(w)
	}
	fmt.Fprintln(w, "\tRET")
}

func tagsPPC64x(w io.Writer) {
	fmt.Fprintln(w)
	fmt.Fprintln(w, "// +build ppc64 ppc64le")
	fmt.Fprintln(w)
}

func zeroPPC64x(w io.Writer) {
	// R0: always zero
	// R3 (aka REGRT1): ptr to memory to be zeroed - 8
	// On return, R3 points to the last zeroed dword.
	fmt.Fprintln(w, "TEXT runtime·duffzero(SB), NOSPLIT|NOFRAME, $0-0")
	for i := 0; i < 128; i++ {
		fmt.Fprintln(w, "\tMOVDU\tR0, 8(R3)")
	}
	fmt.Fprintln(w, "\tRET")
}

func copyPPC64x(w io.Writer) {
	fmt.Fprintln(w, "// TODO: Implement runtime·duffcopy.")
}

func tagsMIPS64x(w io.Writer) {
	fmt.Fprintln(w)
	fmt.Fprintln(w, "// +build mips64 mips64le")
	fmt.Fprintln(w)
}

func zeroMIPS64x(w io.Writer) {
	// R0: always zero
	// R1 (aka REGRT1): ptr to memory to be zeroed - 8
	// On return, R1 points to the last zeroed dword.
	fmt.Fprintln(w, "TEXT runtime·duffzero(SB), NOSPLIT, $-8-0")
	for i := 0; i < 128; i++ {
		fmt.Fprintln(w, "\tMOVV\tR0, 8(R1)")
		fmt.Fprintln(w, "\tADDV\t$8, R1")
	}
	fmt.Fprintln(w, "\tRET")
}

func copyMIPS64x(w io.Writer) {
	fmt.Fprintln(w, "// TODO: Implement runtime·duffcopy.")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"time"

	"golang.org/x/net/context"
	"k8s.io/kubernetes/pkg/api"
	uapi "k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/labels"
	"k8s.io/kubernetes/pkg/watch"
)

type watchFunc func() (watch.Interface, error)

type event struct {
	watch.Event
}

func (e event) String() string {
	m, er := getMeta(e.Object)
	if er != nil {
		return fmt.Sprintf("Event: type=%v object=Unknown", e.Type)
	}
	if s, ok := e.Object.(*uapi.Status); ok {
		return fmt.Sprintf("Status: [%s] code=%d %q", s.Status, s.Code, s.Message)
	}
	return fmt.Sprintf(
		"Event: [%v] object={Kind: %q, Name: %q, Namespace: %q} registerable=%v",
		e.Type, m.kind, m.name, m.ns, registerable(e.Object),
	)
}

func startWatches(c context.Context) (chan event, error) {
	resourceVersion = "0"
	out := make(chan event, 100)
	kc, er := kubeClient()
	if er != nil {
		return out, er
	}
	sv := func() (watch.Interface, error) {
		debugf("Attempting to set watch on Services")
		return kc.Services(api.NamespaceAll).Watch(labels.Everything(), fields.Everything(), resourceVersion)
	}
	en := func() (watch.Interface, error) {
		debugf("Attempting to set watch on Endpoints")
		return kc.Endpoints(api.NamespaceAll).Watch(labels.Everything(), fields.Everything(), resourceVersion)
	}

	go watcher("Services", sv, out, c)
	go watcher("Endpoints", en, out, c)
	return out, nil
}

func acquireWatch(fn watchFunc, out chan<- watch.Interface, c context.Context) {
	retry := 2 * time.Second
	t := time.NewTicker(retry)
	defer t.Stop()

	w, e := fn()
	if e == nil && c.Err() == nil {
		out <- w
		return
	}

	for {
		debugf("Setting watch failed, retry in (%v): %v", retry, e)
		select {
		case <-c.Done():
			return
		case <-t.C:
			w, e := fn()
			if e == nil && c.Err() == nil {
				out <- w
				return
			}
		}
	}
}

func watcher(name string, fn watchFunc, out chan<- event, c context.Context) {
	var w watch.Interface
	var wc = make(chan watch.Interface, 1)
	defer close(wc)

Acquire:
	go acquireWatch(fn, wc, c)
	select {
	case <-c.Done():
		infof("Closing %s watch channel", name)
		return
	case w = <-wc:
		debugf("%s watch set", name)
	}

EventLoop:
	for {
		select {
		case <-c.Done():
			infof("Closing %s watch channel", name)
			return
		case e := event{<-w.ResultChan()}:
			switch {
			case isClosed(e):
				warnf("%s watch closed: %v", name, e)
				goto Acquire
			case isError(e):
				errorf("%s watch error: %v", name, e)
				goto EventLoop
			case c.Err() == nil:
				out <- e
			}
		}
	}
}

func isClosed(e event) bool {
	return e.Event == watch.Event{}
}

func isError(e event) bool {
	return e.Type == watch.Error
}
<commit_msg>chan receive fix<commit_after>package main

import (
	"fmt"
	"time"

	"golang.org/x/net/context"
	"k8s.io/kubernetes/pkg/api"
	uapi "k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/labels"
	"k8s.io/kubernetes/pkg/watch"
)

type watchFunc func() (watch.Interface, error)

type event struct {
	watch.Event
}

func (e event) String() string {
	m, er := getMeta(e.Object)
	if er != nil {
		return fmt.Sprintf("Event: type=%v object=Unknown", e.Type)
	}
	if s, ok := e.Object.(*uapi.Status); ok {
		return fmt.Sprintf("Status: [%s] code=%d %q", s.Status, s.Code, s.Message)
	}
	return fmt.Sprintf(
		"Event: [%v] object={Kind: %q, Name: %q, Namespace: %q} registerable=%v",
		e.Type, m.kind, m.name, m.ns, registerable(e.Object),
	)
}

func startWatches(c context.Context) (chan event, error) {
	resourceVersion = "0"
	out := make(chan event, 100)
	kc, er := kubeClient()
	if er != nil {
		return out, er
	}
	sv := func() (watch.Interface, error) {
		debugf("Attempting to set watch on Services")
		return kc.Services(api.NamespaceAll).Watch(labels.Everything(), fields.Everything(), resourceVersion)
	}
	en := func() (watch.Interface, error) {
		debugf("Attempting to set watch on Endpoints")
		return kc.Endpoints(api.NamespaceAll).Watch(labels.Everything(), fields.Everything(), resourceVersion)
	}

	go watcher("Services", sv, out, c)
	go watcher("Endpoints", en, out, c)
	return out, nil
}

func acquireWatch(fn watchFunc, out chan<- watch.Interface, c context.Context) {
	retry := 2 * time.Second
	t := time.NewTicker(retry)
	defer t.Stop()

	w, e := fn()
	if e == nil && c.Err() == nil {
		out <- w
		return
	}

	for {
		debugf("Setting watch failed, retry in (%v): %v", retry, e)
		select {
		case <-c.Done():
			return
		case <-t.C:
			w, e := fn()
			if e == nil && c.Err() == nil {
				out <- w
				return
			}
		}
	}
}

func watcher(name string, fn watchFunc, out chan<- event, c context.Context) {
	var w watch.Interface
	var wc = make(chan watch.Interface, 1)
	defer close(wc)

Acquire:
	go acquireWatch(fn, wc, c)
	select {
	case <-c.Done():
		infof("Closing %s watch channel", name)
		return
	case w = <-wc:
		debugf("%s watch set", name)
	}

EventLoop:
	for {
		select {
		case <-c.Done():
			infof("Closing %s watch channel", name)
			return
		case ev := <-w.ResultChan():
			e := event{ev}
			switch {
			case isClosed(e):
				warnf("%s watch closed: %v", name, e)
				goto Acquire
			case isError(e):
				errorf("%s watch error: %v", name, e)
				goto EventLoop
			case c.Err() == nil:
				out <- e
			}
		}
	}
}

func isClosed(e event) bool {
	return e.Event == watch.Event{}
}

func isError(e event) bool {
	return e.Type == watch.Error
}
<|endoftext|>


<commit_before>package validation_test

import (
	"encoding/json"
	"fmt"
)

import (
	"github.com/xdave/validation"
)

type Employer struct {
	Name string `json:"name" valid:"Required"`
}

type Person struct {
	Name     string `json:"name" valid:"Required"`
	Age      int    `json:"age" valid:"Required;Min(18)"`
	Employer Employer
}

func ExampleValidationSuccess() {
	obj := Person{}
	input := `{
                "name": "John",
                "age": 35,
                "employer": {
                  "name": "Widgets, Inc."
                }
              }`
	if err := json.Unmarshal([]byte(input), &obj); err != nil {
		fmt.Println(err)
		return
	}
	fmt.Println(obj)
	// Output: {John 35 {Widgets, Inc.}}
}

func ExampleValidationFailure() {
	obj := Person{}
	input := `{ "age": 17 }`
	if err := json.Unmarshal([]byte(input), &obj); err != nil {
		fmt.Println(err)
		return
	}
	if errs := validation.Validate(obj); len(errs) > 0 {
		for _, err := range errs {
			fmt.Println(err)
		}
		return
	}
	fmt.Println(obj)
	// Output:
	// validation_test.Person validation failed: `Name` Can not be empty (actual value: "")
	// validation_test.Person validation failed: `Age` Minimum is 18 (actual value: 17)
	// validation_test.Person.Employer validation failed: `Name` Can not be empty (actual value: "")
}

func ExampleValidationFailure2() {
	obj := Person{}
	input := `{ "name": "Sam", "age": 18 }`
	if err := json.Unmarshal([]byte(input), &obj); err != nil {
		fmt.Println(err)
		return
	}
	if errs := validation.Validate(obj); len(errs) > 0 {
		for _, err := range errs {
			fmt.Println(err)
		}
		return
	}
	fmt.Println(obj)
	// Output:
	// validation_test.Person.Employer validation failed: `Name` Can not be empty (actual value: "")
}
<commit_msg>test more branches.  code coverage 100%.<commit_after>package validation_test

import (
	"encoding/json"
	"fmt"
	"testing"
)

import (
	"github.com/xdave/validation"
)

type Employer struct {
	Name string `json:"name" valid:"Required"`
}

type Person struct {
	Name     string `json:"name" valid:"Required"`
	Age      int    `json:"age" valid:"Required;Min(18)"`
	Employer Employer
}

func ExampleValidationSuccess() {
	obj := Person{}
	input := `{
                "name": "John",
                "age": 35,
                "employer": {
                  "name": "Widgets, Inc."
                }
              }`
	if err := json.Unmarshal([]byte(input), &obj); err != nil {
		fmt.Println(err)
		return
	}
	fmt.Println(obj)
	// Output: {John 35 {Widgets, Inc.}}
}

func ExampleValidationFailure() {
	obj := Person{}
	input := `{ "age": 17 }`
	if err := json.Unmarshal([]byte(input), &obj); err != nil {
		fmt.Println(err)
		return
	}
	if errs := validation.Validate(obj); len(errs) > 0 {
		for _, err := range errs {
			fmt.Println(err)
		}
		return
	}
	fmt.Println(obj)
	// Output:
	// validation_test.Person validation failed: `Name` Can not be empty (actual value: "")
	// validation_test.Person validation failed: `Age` Minimum is 18 (actual value: 17)
	// validation_test.Person.Employer validation failed: `Name` Can not be empty (actual value: "")
}

func ExampleValidationFailure2() {
	obj := Person{}
	input := `{ "name": "Sam", "age": 18 }`
	if err := json.Unmarshal([]byte(input), &obj); err != nil {
		fmt.Println(err)
		return
	}
	if errs := validation.Validate(obj); len(errs) > 0 {
		for _, err := range errs {
			fmt.Println(err)
		}
		return
	}
	fmt.Println(obj)
	// Output:
	// validation_test.Person.Employer validation failed: `Name` Can not be empty (actual value: "")
}

func TestInvalidObjectPassed(t *testing.T) {
	invalid := "a string"
	if errs := validation.Validate(invalid); len(errs) == 0 {
		t.Error("Length of errors is not greater than 0")
	}
}

func TestPointerPassed(t *testing.T) {
	obj := Person{
		Name: "Sam",
		Age:  18,
		Employer: Employer{
			Name: "Widgets, Inc.",
		},
	}
	errs := validation.Validate(&obj)
	if len(errs) > 0 {
		for _, err := range errs {
			t.Log(err)
		}
		t.Error("Expected validation of a pointer")
	}
}

func TestObjectWithSlicePassed(t *testing.T) {
	person := Person{
		Name: "bob",
		Age:  33,
		Employer: Employer{
			Name: "widgets",
		},
	}
	withslice := struct {
		People []Person
	}{
		People: []Person{person},
	}
	errs := validation.Validate(&withslice)
	if len(errs) > 0 {
		for _, err := range errs {
			t.Log(err)
		}
		t.Error("Expected validation of struct with slice of structs")
	}
}
<|endoftext|>


<commit_before>package routers

import (
	"net/http"

	"github.com/Unknwon/macaron"
)

func DownloadStats(ctx *macaron.Context, r *http.Request) {
	org := ctx.Params(":org")
	name := ctx.Params(":name")
	branch := ctx.Params(":branch")
	_ = branch
	repo := org + "/" + name

	domain := rdx.Get("domain:" + repo).Val()
	if domain == "" {
		ctx.Error(405, "repo not registed in gorelease, not open register for now")
		return
	}

	osarch := ctx.Params(":os") + "-" + ctx.Params(":arch")
	rdx.Incr("downloads:" + repo)
	rdx.Incr("downloads:" + repo + ":" + osarch)
	ctx.JSON(200, "update success")
	/*
		realURL := goutils.StrFormat("http://{domain}/gorelease/{name}/{branch}/{osarch}/{name}",
			map[string]interface{}{
				"domain": domain,
				"name":   name,
				"branch": branch,
				"osarch": osarch,
			})
		if ctx.Params(":os") == "windows" {
			realURL += ".exe"
		}
		ctx.Redirect(realURL, 302)
	*/
}
<commit_msg>clean code<commit_after>package routers

import (
	"net/http"

	"github.com/Unknwon/macaron"
)

func DownloadStats(ctx *macaron.Context, r *http.Request) {
	org := ctx.Params(":org")
	name := ctx.Params(":name")
	branch := ctx.Params(":branch")
	_ = branch
	repo := org + "/" + name

	/*
		domain := rdx.Get("domain:" + repo).Val()
		if domain == "" {
			ctx.Error(405, "repo not registed in gorelease, not open register for now")
			return
		}
	*/

	osarch := ctx.Params(":os") + "-" + ctx.Params(":arch")
	rdx.Incr("downloads:" + repo)
	rdx.Incr("downloads:" + repo + ":" + osarch)
	ctx.JSON(200, "update success")
}
<|endoftext|>


<commit_before>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Kathy Spradlin (kathyspradlin@gmail.com)

package rpc

import (
	"fmt"
	"math"
	"sort"
	"sync"
	"time"

	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/hlc"
	"github.com/cockroachdb/cockroach/util/log"
	"github.com/cockroachdb/cockroach/util/stop"
)

// How often the cluster offset is measured.
var monitorInterval = defaultHeartbeatInterval * 10

// RemoteClockMonitor keeps track of the most recent measurements of remote
// offsets from this node to connected nodes.
type RemoteClockMonitor struct {
	offsets map[string]RemoteOffset // Maps remote string addr to offset.
	lClock  *hlc.Clock              // The server clock.
	mu      sync.Mutex
	// Wall time in nanoseconds when we last monitored cluster offset.
	lastMonitoredAt int64
}

// ClusterOffsetInterval is the best interval we can construct to estimate this
// node's offset from the cluster.
type ClusterOffsetInterval struct {
	Lowerbound int64 // The lowerbound on the offset in nanoseconds.
	Upperbound int64 // The upperbound on the offset in nanoseconds.
}

func (i ClusterOffsetInterval) String() string {
	return fmt.Sprintf("{%s, %s}", time.Duration(i.Lowerbound), time.Duration(i.Upperbound))
}

// majorityIntervalNotFoundError indicates that we could not find a majority
// overlap in our estimate of remote clocks.
type majorityIntervalNotFoundError struct {
	endpoints endpointList
}

func (m *majorityIntervalNotFoundError) Error() string {
	return fmt.Sprintf("unable to determine the true cluster time from remote clock endpoints %v", m.endpoints)
}

// endpoint represents an endpoint in the interval estimation of a single
// remote clock. It could be either the lowpoint or the highpoint of the
// interval.
//
// For example, if the remote clock offset bounds are [-5, 10], then it
// will be converted into two endpoints:
// endpoint{offset: -5, endType: -1}
// endpoint{offset: 10, endType: +1}
type endpoint struct {
	offset  int64 // The boundary offset represented by this endpoint.
	endType int   // -1 if lowpoint, +1 if highpoint.
}

// endpointList is a slice of endpoints, sorted by endpoint offset.
type endpointList []endpoint

// Implementation of sort.Interface.
func (l endpointList) Len() int {
	return len(l)
}
func (l endpointList) Swap(i, j int) {
	l[i], l[j] = l[j], l[i]
}
func (l endpointList) Less(i, j int) bool {
	if l[i].offset == l[j].offset {
		return l[i].endType < l[j].endType
	}
	return l[i].offset < l[j].offset
}

// newRemoteClockMonitor returns a monitor with the given server clock.
func newRemoteClockMonitor(clock *hlc.Clock) *RemoteClockMonitor {
	return &RemoteClockMonitor{
		offsets: map[string]RemoteOffset{},
		lClock:  clock,
	}
}

// UpdateOffset is a thread-safe way to update the remote clock measurements.
//
// It only updates the offset for addr if one the following three cases holds:
// 1. There is no prior offset for that address.
// 2. The old offset for addr was measured before r.lastMonitoredAt. We never
// use values during monitoring that are older than r.lastMonitoredAt.
// 3. The new offset's error is smaller than the old offset's error.
//
// The third case allows the monitor to use the most precise clock reading of
// the remote addr during the next findOffsetInterval() invocation. We may
// measure the remote clock several times before we next calculate the cluster
// offset. When we do the measurement, we want to use the reading with the
// smallest error. Because monitorInterval > heartbeatInterval, this gives us
// several chances to accurately read the remote clock. Note that we don't want
// monitorInterval to be too large, else we might end up relying on old
// information.
func (r *RemoteClockMonitor) UpdateOffset(addr string, offset RemoteOffset) {
	if r == nil {
		return
	}

	r.mu.Lock()
	defer r.mu.Unlock()

	if oldOffset, ok := r.offsets[addr]; !ok {
		r.offsets[addr] = offset
	} else if oldOffset.MeasuredAt < r.lastMonitoredAt {
		// No matter what offset is, we weren't going to use oldOffset again,
		// because it was measured before the last cluster offset calculation.
		r.offsets[addr] = offset
	} else if offset.Uncertainty < oldOffset.Uncertainty {
		r.offsets[addr] = offset
	}

	if log.V(2) {
		log.Infof("update offset: %s %v", addr, r.offsets[addr])
	}
}

// MonitorRemoteOffsets periodically checks that the offset of this server's
// clock from the true cluster time is within MaxOffset. If the offset exceeds
// MaxOffset, then this method will trigger a fatal error, causing the node to
// suicide.
func (r *RemoteClockMonitor) MonitorRemoteOffsets(stopper *stop.Stopper) {
	if log.V(1) {
		log.Infof("monitoring cluster offset")
	}
	var monitorTimer util.Timer
	defer monitorTimer.Stop()
	for {
		monitorTimer.Reset(monitorInterval)
		select {
		case <-stopper.ShouldStop():
			return
		case <-monitorTimer.C:
			monitorTimer.Read = true
			offsetInterval, err := r.findOffsetInterval()
			// By the contract of the hlc, if the value is 0, then safety checking
			// of the max offset is disabled. However we may still want to
			// propagate the information to a status node.
			// TODO(embark): once there is a framework for collecting timeseries
			// data about the db, propagate the offset status to that.
			if r.lClock.MaxOffset() != 0 {
				if err != nil {
					log.Fatalf("clock offset from the cluster time "+
						"for remote clocks %v could not be determined: %s",
						r.offsets, err)
				}

				if !isHealthyOffsetInterval(offsetInterval, r.lClock.MaxOffset()) {
					log.Fatalf("clock offset from the cluster time "+
						"for remote clocks: %v is in interval: %s, which "+
						"indicates that the true offset is greater than %s",
						r.offsets, offsetInterval, r.lClock.MaxOffset())
				}
				if log.V(1) {
					log.Infof("healthy cluster offset: %s", offsetInterval)
				}
			}
			r.mu.Lock()
			r.lastMonitoredAt = r.lClock.PhysicalNow()
			r.mu.Unlock()
		}
	}
}

// isHealthyOffsetInterval returns true if the ClusterOffsetInterval indicates
// that the node's offset is within maxOffset, else false. For example, if the
// offset interval is [-20, -11] and the maxOffset is 10 nanoseconds, then the
// clock offset must be too great, because no point in the interval is within
// the maxOffset.
func isHealthyOffsetInterval(i ClusterOffsetInterval, maxOffset time.Duration) bool {
	if i.Lowerbound > maxOffset.Nanoseconds() ||
		i.Upperbound < -maxOffset.Nanoseconds() {
		return false
	}
	return true
}

// The routine that measures this node's probable offset from the rest of the
// cluster. This offset is measured as a ClusterOffsetInterval. For example,
// the output might be [-5, 10], which would indicate that this node's offset
// is likely between -5 and 10 nanoseconds from the average clock of the
// cluster.
//
// The intersection algorithm used here is documented at:
// http://infolab.stanford.edu/pub/cstr/reports/csl/tr/83/247/CSL-TR-83-247.pdf,
// commonly known as Marzullo's algorithm. If a remote clock is correct, its
// offset interval should encompass this clock's offset from the cluster time
// (see buildEndpointList()). If the majority of remote clock are correct, then
// their intervals should overlap over some region, which should include the
// true offset from the cluster time. This algorithm returns this region.
//
// If an interval cannot be found, an error is returned, indicating that
// a majority of remote node offset intervals do not overlap the cluster time.
func (r *RemoteClockMonitor) findOffsetInterval() (ClusterOffsetInterval, error) {
	endpoints := r.buildEndpointList()
	sort.Sort(endpoints)
	numClocks := len(endpoints) / 2
	if log.V(1) {
		log.Infof("finding offset interval for monitorInterval: %s, numOffsets %d",
			monitorInterval, numClocks)
	}
	if numClocks == 0 {
		return ClusterOffsetInterval{
			Lowerbound: 0,
			Upperbound: 0,
		}, nil
	}

	best := 0
	count := 0
	var lowerbound int64
	var upperbound int64

	// Find the interval which the most offset intervals overlap.
	for i, endpoint := range endpoints {
		count -= endpoint.endType
		if count > best {
			best = count
			lowerbound = endpoint.offset
			// Note the endType of the last endpoint is +1, so count < best.
			// Thus this code will never run when i = len(endpoint)-1.
			upperbound = endpoints[i+1].offset
		}
	}

	// Indicates that fewer than a majority of connected remote clocks seem to
	// encompass the central offset from the cluster, an error condition.
	if best <= numClocks/2 {
		return ClusterOffsetInterval{
				Lowerbound: math.MaxInt64,
				Upperbound: math.MaxInt64,
			}, &majorityIntervalNotFoundError{
				endpoints: endpoints,
			}
	}

	// A majority of offset intervals overlap at this interval, which should
	// contain the true cluster offset.
	return ClusterOffsetInterval{
		Lowerbound: lowerbound,
		Upperbound: upperbound,
	}, nil
}

// buildEndpointList() takes all the RemoteOffsets that are in the monitor, and
// turns these offsets into intervals which should encompass this node's true
// offset from the cluster time. It returns a list including the two endpoints
// of each interval.
//
// As a side effect, any RemoteOffsets that haven't been
// updated since the last monitoring are removed. (Side effects are nasty, but
// prevent us from running through the list an extra time under a lock).
//
// A RemoteOffset r is represented by this interval:
// [r.Offset - r.Error - MaxOffset, r.Offset + r.Error + MaxOffset],
// where MaxOffset is the furthest a node's clock can deviate from the cluster
// time. While the offset between this node and the remote time is actually
// within [r.Offset - r.Error, r.Offset + r.Error], we also must expand the
// interval by MaxOffset. This accounts for the fact that the remote clock is at
// most MaxOffset distance from the cluster time. Thus the expanded interval
// ought to contain this node's offset from the true cluster time, not just the
// offset from the remote clock's time.
func (r *RemoteClockMonitor) buildEndpointList() endpointList {
	r.mu.Lock()
	defer r.mu.Unlock()

	endpoints := make(endpointList, 0, len(r.offsets)*2)
	for addr, o := range r.offsets {
		// Remove anything that hasn't been updated since the last time offest
		// was measured. This indicates that we no longer have a connection to
		// that addr.
		if o.MeasuredAt < r.lastMonitoredAt {
			delete(r.offsets, addr)
			continue
		}

		lowpoint := endpoint{
			offset:  o.Offset - o.Uncertainty - r.lClock.MaxOffset().Nanoseconds(),
			endType: -1,
		}
		highpoint := endpoint{
			offset:  o.Offset + o.Uncertainty + r.lClock.MaxOffset().Nanoseconds(),
			endType: +1,
		}
		endpoints = append(endpoints, lowpoint, highpoint)
	}
	return endpoints
}
<commit_msg>rpc: update comment<commit_after>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Kathy Spradlin (kathyspradlin@gmail.com)

package rpc

import (
	"fmt"
	"math"
	"sort"
	"sync"
	"time"

	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/hlc"
	"github.com/cockroachdb/cockroach/util/log"
	"github.com/cockroachdb/cockroach/util/stop"
)

// How often the cluster offset is measured.
var monitorInterval = defaultHeartbeatInterval * 10

// RemoteClockMonitor keeps track of the most recent measurements of remote
// offsets from this node to connected nodes.
type RemoteClockMonitor struct {
	offsets map[string]RemoteOffset // Maps remote string addr to offset.
	lClock  *hlc.Clock              // The server clock.
	mu      sync.Mutex
	// Wall time in nanoseconds when we last monitored cluster offset.
	lastMonitoredAt int64
}

// ClusterOffsetInterval is the best interval we can construct to estimate this
// node's offset from the cluster.
type ClusterOffsetInterval struct {
	Lowerbound int64 // The lowerbound on the offset in nanoseconds.
	Upperbound int64 // The upperbound on the offset in nanoseconds.
}

func (i ClusterOffsetInterval) String() string {
	return fmt.Sprintf("{%s, %s}", time.Duration(i.Lowerbound), time.Duration(i.Upperbound))
}

// majorityIntervalNotFoundError indicates that we could not find a majority
// overlap in our estimate of remote clocks.
type majorityIntervalNotFoundError struct {
	endpoints endpointList
}

func (m *majorityIntervalNotFoundError) Error() string {
	return fmt.Sprintf("unable to determine the true cluster time from remote clock endpoints %v", m.endpoints)
}

// endpoint represents an endpoint in the interval estimation of a single
// remote clock. It could be either the lowpoint or the highpoint of the
// interval.
//
// For example, if the remote clock offset bounds are [-5, 10], then it
// will be converted into two endpoints:
// endpoint{offset: -5, endType: -1}
// endpoint{offset: 10, endType: +1}
type endpoint struct {
	offset  int64 // The boundary offset represented by this endpoint.
	endType int   // -1 if lowpoint, +1 if highpoint.
}

// endpointList is a slice of endpoints, sorted by endpoint offset.
type endpointList []endpoint

// Implementation of sort.Interface.
func (l endpointList) Len() int {
	return len(l)
}
func (l endpointList) Swap(i, j int) {
	l[i], l[j] = l[j], l[i]
}
func (l endpointList) Less(i, j int) bool {
	if l[i].offset == l[j].offset {
		return l[i].endType < l[j].endType
	}
	return l[i].offset < l[j].offset
}

// newRemoteClockMonitor returns a monitor with the given server clock.
func newRemoteClockMonitor(clock *hlc.Clock) *RemoteClockMonitor {
	return &RemoteClockMonitor{
		offsets: map[string]RemoteOffset{},
		lClock:  clock,
	}
}

// UpdateOffset is a thread-safe way to update the remote clock measurements.
//
// It only updates the offset for addr if one the following three cases holds:
// 1. There is no prior offset for that address.
// 2. The old offset for addr was measured before r.lastMonitoredAt. We never
// use values during monitoring that are older than r.lastMonitoredAt.
// 3. The new offset's error is smaller than the old offset's error.
//
// The third case allows the monitor to use the most precise clock reading of
// the remote addr during the next findOffsetInterval() invocation. We may
// measure the remote clock several times before we next calculate the cluster
// offset. When we do the measurement, we want to use the reading with the
// smallest error. Because monitorInterval > heartbeatInterval, this gives us
// several chances to accurately read the remote clock. Note that we don't want
// monitorInterval to be too large, else we might end up relying on old
// information.
func (r *RemoteClockMonitor) UpdateOffset(addr string, offset RemoteOffset) {
	if r == nil {
		return
	}

	r.mu.Lock()
	defer r.mu.Unlock()

	if oldOffset, ok := r.offsets[addr]; !ok {
		r.offsets[addr] = offset
	} else if oldOffset.MeasuredAt < r.lastMonitoredAt {
		// No matter what offset is, we weren't going to use oldOffset again,
		// because it was measured before the last cluster offset calculation.
		r.offsets[addr] = offset
	} else if offset.Uncertainty < oldOffset.Uncertainty {
		r.offsets[addr] = offset
	}

	if log.V(2) {
		log.Infof("update offset: %s %v", addr, r.offsets[addr])
	}
}

// MonitorRemoteOffsets periodically checks that the offset of this server's
// clock from the true cluster time is within MaxOffset. If the offset exceeds
// MaxOffset, then this method will trigger a fatal error, causing the node to
// suicide.
func (r *RemoteClockMonitor) MonitorRemoteOffsets(stopper *stop.Stopper) {
	if log.V(1) {
		log.Infof("monitoring cluster offset")
	}
	var monitorTimer util.Timer
	defer monitorTimer.Stop()
	for {
		monitorTimer.Reset(monitorInterval)
		select {
		case <-stopper.ShouldStop():
			return
		case <-monitorTimer.C:
			monitorTimer.Read = true
			offsetInterval, err := r.findOffsetInterval()
			// By the contract of the hlc, if the value is 0, then safety checking
			// of the max offset is disabled. However we may still want to
			// propagate the information to a status node.
			// TODO(embark): once there is a framework for collecting timeseries
			// data about the db, propagate the offset status to that.
			if r.lClock.MaxOffset() != 0 {
				if err != nil {
					log.Fatalf("clock offset from the cluster time "+
						"for remote clocks %v could not be determined: %s",
						r.offsets, err)
				}

				if !isHealthyOffsetInterval(offsetInterval, r.lClock.MaxOffset()) {
					log.Fatalf("clock offset from the cluster time "+
						"for remote clocks: %v is in interval: %s, which "+
						"indicates that the true offset is greater than %s",
						r.offsets, offsetInterval, r.lClock.MaxOffset())
				}
				if log.V(1) {
					log.Infof("healthy cluster offset: %s", offsetInterval)
				}
			}
			r.mu.Lock()
			r.lastMonitoredAt = r.lClock.PhysicalNow()
			r.mu.Unlock()
		}
	}
}

// isHealthyOffsetInterval returns true if the ClusterOffsetInterval indicates
// that the node's offset is within maxOffset, else false. For example, if the
// offset interval is [-20, -11] and the maxOffset is 10 nanoseconds, then the
// clock offset must be too great, because no point in the interval is within
// the maxOffset.
func isHealthyOffsetInterval(i ClusterOffsetInterval, maxOffset time.Duration) bool {
	if i.Lowerbound > maxOffset.Nanoseconds() ||
		i.Upperbound < -maxOffset.Nanoseconds() {
		return false
	}
	return true
}

// The routine that measures this node's probable offset from the rest of the
// cluster. This offset is measured as a ClusterOffsetInterval. For example,
// the output might be [-5, 10], which would indicate that this node's offset
// is likely between -5 and 10 nanoseconds from the average clock of the
// cluster.
//
// The intersection algorithm used here is documented at:
// http://infolab.stanford.edu/pub/cstr/reports/csl/tr/83/247/CSL-TR-83-247.pdf,
// commonly known as Marzullo's algorithm. If a remote clock is correct, its
// offset interval should encompass this clock's offset from the cluster time
// (see buildEndpointList()). If the majority of remote clock are correct, then
// their intervals should overlap over some region, which should include the
// true offset from the cluster time. This algorithm returns this region.
//
// If an interval cannot be found, an error is returned, indicating that
// a majority of remote node offset intervals do not overlap the cluster time.
func (r *RemoteClockMonitor) findOffsetInterval() (ClusterOffsetInterval, error) {
	endpoints := r.buildEndpointList()
	sort.Sort(endpoints)
	numClocks := len(endpoints) / 2
	if log.V(1) {
		log.Infof("finding offset interval for monitorInterval: %s, numOffsets %d",
			monitorInterval, numClocks)
	}
	if numClocks == 0 {
		return ClusterOffsetInterval{
			Lowerbound: 0,
			Upperbound: 0,
		}, nil
	}

	best := 0
	count := 0
	var lowerbound int64
	var upperbound int64

	// Find the interval which the most offset intervals overlap.
	for i, endpoint := range endpoints {
		count -= endpoint.endType
		if count > best {
			best = count
			lowerbound = endpoint.offset
			// Note the endType of the last endpoint is +1, so count < best.
			// Thus this code will never run when i = len(endpoint)-1.
			upperbound = endpoints[i+1].offset
		}
	}

	// Indicates that fewer than a majority of connected remote clocks seem to
	// encompass the central offset from the cluster, an error condition.
	if best <= numClocks/2 {
		return ClusterOffsetInterval{
				Lowerbound: math.MaxInt64,
				Upperbound: math.MaxInt64,
			}, &majorityIntervalNotFoundError{
				endpoints: endpoints,
			}
	}

	// A majority of offset intervals overlap at this interval, which should
	// contain the true cluster offset.
	return ClusterOffsetInterval{
		Lowerbound: lowerbound,
		Upperbound: upperbound,
	}, nil
}

// buildEndpointList() takes all the RemoteOffsets that are in the monitor, and
// turns these offsets into intervals which should encompass this node's true
// offset from the cluster time. It returns a list including the two endpoints
// of each interval.
//
// As a side effect, any RemoteOffsets that haven't been
// updated since the last monitoring are removed. (Side effects are nasty, but
// prevent us from running through the list an extra time under a lock).
//
// A RemoteOffset r is represented by this interval:
// [r.Offset - r.Uncertainty - MaxOffset, r.Offset + r.Uncertainty + MaxOffset],
// where MaxOffset is the furthest a node's clock can deviate from the cluster
// time. While the offset between this node and the remote time is actually
// within [r.Offset - r.Uncertainty, r.Offset + r.Uncertainty], we also must expand the
// interval by MaxOffset. This accounts for the fact that the remote clock is at
// most MaxOffset distance from the cluster time. Thus the expanded interval
// ought to contain this node's offset from the true cluster time, not just the
// offset from the remote clock's time.
func (r *RemoteClockMonitor) buildEndpointList() endpointList {
	r.mu.Lock()
	defer r.mu.Unlock()

	endpoints := make(endpointList, 0, len(r.offsets)*2)
	for addr, o := range r.offsets {
		// Remove anything that hasn't been updated since the last time offest
		// was measured. This indicates that we no longer have a connection to
		// that addr.
		if o.MeasuredAt < r.lastMonitoredAt {
			delete(r.offsets, addr)
			continue
		}

		lowpoint := endpoint{
			offset:  o.Offset - o.Uncertainty - r.lClock.MaxOffset().Nanoseconds(),
			endType: -1,
		}
		highpoint := endpoint{
			offset:  o.Offset + o.Uncertainty + r.lClock.MaxOffset().Nanoseconds(),
			endType: +1,
		}
		endpoints = append(endpoints, lowpoint, highpoint)
	}
	return endpoints
}
<|endoftext|>


<commit_before>package egl

/*
#include "bcm_host.h"
*/
import "C"

func BCMHostInit() {
	C.bcm_host_init()
}

func GraphicsGetDisplaySize(displayNumber uint16) (uint32,uint32) {
	var w,h uint32
	C.graphics_get_display_size((C.uint16_t)(displayNumber),(*C.uint32_t)(&w),(*C.uint32_t)(&h))
	return w,h
}<commit_msg>Updated<commit_after>package egl

/*
	#cgo CFLAGS: -I/opt/vc/include
	#cgo LDFLAGS: -L/opt/vc/lib -llibbcm_host
	#include "bcm_host.h"
*/
import "C"

func BCMHostInit() {
	C.bcm_host_init()
}

func GraphicsGetDisplaySize(displayNumber uint16) (uint32,uint32) {
	var w,h uint32
	C.graphics_get_display_size((C.uint16_t)(displayNumber),(*C.uint32_t)(&w),(*C.uint32_t)(&h))
	return w,h
}<|endoftext|>