<commit_before>package dns

// A structure for handling zone data

import (
	"github.com/miekg/radix"
	"strings"
	"sync"
	"time"
)

// Zone represents a DNS zone. It's safe for concurrent use by
// multilpe goroutines.
type Zone struct {
	Origin       string // Origin of the zone
	Wildcard     int    // Whenever we see a wildcard name, this is incremented
	*radix.Radix        // Zone data
	mutex        *sync.RWMutex
	// timemodified?
	expired bool // Slave zone is expired
}

// SignatureConfig holds the parameters for zone (re)signing. This
// is copied from OpenDNSSEC. See:
// https://wiki.opendnssec.org/display/DOCS/kasp.xml
type SignatureConfig struct {
	// Validity period of the signatures, typically 2 to 4 weeks.
	Validity time.Duration
	// When the end of the validity approaches, how much time should remain
	// before we start to resign. Typical value is 3 days.
	Refresh time.Duration
	// Jitter is an amount of time added or subtracted from the
	// expiration time to ensure not all signatures expire a the same time.
	// Typical value is 12 hours.
	Jitter time.Duration
	// InceptionOffset is subtracted from the inception time to ensure badly
	// calibrated clocks on the internet can still validate a signature.
	// Typical value is 300 seconds.
	InceptionOffset time.Duration
}

func newSignatureConfig() *SignatureConfig {
	return &SignatureConfig{time.Duration(4*7*24) * time.Hour, time.Duration(3*24) * time.Hour, time.Duration(12) * time.Hour, time.Duration(300) * time.Second}
}

// DefaultSignaturePolicy has the following values. Validity is 4 weeks,
// Refresh is set to 3 days, Jitter to 12 hours and InceptionOffset to 300 seconds.
var DefaultSignatureConfig = newSignatureConfig()

// NewZone creates an initialized zone with Origin set to origin.
func NewZone(origin string) *Zone {
	if origin == "" {
		origin = "."
	}
	if _, _, ok := IsDomainName(origin); !ok {
		return nil
	}
	z := new(Zone)
	z.mutex = new(sync.RWMutex)
	z.Origin = Fqdn(origin)
	z.Radix = radix.New()
	return z
}

// ZoneData holds all the RRs having their owner name equal to Name.
type ZoneData struct {
	Name       string                 // Domain name for this node
	RR         map[uint16][]RR        // Map of the RR type to the RR
	Signatures map[uint16][]*RR_RRSIG // DNSSEC signatures for the RRs, stored under type covered
	NonAuth    bool                   // Always false, except for NSsets that differ from z.Origin
	mutex      *sync.RWMutex
}

// newZoneData creates a new zone data element
func newZoneData(s string) *ZoneData {
	zd := new(ZoneData)
	zd.Name = s
	zd.RR = make(map[uint16][]RR)
	zd.Signatures = make(map[uint16][]*RR_RRSIG)
	zd.mutex = new(sync.RWMutex)
	return zd
}

// toRadixName reverses a domain name so that when we store it in the radix tree
// we preserve the nsec ordering of the zone (this idea was stolen from NSD).
// each label is also lowercased.
func toRadixName(d string) string {
	if d == "." {
		return "."
	}
	s := ""
	for _, l := range SplitLabels(d) {
		s = strings.ToLower(l) + "." + s
	}
	return "." + s
}

func (z *Zone) String() string {
	return z.Radix.String()
}

// Insert inserts an RR into the zone. There is no check for duplicate data, although
// Remove will remove all duplicates.
func (z *Zone) Insert(r RR) error {
	if !IsSubDomain(z.Origin, r.Header().Name) {
		return &Error{Err: "out of zone data", Name: r.Header().Name}
	}

	// TODO(mg): quick check for doubles
	key := toRadixName(r.Header().Name)
	z.mutex.Lock()
	zd := z.Radix.Find(key)
	if zd == nil {
		defer z.mutex.Unlock()
		// Check if it's a wildcard name
		if len(r.Header().Name) > 1 && r.Header().Name[0] == '*' && r.Header().Name[1] == '.' {
			z.Wildcard++
		}
		zd := newZoneData(r.Header().Name)
		switch t := r.Header().Rrtype; t {
		case TypeRRSIG:
			sigtype := r.(*RR_RRSIG).TypeCovered
			zd.Signatures[sigtype] = append(zd.Signatures[sigtype], r.(*RR_RRSIG))
		case TypeNS:
			// NS records with other names than z.Origin are non-auth
			if r.Header().Name != z.Origin {
				zd.NonAuth = true
			}
			fallthrough
		default:
			zd.RR[t] = append(zd.RR[t], r)
		}
		z.Radix.Insert(key, zd)
		return nil
	}
	z.mutex.Unlock()
	zd.Value.(*ZoneData).mutex.Lock()
	defer zd.Value.(*ZoneData).mutex.Unlock()
	// Name already there
	switch t := r.Header().Rrtype; t {
	case TypeRRSIG:
		sigtype := r.(*RR_RRSIG).TypeCovered
		zd.Value.(*ZoneData).Signatures[sigtype] = append(zd.Value.(*ZoneData).Signatures[sigtype], r.(*RR_RRSIG))
	case TypeNS:
		if r.Header().Name != z.Origin {
			zd.Value.(*ZoneData).NonAuth = true
		}
		fallthrough
	default:
		zd.Value.(*ZoneData).RR[t] = append(zd.Value.(*ZoneData).RR[t], r)
	}
	return nil
}

// Remove removes the RR r from the zone. If the RR can not be found,
// this is a no-op.
func (z *Zone) Remove(r RR) error {
	key := toRadixName(r.Header().Name)
	z.mutex.Lock()
	zd := z.Radix.Find(key)
	if zd == nil {
		defer z.mutex.Unlock()
		return nil
	}
	z.mutex.Unlock()
	zd.Value.(*ZoneData).mutex.Lock()
	defer zd.Value.(*ZoneData).mutex.Unlock()
	remove := false
	switch t := r.Header().Rrtype; t {
	case TypeRRSIG:
		sigtype := r.(*RR_RRSIG).TypeCovered
		for i, zr := range zd.Value.(*ZoneData).RR[sigtype] {
			if r == zr {
				zd.Value.(*ZoneData).RR[sigtype] = append(zd.Value.(*ZoneData).RR[sigtype][:i], zd.Value.(*ZoneData).RR[sigtype][i+1:]...)
				remove = true
			}
		}
	default:
		for i, zr := range zd.Value.(*ZoneData).RR[t] {
			if r == zr {
				zd.Value.(*ZoneData).RR[t] = append(zd.Value.(*ZoneData).RR[t][:i], zd.Value.(*ZoneData).RR[t][i+1:]...)
				remove = true
			}
		}
	}
	if remove && len(r.Header().Name) > 1 && r.Header().Name[0] == '*' && r.Header().Name[1] == '.' {
		z.Wildcard--
		if z.Wildcard < 0 {
			z.Wildcard = 0
		}
	}
	// TODO(mg): what to do if the whole structure is empty? Set it to nil?
	return nil
}

// Find looks up the ownername s in the zone and returns the
// data when found or nil when nothing is found.
func (z *Zone) Find(s string) *ZoneData {
	z.mutex.RLock()
	defer z.mutex.RUnlock()
	zd := z.Radix.Find(toRadixName(s))
	if zd == nil {
		return nil
	}
	return zd.Value.(*ZoneData)
}

// Predecessor searches the zone for a name shorter than s.
func (z *Zone) Predecessor(s string) *ZoneData {
	z.mutex.RLock()
	defer z.mutex.RUnlock()
	zd := z.Radix.Predecessor(toRadixName(s))
	if zd == nil {
		return nil
	}
	return zd.Value.(*ZoneData)
}

// Sign (re)signes the zone z. It adds keys to the zone (if not already there)
// and signs the keys with the KSKs and the rest of the zone with the ZSKs.
// NSEC is used for authenticated denial
// of existence. If config is nil DefaultSignatureConfig is used.
// TODO(mg): allow interaction with hsm
func (z *Zone) Sign(keys []*RR_DNSKEY, privkeys []PrivateKey, config *SignatureConfig) error {
	if config == nil {
		config = DefaultSignatureConfig
	}
	// TODO(mg): concurrently walk the zone and sign the rrsets
	// TODO(mg): nsec, or next pointer. Need to be a single tree-op

	return nil
}

// Sign each ZoneData in place.
// TODO(mg): assume not signed
func signZoneData(zd *ZoneData, privkeys []PrivateKey, signername string, config *SignatureConfig) {
	if zd.NonAuth == true {
		return
	}
	//s := new(RR_RRSIG)
	// signername

}
<commit_msg>dont append root here<commit_after>package dns

// A structure for handling zone data

import (
	"github.com/miekg/radix"
	"strings"
	"sync"
	"time"
)

// Zone represents a DNS zone. It's safe for concurrent use by
// multilpe goroutines.
type Zone struct {
	Origin       string // Origin of the zone
	Wildcard     int    // Whenever we see a wildcard name, this is incremented
	*radix.Radix        // Zone data
	mutex        *sync.RWMutex
	// timemodified?
	expired bool // Slave zone is expired
}

// SignatureConfig holds the parameters for zone (re)signing. This
// is copied from OpenDNSSEC. See:
// https://wiki.opendnssec.org/display/DOCS/kasp.xml
type SignatureConfig struct {
	// Validity period of the signatures, typically 2 to 4 weeks.
	Validity time.Duration
	// When the end of the validity approaches, how much time should remain
	// before we start to resign. Typical value is 3 days.
	Refresh time.Duration
	// Jitter is an amount of time added or subtracted from the
	// expiration time to ensure not all signatures expire a the same time.
	// Typical value is 12 hours.
	Jitter time.Duration
	// InceptionOffset is subtracted from the inception time to ensure badly
	// calibrated clocks on the internet can still validate a signature.
	// Typical value is 300 seconds.
	InceptionOffset time.Duration
}

func newSignatureConfig() *SignatureConfig {
	return &SignatureConfig{time.Duration(4*7*24) * time.Hour, time.Duration(3*24) * time.Hour, time.Duration(12) * time.Hour, time.Duration(300) * time.Second}
}

// DefaultSignaturePolicy has the following values. Validity is 4 weeks,
// Refresh is set to 3 days, Jitter to 12 hours and InceptionOffset to 300 seconds.
var DefaultSignatureConfig = newSignatureConfig()

// NewZone creates an initialized zone with Origin set to origin.
func NewZone(origin string) *Zone {
	if origin == "" {
		origin = "."
	}
	if _, _, ok := IsDomainName(origin); !ok {
		return nil
	}
	z := new(Zone)
	z.mutex = new(sync.RWMutex)
	z.Origin = Fqdn(origin)
	z.Radix = radix.New()
	return z
}

// ZoneData holds all the RRs having their owner name equal to Name.
type ZoneData struct {
	Name       string                 // Domain name for this node
	RR         map[uint16][]RR        // Map of the RR type to the RR
	Signatures map[uint16][]*RR_RRSIG // DNSSEC signatures for the RRs, stored under type covered
	NonAuth    bool                   // Always false, except for NSsets that differ from z.Origin
	mutex      *sync.RWMutex
}

// newZoneData creates a new zone data element
func newZoneData(s string) *ZoneData {
	zd := new(ZoneData)
	zd.Name = s
	zd.RR = make(map[uint16][]RR)
	zd.Signatures = make(map[uint16][]*RR_RRSIG)
	zd.mutex = new(sync.RWMutex)
	return zd
}

// toRadixName reverses a domain name so that when we store it in the radix tree
// we preserve the nsec ordering of the zone (this idea was stolen from NSD).
// each label is also lowercased.
func toRadixName(d string) string {
	if d == "." {
		return "."
	}
	s := ""
	for _, l := range SplitLabels(d) {
		s = strings.ToLower(l) + "." + s
	}
	return s
}

func (z *Zone) String() string {
	return z.Radix.String()
}

// Insert inserts an RR into the zone. There is no check for duplicate data, although
// Remove will remove all duplicates.
func (z *Zone) Insert(r RR) error {
	if !IsSubDomain(z.Origin, r.Header().Name) {
		return &Error{Err: "out of zone data", Name: r.Header().Name}
	}

	// TODO(mg): quick check for doubles
	key := toRadixName(r.Header().Name)
	z.mutex.Lock()
	zd := z.Radix.Find(key)
	if zd == nil {
		defer z.mutex.Unlock()
		// Check if it's a wildcard name
		if len(r.Header().Name) > 1 && r.Header().Name[0] == '*' && r.Header().Name[1] == '.' {
			z.Wildcard++
		}
		zd := newZoneData(r.Header().Name)
		switch t := r.Header().Rrtype; t {
		case TypeRRSIG:
			sigtype := r.(*RR_RRSIG).TypeCovered
			zd.Signatures[sigtype] = append(zd.Signatures[sigtype], r.(*RR_RRSIG))
		case TypeNS:
			// NS records with other names than z.Origin are non-auth
			if r.Header().Name != z.Origin {
				zd.NonAuth = true
			}
			fallthrough
		default:
			zd.RR[t] = append(zd.RR[t], r)
		}
		z.Radix.Insert(key, zd)
		return nil
	}
	z.mutex.Unlock()
	zd.Value.(*ZoneData).mutex.Lock()
	defer zd.Value.(*ZoneData).mutex.Unlock()
	// Name already there
	switch t := r.Header().Rrtype; t {
	case TypeRRSIG:
		sigtype := r.(*RR_RRSIG).TypeCovered
		zd.Value.(*ZoneData).Signatures[sigtype] = append(zd.Value.(*ZoneData).Signatures[sigtype], r.(*RR_RRSIG))
	case TypeNS:
		if r.Header().Name != z.Origin {
			zd.Value.(*ZoneData).NonAuth = true
		}
		fallthrough
	default:
		zd.Value.(*ZoneData).RR[t] = append(zd.Value.(*ZoneData).RR[t], r)
	}
	return nil
}

// Remove removes the RR r from the zone. If the RR can not be found,
// this is a no-op.
func (z *Zone) Remove(r RR) error {
	key := toRadixName(r.Header().Name)
	z.mutex.Lock()
	zd := z.Radix.Find(key)
	if zd == nil {
		defer z.mutex.Unlock()
		return nil
	}
	z.mutex.Unlock()
	zd.Value.(*ZoneData).mutex.Lock()
	defer zd.Value.(*ZoneData).mutex.Unlock()
	remove := false
	switch t := r.Header().Rrtype; t {
	case TypeRRSIG:
		sigtype := r.(*RR_RRSIG).TypeCovered
		for i, zr := range zd.Value.(*ZoneData).RR[sigtype] {
			if r == zr {
				zd.Value.(*ZoneData).RR[sigtype] = append(zd.Value.(*ZoneData).RR[sigtype][:i], zd.Value.(*ZoneData).RR[sigtype][i+1:]...)
				remove = true
			}
		}
	default:
		for i, zr := range zd.Value.(*ZoneData).RR[t] {
			if r == zr {
				zd.Value.(*ZoneData).RR[t] = append(zd.Value.(*ZoneData).RR[t][:i], zd.Value.(*ZoneData).RR[t][i+1:]...)
				remove = true
			}
		}
	}
	if remove && len(r.Header().Name) > 1 && r.Header().Name[0] == '*' && r.Header().Name[1] == '.' {
		z.Wildcard--
		if z.Wildcard < 0 {
			z.Wildcard = 0
		}
	}
	// TODO(mg): what to do if the whole structure is empty? Set it to nil?
	return nil
}

// Find looks up the ownername s in the zone and returns the
// data when found or nil when nothing is found.
func (z *Zone) Find(s string) *ZoneData {
	z.mutex.RLock()
	defer z.mutex.RUnlock()
	zd := z.Radix.Find(toRadixName(s))
	if zd == nil {
		return nil
	}
	return zd.Value.(*ZoneData)
}

// Predecessor searches the zone for a name shorter than s.
func (z *Zone) Predecessor(s string) *ZoneData {
	z.mutex.RLock()
	defer z.mutex.RUnlock()
	zd := z.Radix.Predecessor(toRadixName(s))
	if zd == nil {
		return nil
	}
	return zd.Value.(*ZoneData)
}

// Sign (re)signes the zone z. It adds keys to the zone (if not already there)
// and signs the keys with the KSKs and the rest of the zone with the ZSKs.
// NSEC is used for authenticated denial
// of existence. If config is nil DefaultSignatureConfig is used.
// TODO(mg): allow interaction with hsm
func (z *Zone) Sign(keys []*RR_DNSKEY, privkeys []PrivateKey, config *SignatureConfig) error {
	if config == nil {
		config = DefaultSignatureConfig
	}
	// TODO(mg): concurrently walk the zone and sign the rrsets
	// TODO(mg): nsec, or next pointer. Need to be a single tree-op

	return nil
}

// Sign each ZoneData in place.
// TODO(mg): assume not signed
func signZoneData(zd *ZoneData, privkeys []PrivateKey, signername string, config *SignatureConfig) {
	if zd.NonAuth == true {
		return
	}
	//s := new(RR_RRSIG)
	// signername

}
<|endoftext|>


<commit_before>package disgo

import (
	"bytes"
	"compress/zlib"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/gorilla/websocket"
	"github.com/slf4go/logger"
)

type shard struct {
	// Parent session
	session *Session

	// Websocket information used for identification and reconnection
	webSocket *websocket.Conn
	shard     int
	sessionID string
	sequence  int
	heartbeat int

	// Mutex locks, reconnect to make sure there is only 1 process reconnecting and concurrent read/write accesses on the socket
	reconnectLock sync.Mutex
	readL, writeL sync.Mutex

	// Channels to pass around messages
	closeMessage chan int
	stopListen   chan bool
	stopRead     chan bool
}

func connectShard(session *Session, shardNum int) (*shard, error) {
	conn, _, err := websocket.DefaultDialer.Dial(session.wsUrl, http.Header{})
	if err != nil {
		return nil, err
	}

	s := &shard{
		session:      session,
		webSocket:    conn,
		shard:        shardNum,
		closeMessage: make(chan int, 1),
		stopListen:   make(chan bool, 1),
		stopRead:     make(chan bool),
	}

	if err = s.handshake(); err != nil {
		return nil, err
	}

	return s, nil
}

func (s *shard) handshake() error {
	s.webSocket.SetCloseHandler(s.onClose)

	helloFrame, err := s.readFrame()
	if err != nil {
		return err
	} else if helloFrame.Op != opHello {
		return errors.New("First frame sent from the Discord Gateway is not hello")
	}

	hello := helloPayload{}
	if err = json.Unmarshal(helloFrame.Data, &hello); err != nil {
		return err
	}

	logger.Debugf("Connected to Discord servers: %s", strings.Join(hello.Servers, ", "))
	logger.Debugf("Setting up a heartbeat interval of %d ms", hello.HeartbeatInterval)
	s.heartbeat = hello.HeartbeatInterval

	if err = s.identify(); err != nil {
		return err
	}

	go s.mainLoop()
	return nil
}

func (s *shard) identify() error {
	if s.sessionID != "" {
		logger.Debugf("Resuming connection starting at sequence %d.", s.sequence)
		s.sendFrame(&gatewayFrame{opResume, resumePayload{
			Token:     s.session.token,
			SessionID: s.sessionID,
			Sequence:  s.sequence,
		}})
	} else {
		logger.Debugf("Identifying to websocket")
		s.sendFrame(&gatewayFrame{opIdentify, identifyPayload{
			Token:          s.session.token,
			Compress:       true,
			LargeThreshold: 250,
			Shard:          [2]int{s.shard, cap(s.session.shards)},
			Properties: propertiesPayload{
				OS:      runtime.GOOS,
				Browser: "DisGo",
				Device:  "DisGo",
			},
		}})
	}

	frame, err := s.readFrame()
	if err != nil {
		return err
	}

	if frame.Op == opDispatch {
		if frame.EventName == "READY" {
			ready := ReadyEvent{}
			if err := json.Unmarshal(frame.Data, &ready); err != nil {
				return err
			}

			s.sessionID = ready.SessionID
			s.session.dispatchEvent(frame)
			return nil
		} else if frame.EventName == "RESUMED" {
			s.session.dispatchEvent(frame)
			return nil
		} else {
			return fmt.Errorf("Discord sent event of type '%s', expected 'READY' or 'RESUMED'", frame.EventName)
		}
	} else if frame.Op == opInvalidSession {
		s.sessionID = "" // Invalidate session and retry
		s.identify()
		return nil
	} else {
		return fmt.Errorf("Unexpected opCode received from Discord: %d", frame.Op)
	}
}

func (s *shard) mainLoop() {
	logger.Debugf("Starting main loop for shard [%d/%d]", s.shard+1, cap(s.session.shards))
	defer logger.Debugf("Exiting main loop for shard [%d/%d]", s.shard+1, cap(s.session.shards))

	heartbeat := time.NewTicker(time.Duration(s.heartbeat) * time.Millisecond)
	defer heartbeat.Stop()
	sentHeartBeat := false

	reader := make(chan *receivedFrame, 1)
	go s.readWebSocket(reader)

	for {
		select {
		case <-heartbeat.C:
			if !sentHeartBeat {
				s.sendFrame(&gatewayFrame{opHeartbeat, s.sequence})
				sentHeartBeat = true
			} else {
				s.disconnect(websocket.CloseAbnormalClosure, "Did not respond to previous ping")
			}
		case <-s.stopListen:
			return
		case frame := <-reader:
			switch frame.Op {
			case opHeartbeat:
				s.sendFrame(&gatewayFrame{Op: opHeartbeatAck})
			case opHeartbeatAck:
				sentHeartBeat = false
			case opReconnect:
				s.disconnect(websocket.CloseNormalClosure, "op Reconnect")
			case opDispatch:
				s.session.dispatchEvent(frame)
			default:
				logger.Errorf("Unknown opCode received: %d", frame.Op)
			}
		}
	}
}

func (s *shard) sendFrame(frame *gatewayFrame) {
	// Resume will only be sent within the reconnect reconnectLock, we don't relock to prevent a deadlock
	if frame.Op != opResume {
		s.reconnectLock.Lock()
		defer s.reconnectLock.Unlock()
	}

	s.writeL.Lock()
	defer s.writeL.Unlock()

	logger.Debugf("Sending frame with opCode: %d", frame.Op)
	s.webSocket.WriteJSON(frame)
}

func (s *shard) readWebSocket(reader chan *receivedFrame) {
	logger.Debugf("Starting read loop for shard [%d/%d]", s.shard+1, cap(s.session.shards))
	defer logger.Debugf("Exiting read loop for shard [%d/%d]", s.shard+1, cap(s.session.shards))

	for {
		select {
		case <-s.stopRead:
			return
		default:
			frame, err := s.readFrame()
			if err != nil {
				if !s.session.isShuttingDown() {
					if closeErr, isClose := err.(*websocket.CloseError); isClose {
						s.closeMessage <- closeErr.Code
					}
					logger.ErrorE(err)
					go s.disconnect(websocket.CloseAbnormalClosure, err.Error())
				}
				<-s.stopRead // Wait for the connection to be closed
				return
			}

			reader <- frame
		}
	}

}

func (s *shard) readFrame() (*receivedFrame, error) {
	s.readL.Lock()
	defer s.readL.Unlock()

	logger.Tracef("Shard.readFrame() called")
	msgType, msg, err := s.webSocket.ReadMessage()
	if err != nil {
		return nil, err
	}

	var reader io.Reader
	reader = bytes.NewBuffer(msg)

	if msgType == websocket.BinaryMessage {
		zReader, err := zlib.NewReader(reader)
		if err != nil {
			return nil, err
		}

		defer zReader.Close()

		reader = zReader
	}

	frame := receivedFrame{Sequence: -1}
	json.NewDecoder(reader).Decode(&frame)

	logger.Debugf("Received frame with opCode: %d", frame.Op)

	if frame.Sequence != -1 {
		logger.Tracef("Last sequence received set to %d.", frame.Sequence)
		s.sequence = frame.Sequence
	}
	return &frame, nil
}

func (s *shard) reconnect(recursive bool) {
	if !recursive {
		s.reconnectLock.Lock()
	}

	if !s.session.isShuttingDown() {
		logger.Noticef("Reconnecting shard [%d/%d]", s.shard+1, cap(s.session.shards))
		conn, _, err := websocket.DefaultDialer.Dial(s.session.wsUrl, http.Header{})

		if err == nil {
			s.webSocket = conn
			err = s.handshake()

			if err != nil {
				conn.Close()
				s.webSocket = nil
			}
		}

		if err != nil {
			logger.ErrorE(err)
			time.Sleep(1 * time.Second)
			go s.reconnect(true)
		} else {
			s.reconnectLock.Unlock()
		}
	}
}

func (s *shard) onClose(code int, text string) error {
	logger.Debugf("Received Close Frame from Discord. Code: %d. Text: %s", code, text)
	s.closeMessage <- code
	s.reconnect(false)
	return nil
}

func (s *shard) disconnect(code int, text string) {
	logger.Tracef("Shard.disconnect() called")
	s.stopListen <- true

	s.writeL.Lock()
	err := s.webSocket.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(code, text))
	s.writeL.Unlock()
	if err != nil {
		logger.ErrorE(err)
	}

	select {
	case code := <-s.closeMessage:
		logger.Debugf("Discord connection closed with code %d", code)
	case <-time.After(1 * time.Second):
		logger.Warn("Discord did not reply to the close message, force-closing connection.")
	}

	s.webSocket.Close()
	s.stopRead <- true

	go s.reconnect(false)
}
<commit_msg>Heartbeat disconnections now start a new goroutine<commit_after>package disgo

import (
	"bytes"
	"compress/zlib"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/gorilla/websocket"
	"github.com/slf4go/logger"
)

type shard struct {
	// Parent session
	session *Session

	// Websocket information used for identification and reconnection
	webSocket *websocket.Conn
	shard     int
	sessionID string
	sequence  int
	heartbeat int

	// Mutex locks, reconnect to make sure there is only 1 process reconnecting and concurrent read/write accesses on the socket
	reconnectLock sync.Mutex
	readL, writeL sync.Mutex

	// Channels to pass around messages
	closeMessage chan int
	stopListen   chan bool
	stopRead     chan bool
}

func connectShard(session *Session, shardNum int) (*shard, error) {
	conn, _, err := websocket.DefaultDialer.Dial(session.wsUrl, http.Header{})
	if err != nil {
		return nil, err
	}

	s := &shard{
		session:      session,
		webSocket:    conn,
		shard:        shardNum,
		closeMessage: make(chan int, 1),
		stopListen:   make(chan bool),
		stopRead:     make(chan bool),
	}

	if err = s.handshake(); err != nil {
		return nil, err
	}

	return s, nil
}

func (s *shard) handshake() error {
	s.webSocket.SetCloseHandler(s.onClose)

	helloFrame, err := s.readFrame()
	if err != nil {
		return err
	} else if helloFrame.Op != opHello {
		return errors.New("First frame sent from the Discord Gateway is not hello")
	}

	hello := helloPayload{}
	if err = json.Unmarshal(helloFrame.Data, &hello); err != nil {
		return err
	}

	logger.Debugf("Connected to Discord servers: %s", strings.Join(hello.Servers, ", "))
	logger.Debugf("Setting up a heartbeat interval of %d ms", hello.HeartbeatInterval)
	s.heartbeat = hello.HeartbeatInterval

	if err = s.identify(); err != nil {
		return err
	}

	go s.mainLoop()
	return nil
}

func (s *shard) identify() error {
	if s.sessionID != "" {
		logger.Debugf("Resuming connection starting at sequence %d.", s.sequence)
		s.sendFrame(&gatewayFrame{opResume, resumePayload{
			Token:     s.session.token,
			SessionID: s.sessionID,
			Sequence:  s.sequence,
		}})
	} else {
		logger.Debugf("Identifying to websocket")
		s.sendFrame(&gatewayFrame{opIdentify, identifyPayload{
			Token:          s.session.token,
			Compress:       true,
			LargeThreshold: 250,
			Shard:          [2]int{s.shard, cap(s.session.shards)},
			Properties: propertiesPayload{
				OS:      runtime.GOOS,
				Browser: "DisGo",
				Device:  "DisGo",
			},
		}})
	}

	frame, err := s.readFrame()
	if err != nil {
		return err
	}

	if frame.Op == opDispatch {
		if frame.EventName == "READY" {
			ready := ReadyEvent{}
			if err := json.Unmarshal(frame.Data, &ready); err != nil {
				return err
			}

			s.sessionID = ready.SessionID
			s.session.dispatchEvent(frame)
			return nil
		} else if frame.EventName == "RESUMED" {
			s.session.dispatchEvent(frame)
			return nil
		} else {
			return fmt.Errorf("Discord sent event of type '%s', expected 'READY' or 'RESUMED'", frame.EventName)
		}
	} else if frame.Op == opInvalidSession {
		s.sessionID = "" // Invalidate session and retry
		s.identify()
		return nil
	} else {
		return fmt.Errorf("Unexpected opCode received from Discord: %d", frame.Op)
	}
}

func (s *shard) mainLoop() {
	logger.Debugf("Starting main loop for shard [%d/%d]", s.shard+1, cap(s.session.shards))
	defer logger.Debugf("Exiting main loop for shard [%d/%d]", s.shard+1, cap(s.session.shards))

	heartbeat := time.NewTicker(time.Duration(s.heartbeat) * time.Millisecond)
	defer heartbeat.Stop()
	sentHeartBeat := false

	reader := make(chan *receivedFrame, 1)
	go s.readWebSocket(reader)

	for {
		select {
		case <-heartbeat.C:
			if !sentHeartBeat {
				s.sendFrame(&gatewayFrame{opHeartbeat, s.sequence})
				sentHeartBeat = true
			} else {
				go s.disconnect(websocket.CloseAbnormalClosure, "Did not respond to previous ping")
			}
		case <-s.stopListen:
			return
		case frame := <-reader:
			switch frame.Op {
			case opHeartbeat:
				s.sendFrame(&gatewayFrame{Op: opHeartbeatAck})
			case opHeartbeatAck:
				sentHeartBeat = false
			case opReconnect:
				s.disconnect(websocket.CloseNormalClosure, "op Reconnect")
			case opDispatch:
				s.session.dispatchEvent(frame)
			default:
				logger.Errorf("Unknown opCode received: %d", frame.Op)
			}
		}
	}
}

func (s *shard) sendFrame(frame *gatewayFrame) {
	// Resume will only be sent within the reconnect reconnectLock, we don't relock to prevent a deadlock
	if frame.Op != opResume {
		s.reconnectLock.Lock()
		defer s.reconnectLock.Unlock()
	}

	s.writeL.Lock()
	defer s.writeL.Unlock()

	logger.Debugf("Sending frame with opCode: %d", frame.Op)
	s.webSocket.WriteJSON(frame)
}

func (s *shard) readWebSocket(reader chan *receivedFrame) {
	logger.Debugf("Starting read loop for shard [%d/%d]", s.shard+1, cap(s.session.shards))
	defer logger.Debugf("Exiting read loop for shard [%d/%d]", s.shard+1, cap(s.session.shards))

	for {
		select {
		case <-s.stopRead:
			return
		default:
			frame, err := s.readFrame()
			if err != nil {
				if !s.session.isShuttingDown() {
					if closeErr, isClose := err.(*websocket.CloseError); isClose {
						s.closeMessage <- closeErr.Code
					}
					logger.ErrorE(err)
					go s.disconnect(websocket.CloseAbnormalClosure, err.Error())
				}
				<-s.stopRead // Wait for the connection to be closed
				return
			}

			reader <- frame
		}
	}

}

func (s *shard) readFrame() (*receivedFrame, error) {
	s.readL.Lock()
	defer s.readL.Unlock()

	logger.Tracef("Shard.readFrame() called")
	msgType, msg, err := s.webSocket.ReadMessage()
	if err != nil {
		return nil, err
	}

	var reader io.Reader
	reader = bytes.NewBuffer(msg)

	if msgType == websocket.BinaryMessage {
		zReader, err := zlib.NewReader(reader)
		if err != nil {
			return nil, err
		}

		defer zReader.Close()

		reader = zReader
	}

	frame := receivedFrame{Sequence: -1}
	json.NewDecoder(reader).Decode(&frame)

	logger.Debugf("Received frame with opCode: %d", frame.Op)

	if frame.Sequence != -1 {
		logger.Tracef("Last sequence received set to %d.", frame.Sequence)
		s.sequence = frame.Sequence
	}
	return &frame, nil
}

func (s *shard) reconnect(recursive bool) {
	if !recursive {
		s.reconnectLock.Lock()
	}

	if !s.session.isShuttingDown() {
		logger.Noticef("Reconnecting shard [%d/%d]", s.shard+1, cap(s.session.shards))
		conn, _, err := websocket.DefaultDialer.Dial(s.session.wsUrl, http.Header{})

		if err == nil {
			s.webSocket = conn
			err = s.handshake()

			if err != nil {
				conn.Close()
				s.webSocket = nil
			}
		}

		if err != nil {
			logger.ErrorE(err)
			time.Sleep(1 * time.Second)
			go s.reconnect(true)
		} else {
			s.reconnectLock.Unlock()
		}
	}
}

func (s *shard) onClose(code int, text string) error {
	logger.Debugf("Received Close Frame from Discord. Code: %d. Text: %s", code, text)
	s.closeMessage <- code
	s.reconnect(false)
	return nil
}

func (s *shard) disconnect(code int, text string) {
	logger.Tracef("Shard.disconnect() called")
	s.stopListen <- true

	s.writeL.Lock()
	err := s.webSocket.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(code, text))
	s.writeL.Unlock()
	if err != nil {
		logger.ErrorE(err)
	}

	select {
	case code := <-s.closeMessage:
		logger.Debugf("Discord connection closed with code %d", code)
	case <-time.After(1 * time.Second):
		logger.Warn("Discord did not reply to the close message, force-closing connection.")
	}

	s.webSocket.Close()
	s.stopRead <- true

	go s.reconnect(false)
}
<|endoftext|>


<commit_before>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"golang.org/x/crypto/ssh/terminal"
	"io"
	"os"
	"path/filepath"

	"github.com/couchbase/query/errors"
	"github.com/couchbaselabs/go_cbq/command"
	go_n1ql "github.com/couchbaselabs/go_n1ql"
)

/*
   Command line options provided.
*/

/*
   Option        : -engine or -e
   Args          :  <url to the query service or to the cluster>
   Default value : http://localhost:8091/
   Point to the cluser/query endpoint to connect to.
*/
var ServerFlag string

func init() {
	const (
		defaultserver = "http://localhost:8091/"
		usage         = "URL to the query service/cluster. \n\t\t Default : http://localhost:8091\n\n Usage: cbq \n\t\t Connects to local couchbase instance. Same as: cbq -engine=http://localhost:8091\n\t cbq -engine=http://172.23.107.18:8093 \n\t\t Connects to query node at 172.23.107.18 Port 8093 \n\t cbq -engine=https://my.secure.node.com:8093 \n\t\t Connects to query node at my.secure.node.com:8093 using secure https protocol."
	)
	flag.StringVar(&ServerFlag, "engine", defaultserver, usage)
	flag.StringVar(&ServerFlag, "e", defaultserver, "Shorthand for -engine")
}

/*
   Option        : -no-engine or -ne
   Default value : false
   Enable/Disable startup connection to a query service/cluster endpoint.
*/
var NoQueryService bool

func init() {
	const (
		defaultval = false
		usage      = "Start shell without connecting to a query service/cluster endpoint. \n\t\t Default : false \n\t\t Possible Values : true/false"
	)
	flag.BoolVar(&NoQueryService, "no-engine", defaultval, usage)
	flag.BoolVar(&NoQueryService, "ne", defaultval, " Shorthand for -no-engine")
}

/*
   Option        : -quiet
   Default value : false
   Enable/Disable startup connection message for the shell
*/
var quietFlag bool

func init() {
	const (
		defaultval = false
		usage      = "Enable/Disable startup connection message for the shell \n\t\t Default : false \n\t\t Possible Values : true/false"
	)
	flag.BoolVar(&quietFlag, "quiet", defaultval, usage)
	flag.BoolVar(&quietFlag, "q", defaultval, " Shorthand for -quiet")
}

/*
   Option        : -timeout or -t
   Args          : <timeout value>
   Default value : "0ms"
   Query timeout parameter.
*/

var timeoutFlag string

func init() {
	const (
		defaultval = ""
		usage      = "Query timeout parameter. Units are mandatory. For Example : \"10ms\". \n\t\t Valid Units : ns (nanoseconds), us (microseconds), ms (milliseconds), s (seconds), m (minutes), h (hours) "
	)
	flag.StringVar(&timeoutFlag, "timeout", defaultval, usage)
	flag.StringVar(&timeoutFlag, "t", defaultval, " Shorthand for -timeout")
}

/*
   Option        : -user or -u
   Args          : Login username
   Login credentials for users. The shell will prompt for the password.
*/

var userFlag string

func init() {
	const (
		defaultval = ""
		usage      = "Username \n\t For Example : -u=Administrator"
	)
	flag.StringVar(&userFlag, "user", defaultval, usage)
	flag.StringVar(&userFlag, "u", defaultval, " Shorthand for -user")

}

/*
   Option        : -credentials or -c
   Args          : A list of credentials, in the form of user/password objects.
   Login credentials for users as well as SASL Buckets.
*/

var credsFlag string

func init() {
	const (
		defaultval = ""
		usage      = "A list of credentials, in the form user:password. \n\t For Example : Administrator:password, beer-sample:asdasd"
	)
	flag.StringVar(&credsFlag, "credentials", defaultval, usage)
	flag.StringVar(&credsFlag, "c", defaultval, " Shorthand for -credentials")

}

/*
   Option        : -version or -v
   Shell Version
*/

var versionFlag bool

func init() {
	const (
		usage = "Shell Version \n\t Usage: -version"
	)
	flag.BoolVar(&versionFlag, "version", false, usage)
	flag.BoolVar(&versionFlag, "v", false, "Shorthand for -version")

}

/*
   Option        : -script or -s
   Args          : <query>
   Single command mode
*/

var scriptFlag string

func init() {
	const (
		defaultval = ""
		usage      = "Single command mode. Execute input command and exit shell. \n\t For Example : -script=\"select * from system:keyspaces\""
	)
	flag.StringVar(&scriptFlag, "script", defaultval, usage)
	flag.StringVar(&scriptFlag, "s", defaultval, " Shorthand for -script")

}

/*
   Option        : -pretty
   Default value : false
   Pretty print output
*/

var prettyFlag = flag.Bool("pretty", true, "Pretty print the output.")

/*
   Option        : -exit-on-error
   Default value : false
   Exit shell after first error encountered.
*/

var errorExitFlag = flag.Bool("exit-on-error", false, "Exit shell after first error encountered.")

/*
   Option        : -file or -f
   Args          : <filename>
   Input file to run queries from. Exit after the queries are run.
*/

var inputFlag string

func init() {
	const (
		defaultval = ""
		usage      = "File to load commands from. \n\t For Example : -file=temp.txt"
	)
	flag.StringVar(&inputFlag, "file", defaultval, usage)
	flag.StringVar(&inputFlag, "f", defaultval, " Shorthand for -file")

}

/*
   Option        : -ouput or -o
   Args          : <filename>
   Output file to send results of queries to.
*/

var outputFlag string

func init() {
	const (
		defaultval = ""
		usage      = "File to output commands and their results. \n\t For Example : -output=temp.txt"
	)
	flag.StringVar(&outputFlag, "output", defaultval, usage)
	flag.StringVar(&outputFlag, "o", defaultval, " Shorthand for -output")

}

/*
   Option        : -log-file or -l
   Args          : <filename>
   Log commands for session.
*/

var logFlag string

func init() {
	const (
		defaultval = ""
		usage      = "File to log commands into. \n\t For Example : -log-file=temp.txt"
	)
	flag.StringVar(&logFlag, "log-file", defaultval, usage)
	flag.StringVar(&logFlag, "l", defaultval, " Shorthand for -log-file")

}

/* Define credentials as user/pass and convert into
   JSON object credentials
*/

var (
	SERVICE_URL string
	DISCONNECT  bool
	EXIT        bool
)

func main() {

	flag.Parse()
	command.W = os.Stdout

	if scriptFlag != "" {
		err_code, err_str := execute_input(scriptFlag, os.Stdout)
		if err_code != 0 {
			s_err := command.HandleError(err_code, err_str)
			command.PrintError(s_err)
			os.Exit(1)
		}
		os.Exit(0)
	}

	/* Handle options and what they should do */

	// TODO : Readd ...
	//Taken out so as to connect to both cluster and query service
	//using go_n1ql.
	/*
		if strings.HasPrefix(ServerFlag, "http://") == false {
			ServerFlag = "http://" + ServerFlag
		}

		urlRegex := "^(https?://)[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]"
		match, _ := regexp.MatchString(urlRegex, ServerFlag)
		if match == false {
			//TODO Isha : Add error code. Throw invalid url error
			fmt.Println("Invalid url please check" + ServerFlag)
		}

		//-engine
		if strings.HasSuffix(ServerFlag, "/") == false {
			ServerFlag = ServerFlag + "/"
		}
	*/

	/* -quiet : Display Message only if flag not specified
	 */
	if !quietFlag && NoQueryService == false {
		s := fmt.Sprintln("Connect to " + ServerFlag + ". Type Ctrl-D to exit.\n")
		_, werr := io.WriteString(command.W, s)
		if werr != nil {
			s_err := command.HandleError(errors.WRITER_OUTPUT, werr.Error())
			command.PrintError(s_err)
		}
	}

	/* -version : Display the version of the shell and then exit.
	 */
	if versionFlag == true {
		dummy := []string{}
		cmd := command.Version{}
		cmd.ExecCommand(dummy)
		os.Exit(0)
	}

	/* -user : Accept Admin credentials. Prompt for password and set
	   the n1ql_creds. Append to creds so that user can also define
	   bucket credentials using -credentials if they need to.
	*/
	var creds command.Credentials

	if userFlag != "" {
		s := fmt.Sprintln("Enter Password: ")
		_, werr := io.WriteString(command.W, s)
		if werr != nil {
			s_err := command.HandleError(errors.WRITER_OUTPUT, werr.Error())
			command.PrintError(s_err)
		}
		password, err := terminal.ReadPassword(0)
		if err == nil {
			if string(password) == "" {
				s_err := command.HandleError(errors.INVALID_PASSWORD, "")
				command.PrintError(s_err)
				os.Exit(1)
			} else {
				creds = append(creds, command.Credential{"user": userFlag, "pass": string(password)})
			}
		} else {
			s_err := command.HandleError(errors.INVALID_PASSWORD, err.Error())
			command.PrintError(s_err)
			os.Exit(1)
		}
	}

	/* -credentials : Accept credentials to pass to the n1ql endpoint.
	   Ensure that the user inputs credentials in the form a:b.
	   It is important to apend these credentials to those given by
	   -user.
	*/
	if userFlag == "" && credsFlag == "" {
		/* No credentials exist. This can still be used to connect to
		   un-authenticated servers.
		*/
		_, werr := io.WriteString(command.W, "No Input Credentials. In order to connect to a server with authentication, please provide credentials.\n")
		if werr != nil {
			s_err := command.HandleError(errors.WRITER_OUTPUT, werr.Error())
			command.PrintError(s_err)
		}

	} else if credsFlag != "" {

		creds_ret, err_code, err_string := command.ToCreds(credsFlag)
		if err_code != 0 {
			s_err := command.HandleError(err_code, err_string)
			command.PrintError(s_err)
		}
		for _, v := range creds_ret {
			creds = append(creds, v)
		}

	}
	//Append empty credentials. This is used for cases where one of the buckets
	//is a SASL bucket, and we need to access the other unprotected buckets.
	//CBauth works this way.

	//if credsFlag == "" && userFlag != "" {
	creds = append(creds, command.Credential{"user": "", "pass": ""})
	//}

	/* Add the credentials set by -user and -credentials to the
	   go_n1ql creds parameter.
	*/
	if creds != nil {
		ac, err := json.Marshal(creds)
		if err != nil {
			//Error while Marshalling
			s_err := command.HandleError(errors.JSON_MARSHAL, err.Error())
			command.PrintError(s_err)
			os.Exit(1)
		}
		go_n1ql.SetQueryParams("creds", string(ac))
	}

	if timeoutFlag != "0ms" {
		go_n1ql.SetQueryParams("timeout", timeoutFlag)
	}

	if inputFlag != "" {
		//Read each line from the file and call execute query

	}

	go_n1ql.SetPassthroughMode(true)
	//fmt.Println("Input arguments, ", os.Args)
	HandleInteractiveMode(filepath.Base(os.Args[0]))
}
<commit_msg>Fix single command mode panic<commit_after>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"golang.org/x/crypto/ssh/terminal"
	"io"
	"os"
	"path/filepath"

	"github.com/couchbase/query/errors"
	"github.com/couchbaselabs/go_cbq/command"
	go_n1ql "github.com/couchbaselabs/go_n1ql"
)

/*
   Command line options provided.
*/

/*
   Option        : -engine or -e
   Args          :  <url to the query service or to the cluster>
   Default value : http://localhost:8091/
   Point to the cluser/query endpoint to connect to.
*/
var ServerFlag string

func init() {
	const (
		defaultserver = "http://localhost:8091/"
		usage         = "URL to the query service/cluster. \n\t\t Default : http://localhost:8091\n\n Usage: cbq \n\t\t Connects to local couchbase instance. Same as: cbq -engine=http://localhost:8091\n\t cbq -engine=http://172.23.107.18:8093 \n\t\t Connects to query node at 172.23.107.18 Port 8093 \n\t cbq -engine=https://my.secure.node.com:8093 \n\t\t Connects to query node at my.secure.node.com:8093 using secure https protocol."
	)
	flag.StringVar(&ServerFlag, "engine", defaultserver, usage)
	flag.StringVar(&ServerFlag, "e", defaultserver, "Shorthand for -engine")
}

/*
   Option        : -no-engine or -ne
   Default value : false
   Enable/Disable startup connection to a query service/cluster endpoint.
*/
var NoQueryService bool

func init() {
	const (
		defaultval = false
		usage      = "Start shell without connecting to a query service/cluster endpoint. \n\t\t Default : false \n\t\t Possible Values : true/false"
	)
	flag.BoolVar(&NoQueryService, "no-engine", defaultval, usage)
	flag.BoolVar(&NoQueryService, "ne", defaultval, " Shorthand for -no-engine")
}

/*
   Option        : -quiet
   Default value : false
   Enable/Disable startup connection message for the shell
*/
var quietFlag bool

func init() {
	const (
		defaultval = false
		usage      = "Enable/Disable startup connection message for the shell \n\t\t Default : false \n\t\t Possible Values : true/false"
	)
	flag.BoolVar(&quietFlag, "quiet", defaultval, usage)
	flag.BoolVar(&quietFlag, "q", defaultval, " Shorthand for -quiet")
}

/*
   Option        : -timeout or -t
   Args          : <timeout value>
   Default value : "0ms"
   Query timeout parameter.
*/

var timeoutFlag string

func init() {
	const (
		defaultval = ""
		usage      = "Query timeout parameter. Units are mandatory. For Example : \"10ms\". \n\t\t Valid Units : ns (nanoseconds), us (microseconds), ms (milliseconds), s (seconds), m (minutes), h (hours) "
	)
	flag.StringVar(&timeoutFlag, "timeout", defaultval, usage)
	flag.StringVar(&timeoutFlag, "t", defaultval, " Shorthand for -timeout")
}

/*
   Option        : -user or -u
   Args          : Login username
   Login credentials for users. The shell will prompt for the password.
*/

var userFlag string

func init() {
	const (
		defaultval = ""
		usage      = "Username \n\t For Example : -u=Administrator"
	)
	flag.StringVar(&userFlag, "user", defaultval, usage)
	flag.StringVar(&userFlag, "u", defaultval, " Shorthand for -user")

}

/*
   Option        : -credentials or -c
   Args          : A list of credentials, in the form of user/password objects.
   Login credentials for users as well as SASL Buckets.
*/

var credsFlag string

func init() {
	const (
		defaultval = ""
		usage      = "A list of credentials, in the form user:password. \n\t For Example : Administrator:password, beer-sample:asdasd"
	)
	flag.StringVar(&credsFlag, "credentials", defaultval, usage)
	flag.StringVar(&credsFlag, "c", defaultval, " Shorthand for -credentials")

}

/*
   Option        : -version or -v
   Shell Version
*/

var versionFlag bool

func init() {
	const (
		usage = "Shell Version \n\t Usage: -version"
	)
	flag.BoolVar(&versionFlag, "version", false, usage)
	flag.BoolVar(&versionFlag, "v", false, "Shorthand for -version")

}

/*
   Option        : -script or -s
   Args          : <query>
   Single command mode
*/

var scriptFlag string

func init() {
	const (
		defaultval = ""
		usage      = "Single command mode. Execute input command and exit shell. \n\t For Example : -script=\"select * from system:keyspaces\""
	)
	flag.StringVar(&scriptFlag, "script", defaultval, usage)
	flag.StringVar(&scriptFlag, "s", defaultval, " Shorthand for -script")

}

/*
   Option        : -pretty
   Default value : false
   Pretty print output
*/

var prettyFlag = flag.Bool("pretty", true, "Pretty print the output.")

/*
   Option        : -exit-on-error
   Default value : false
   Exit shell after first error encountered.
*/

var errorExitFlag = flag.Bool("exit-on-error", false, "Exit shell after first error encountered.")

/*
   Option        : -file or -f
   Args          : <filename>
   Input file to run queries from. Exit after the queries are run.
*/

var inputFlag string

func init() {
	const (
		defaultval = ""
		usage      = "File to load commands from. \n\t For Example : -file=temp.txt"
	)
	flag.StringVar(&inputFlag, "file", defaultval, usage)
	flag.StringVar(&inputFlag, "f", defaultval, " Shorthand for -file")

}

/*
   Option        : -ouput or -o
   Args          : <filename>
   Output file to send results of queries to.
*/

var outputFlag string

func init() {
	const (
		defaultval = ""
		usage      = "File to output commands and their results. \n\t For Example : -output=temp.txt"
	)
	flag.StringVar(&outputFlag, "output", defaultval, usage)
	flag.StringVar(&outputFlag, "o", defaultval, " Shorthand for -output")

}

/*
   Option        : -log-file or -l
   Args          : <filename>
   Log commands for session.
*/

var logFlag string

func init() {
	const (
		defaultval = ""
		usage      = "File to log commands into. \n\t For Example : -log-file=temp.txt"
	)
	flag.StringVar(&logFlag, "log-file", defaultval, usage)
	flag.StringVar(&logFlag, "l", defaultval, " Shorthand for -log-file")

}

/* Define credentials as user/pass and convert into
   JSON object credentials
*/

var (
	SERVICE_URL string
	DISCONNECT  bool
	EXIT        bool
)

func main() {

	flag.Parse()
	command.W = os.Stdout

	/* Handle options and what they should do */

	// TODO : Readd ...
	//Taken out so as to connect to both cluster and query service
	//using go_n1ql.
	/*
		if strings.HasPrefix(ServerFlag, "http://") == false {
			ServerFlag = "http://" + ServerFlag
		}

		urlRegex := "^(https?://)[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]"
		match, _ := regexp.MatchString(urlRegex, ServerFlag)
		if match == false {
			//TODO Isha : Add error code. Throw invalid url error
			fmt.Println("Invalid url please check" + ServerFlag)
		}

		//-engine
		if strings.HasSuffix(ServerFlag, "/") == false {
			ServerFlag = ServerFlag + "/"
		}
	*/

	/* -quiet : Display Message only if flag not specified
	 */
	if !quietFlag && NoQueryService == false {
		s := fmt.Sprintln("Connect to " + ServerFlag + ". Type Ctrl-D to exit.\n")
		_, werr := io.WriteString(command.W, s)
		if werr != nil {
			s_err := command.HandleError(errors.WRITER_OUTPUT, werr.Error())
			command.PrintError(s_err)
		}
	}

	/* -version : Display the version of the shell and then exit.
	 */
	if versionFlag == true {
		dummy := []string{}
		cmd := command.Version{}
		cmd.ExecCommand(dummy)
		os.Exit(0)
	}

	/* -user : Accept Admin credentials. Prompt for password and set
	   the n1ql_creds. Append to creds so that user can also define
	   bucket credentials using -credentials if they need to.
	*/
	var creds command.Credentials

	if userFlag != "" {
		s := fmt.Sprintln("Enter Password: ")
		_, werr := io.WriteString(command.W, s)
		if werr != nil {
			s_err := command.HandleError(errors.WRITER_OUTPUT, werr.Error())
			command.PrintError(s_err)
		}
		password, err := terminal.ReadPassword(0)
		if err == nil {
			if string(password) == "" {
				s_err := command.HandleError(errors.INVALID_PASSWORD, "")
				command.PrintError(s_err)
				os.Exit(1)
			} else {
				creds = append(creds, command.Credential{"user": userFlag, "pass": string(password)})
			}
		} else {
			s_err := command.HandleError(errors.INVALID_PASSWORD, err.Error())
			command.PrintError(s_err)
			os.Exit(1)
		}
	}

	/* -credentials : Accept credentials to pass to the n1ql endpoint.
	   Ensure that the user inputs credentials in the form a:b.
	   It is important to apend these credentials to those given by
	   -user.
	*/
	if userFlag == "" && credsFlag == "" {
		// No credentials exist. This can still be used to connect to
		// un-authenticated servers.
		// Dont output the statement if we are running in single command
		// mode.
		if scriptFlag == "" {
			_, werr := io.WriteString(command.W, "No Input Credentials. In order to connect to a server with authentication, please provide credentials.\n")

			if werr != nil {
				s_err := command.HandleError(errors.WRITER_OUTPUT, werr.Error())
				command.PrintError(s_err)
			}
		}

	} else if credsFlag != "" {

		creds_ret, err_code, err_string := command.ToCreds(credsFlag)
		if err_code != 0 {
			s_err := command.HandleError(err_code, err_string)
			command.PrintError(s_err)
		}
		for _, v := range creds_ret {
			creds = append(creds, v)
		}

	}
	//Append empty credentials. This is used for cases where one of the buckets
	//is a SASL bucket, and we need to access the other unprotected buckets.
	//CBauth works this way.

	//if credsFlag == "" && userFlag != "" {
	creds = append(creds, command.Credential{"user": "", "pass": ""})
	//}

	/* Add the credentials set by -user and -credentials to the
	   go_n1ql creds parameter.
	*/
	if creds != nil {
		ac, err := json.Marshal(creds)
		if err != nil {
			//Error while Marshalling
			s_err := command.HandleError(errors.JSON_MARSHAL, err.Error())
			command.PrintError(s_err)
			os.Exit(1)
		}
		go_n1ql.SetQueryParams("creds", string(ac))
	}

	if scriptFlag != "" {
		go_n1ql.SetPassthroughMode(true)
		err_code, err_str := execute_input(scriptFlag, os.Stdout)
		if err_code != 0 {
			s_err := command.HandleError(err_code, err_str)
			command.PrintError(s_err)
			os.Exit(1)
		}
		os.Exit(0)
	}

	if timeoutFlag != "0ms" {
		go_n1ql.SetQueryParams("timeout", timeoutFlag)
	}

	if inputFlag != "" {
		//Read each line from the file and call execute query

	}

	go_n1ql.SetPassthroughMode(true)
	//fmt.Println("Input arguments, ", os.Args)
	HandleInteractiveMode(filepath.Base(os.Args[0]))
}
<|endoftext|>


<commit_before>/*
Copyright 2016 Citrix Systems, Inc

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main

import (
	"errors"
	netscaler "github.com/chiradeep/terraform-provider-netscaler/netscaler"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"

	"fmt"
	"log"
)

type Service struct {
	Name        string
	ServiceType string
	Ip          string
	Port        int
}

func (svc *Service) Id() string {
	return svc.Name
}

func (c *NetScalerNitroClient) CreateService(lbName string, svc *Service) error {
	nitroClient := netscaler.NewNitroClient(c.Endpoint, c.Username, c.Password)
	var svcStruct = new(netscaler.NetscalerService)
	svcStruct.Name = svc.Name
	svcStruct.ServiceType = svc.ServiceType
	svcStruct.Ip = svc.Ip
	svcStruct.Port = svc.Port
	err := nitroClient.AddAndBindService(lbName, svcStruct)
	if err != nil {
		log.Fatal("Failed to create service %s", svc.Name)
		return err
	}
	return nil
}

func (c *NetScalerNitroClient) DeleteService(svcName string) error {
	nitroClient := netscaler.NewNitroClient(c.Endpoint, c.Username, c.Password)
	err := nitroClient.DeleteService(svcName)
	if err != nil {
		log.Fatal("Failed to delete service %s", svcName)
		return err
	}
	return nil
}

func resourceNetScalerSvc() *schema.Resource {
	return &schema.Resource{
		SchemaVersion: 1,
		Create:        createSvcFunc,
		Read:          readSvcFunc,
		Update:        updateSvcFunc,
		Delete:        deleteSvcFunc,
		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},
			"ip": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"service_type": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},
			"port": &schema.Schema{
				Type:     schema.TypeInt,
				Required: true,
			},
			"lb": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
		},
	}
}

func createSvcFunc(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*NetScalerNitroClient)
	lbName := d.Get("lb").(string)
	lbFound := client.FindResource("lbvserver", lbName)
	if !lbFound {
		log.Printf("No lb with name %s found", lbName)
		return errors.New(fmt.Sprintf("No lb with name %s found", lbName))
	}
	var svcName string
	if v, ok := d.GetOk("name"); ok {
		svcName = v.(string)
	} else {
		svcName = resource.PrefixedUniqueId("tf-svc-")
		d.Set("name", svcName)
	}
	svc := Service{
		Name:        svcName,
		Ip:          d.Get("ip").(string),
		Port:        d.Get("port").(int),
		ServiceType: d.Get("service_type").(string),
	}

	err := client.CreateService(lbName, &svc)
	if err != nil {
		return err
	}

	d.SetId(svc.Id())

	return nil
}

func readSvcFunc(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*NetScalerNitroClient)
	svcName := d.Id()
	log.Printf("Reading service state %s", svcName)
	found := client.FindResource("service", svcName)
	if !found {
		log.Printf("Clearing service state %s", svcName)
		d.SetId("")
	}
	return nil
}

func updateSvcFunc(d *schema.ResourceData, meta interface{}) error {
	return nil
}

func deleteSvcFunc(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*NetScalerNitroClient)
	svcName := d.Id()
	err := client.DeleteService(svcName)
	if err != nil {
		return err
	}

	d.SetId("")

	return nil
}
<commit_msg>somehow name gets deleted when updating<commit_after>/*
Copyright 2016 Citrix Systems, Inc

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main

import (
	"errors"
	netscaler "github.com/chiradeep/terraform-provider-netscaler/netscaler"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"

	"fmt"
	"log"
)

type Service struct {
	Name        string
	ServiceType string
	Ip          string
	Port        int
}

func (svc *Service) Id() string {
	return svc.Name
}

func (c *NetScalerNitroClient) CreateService(lbName string, svc *Service) error {
	nitroClient := netscaler.NewNitroClient(c.Endpoint, c.Username, c.Password)
	var svcStruct = new(netscaler.NetscalerService)
	svcStruct.Name = svc.Name
	svcStruct.ServiceType = svc.ServiceType
	svcStruct.Ip = svc.Ip
	svcStruct.Port = svc.Port
	err := nitroClient.AddAndBindService(lbName, svcStruct)
	if err != nil {
		log.Fatal("Failed to create service %s", svc.Name)
		return err
	}
	return nil
}

func (c *NetScalerNitroClient) DeleteService(svcName string) error {
	nitroClient := netscaler.NewNitroClient(c.Endpoint, c.Username, c.Password)
	err := nitroClient.DeleteService(svcName)
	if err != nil {
		log.Fatal("Failed to delete service %s", svcName)
		return err
	}
	return nil
}

func resourceNetScalerSvc() *schema.Resource {
	return &schema.Resource{
		SchemaVersion: 1,
		Create:        createSvcFunc,
		Read:          readSvcFunc,
		Update:        updateSvcFunc,
		Delete:        deleteSvcFunc,
		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},
			"ip": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"service_type": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},
			"port": &schema.Schema{
				Type:     schema.TypeInt,
				Required: true,
			},
			"lb": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
		},
	}
}

func createSvcFunc(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*NetScalerNitroClient)
	lbName := d.Get("lb").(string)
	lbFound := client.FindResource("lbvserver", lbName)
	if !lbFound {
		log.Printf("No lb with name %s found", lbName)
		return errors.New(fmt.Sprintf("No lb with name %s found", lbName))
	}
	var svcName string
	if v, ok := d.GetOk("name"); ok {
		svcName = v.(string)
	} else {
		svcName = resource.PrefixedUniqueId("tf-svc-" + lbName + "-")
		d.Set("name", svcName)
	}
	log.Printf("****Creating service %s", svcName)
	svc := Service{
		Name:        svcName,
		Ip:          d.Get("ip").(string),
		Port:        d.Get("port").(int),
		ServiceType: d.Get("service_type").(string),
	}

	err := client.CreateService(lbName, &svc)
	if err != nil {
		return err
	}

	d.SetId(svc.Id())

	return nil
}

func readSvcFunc(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*NetScalerNitroClient)
	svcName := d.Id()
	log.Printf("****Reading service state %s", svcName)
	found := client.FindResource("service", svcName)
	if !found {
		log.Printf("Clearing service state %s", svcName)
		d.SetId("")
	}
	return nil
}

func updateSvcFunc(d *schema.ResourceData, meta interface{}) error {
	svcName := d.Id()
	log.Printf("****Updating service state %s", svcName)
	d.Set("name", svcName) //FIXME: why?
	return nil
}

func deleteSvcFunc(d *schema.ResourceData, meta interface{}) error {
	client := meta.(*NetScalerNitroClient)
	svcName := d.Id()
	err := client.DeleteService(svcName)
	if err != nil {
		return err
	}

	d.SetId("")

	return nil
}
<|endoftext|>


<commit_before>package nn

import (
	"fmt"
	"github.com/jcla1/matrix"
	"math"
)

var (
	_ = fmt.Println
)

type TrainingExample struct {
	Input, ExpectedOutput *matrix.Matrix
}

type Parameters []*matrix.Matrix
type Deltas []*matrix.Matrix

func CostFunction(data []TrainingExample, thetas Parameters, lambda float64) float64 {
	cost := float64(0)
	var estimation []float64
	var expected_output []float64

	// Cost

	for _, datum := range data {
		estimation = Hypothesis(thetas, datum).Values()
		expected_output = datum.ExpectedOutput.Values()

		for k, y := range expected_output {
			// heart of the cost function
			cost += y*math.Log(estimation[k]) + (1-y)*math.Log(1-estimation[k])
		}
	}

	// Regularization
	regularizationCost := float64(0)

	for _, theta := range thetas {
		for i, param := range theta.Values() {

			// ignore theta0
			if i%theta.Columns() == 0 {
				continue
			}

			regularizationCost += param * param
		}
	}

	return cost/float64(len(data)) + (lambda/(2*float64(len(data))))*regularizationCost
}

func Hypothesis(thetas Parameters, trainingEx TrainingExample) *matrix.Matrix {
	// Describes the current working values (a_1, a_2, ...)
	curValues := trainingEx.Input

	// Is simply a 1 in a 1x1 matrix to b
	// inserted into a vector as the bias unit
	biasValueMatrix := matrix.Ones(1, 1)

	for _, theta := range thetas {
		// Insert the bias unit, multiply with theta and apply the sigmoid function
		curValues = theta.Mul(curValues.InsertRows(biasValueMatrix, 0)).Apply(sigmoidMatrix)
	}

	return curValues
}

func HypothesisHistory(thetas Parameters, trainingEx TrainingExample) []*matrix.Matrix {
	// Describes the current working values (a_1, a_2, ...)
	curValues := trainingEx.Input

	// Is simply a 1 in a 1x1 matrix to b
	// inserted into a vector as the bias unit
	biasValueMatrix := matrix.Ones(1, 1)

	history := make([]*matrix.Matrix, 0, len(thetas)+1)
	history = append(history, curValues.InsertRows(biasValueMatrix, 0))

	for i, theta := range thetas {
		// Insert the bias unit, multiply with theta and apply the sigmoid function
		curValues = theta.Mul(history[len(history)-1]).Apply(sigmoidMatrix)

		if i != len(thetas)-1 {
			history = append(history, curValues.InsertRows(biasValueMatrix, 0))
		} else {
			history = append(history, curValues)
		}

	}

	return history
}

func DeltaTerms(thetas Parameters, trainingEx TrainingExample) Deltas {
	deltas := make(Deltas, len(thetas))

	biasValueMatrix := matrix.Ones(1, 1)

	deltas[len(deltas)-1], _ = Hypothesis(thetas, trainingEx).Sub(trainingEx.ExpectedOutput)

	for i := len(deltas) - 2; i >= 0; i-- {
		workingTheta := thetas[i+1]

		levelPrediction := Hypothesis(thetas[:i+1], trainingEx).InsertRows(biasValueMatrix, 0)
		tmp, _ := matrix.Ones(levelPrediction.Rows(), 1).Sub(levelPrediction)
		levelGradient := levelPrediction.Dot(tmp)

		deltas[i] = workingTheta.Transpose().Mul(deltas[i+1]).Dot(levelGradient).RemoveRow(1)
	}

	return deltas
}

func BackProp(thetas Parameters, trainingSet []TrainingExample, lambda float64) []*matrix.Matrix {
	// Make new BigDelta matrix
	bigDeltas := make(Deltas, len(thetas))

	for i, _ := range bigDeltas {
		bigDeltas[i] = matrix.Zeros(thetas[i].Rows(), thetas[i].Columns())
	}

	// Make new gradient matrix
	gradients := make([]*matrix.Matrix, len(thetas))

	var activations []*matrix.Matrix
	var deltaTerms Deltas

	for _, trainingEx := range trainingSet {
		activations = HypothesisHistory(thetas, trainingEx)
		deltaTerms = DeltaTerms(thetas, trainingEx)

		for i := 0; i < len(deltaTerms); i++ {
			bigDeltas[i], _ = bigDeltas[i].Add(deltaTerms[i].Mul(activations[i].Transpose()))
		}
	}

	for i, _ := range gradients {
		gradients[i] = (bigDeltas[i].Add(thetas[i].Scale(lambda))).Scale(1 / float64(len(trainingSet)))
	}

	return gradients
}

// Helper functions

func sigmoidMatrix(index int, value float64) float64 {
	return sigmoid(value)
}

func sigmoid(z float64) float64 {
	return 1 / (1 + math.Pow(math.E, -z))
}
<commit_msg>didn't need to remove and then add a nulled out column in thetas. Thought of this trick with using the identity matrix.<commit_after>package nn

import (
	"fmt"
	"github.com/jcla1/matrix"
	"math"
)

var (
	_ = fmt.Println
)

type TrainingExample struct {
	Input, ExpectedOutput *matrix.Matrix
}

type Parameters []*matrix.Matrix
type Deltas []*matrix.Matrix

func CostFunction(data []TrainingExample, thetas Parameters, lambda float64) float64 {
	cost := float64(0)
	var estimation []float64
	var expected_output []float64

	// Cost

	for _, datum := range data {
		estimation = Hypothesis(thetas, datum).Values()
		expected_output = datum.ExpectedOutput.Values()

		for k, y := range expected_output {
			// heart of the cost function
			cost += y*math.Log(estimation[k]) + (1-y)*math.Log(1-estimation[k])
		}
	}

	// Regularization
	regularizationCost := float64(0)

	for _, theta := range thetas {
		for i, param := range theta.Values() {

			// ignore theta0
			if i%theta.Columns() == 0 {
				continue
			}

			regularizationCost += param * param
		}
	}

	return cost/float64(len(data)) + (lambda/(2*float64(len(data))))*regularizationCost
}

func Hypothesis(thetas Parameters, trainingEx TrainingExample) *matrix.Matrix {
	// Describes the current working values (a_1, a_2, ...)
	curValues := trainingEx.Input

	// Is simply a 1 in a 1x1 matrix to b
	// inserted into a vector as the bias unit
	biasValueMatrix := matrix.Ones(1, 1)

	for _, theta := range thetas {
		// Insert the bias unit, multiply with theta and apply the sigmoid function
		curValues = theta.Mul(curValues.InsertRows(biasValueMatrix, 0)).Apply(sigmoidMatrix)
	}

	return curValues
}

func HypothesisHistory(thetas Parameters, trainingEx TrainingExample) []*matrix.Matrix {
	// Describes the current working values (a_1, a_2, ...)
	curValues := trainingEx.Input

	// Is simply a 1 in a 1x1 matrix to b
	// inserted into a vector as the bias unit
	biasValueMatrix := matrix.Ones(1, 1)

	history := make([]*matrix.Matrix, 0, len(thetas)+1)
	history = append(history, curValues.InsertRows(biasValueMatrix, 0))

	for i, theta := range thetas {
		// Insert the bias unit, multiply with theta and apply the sigmoid function
		curValues = theta.Mul(history[len(history)-1]).Apply(sigmoidMatrix)

		if i != len(thetas)-1 {
			history = append(history, curValues.InsertRows(biasValueMatrix, 0))
		} else {
			history = append(history, curValues)
		}

	}

	return history
}

func DeltaTerms(thetas Parameters, trainingEx TrainingExample) Deltas {
	deltas := make(Deltas, len(thetas))

	biasValueMatrix := matrix.Ones(1, 1)

	deltas[len(deltas)-1], _ = Hypothesis(thetas, trainingEx).Sub(trainingEx.ExpectedOutput)

	for i := len(deltas) - 2; i >= 0; i-- {
		workingTheta := thetas[i+1]

		levelPrediction := Hypothesis(thetas[:i+1], trainingEx).InsertRows(biasValueMatrix, 0)
		tmp, _ := matrix.Ones(levelPrediction.Rows(), 1).Sub(levelPrediction)
		levelGradient := levelPrediction.Dot(tmp)

		deltas[i] = workingTheta.Transpose().Mul(deltas[i+1]).Dot(levelGradient).RemoveRow(1)
	}

	return deltas
}

func BackProp(thetas Parameters, trainingSet []TrainingExample, lambda float64) []*matrix.Matrix {
	// Make new BigDelta matrix
	bigDeltas := make(Deltas, len(thetas))

	for i, _ := range bigDeltas {
		bigDeltas[i] = matrix.Zeros(thetas[i].Rows(), thetas[i].Columns())
	}

	// Make new gradient matrix
	gradients := make([]*matrix.Matrix, len(thetas))

	var activations []*matrix.Matrix
	var deltaTerms Deltas

	for _, trainingEx := range trainingSet {
		activations = HypothesisHistory(thetas, trainingEx)
		deltaTerms = DeltaTerms(thetas, trainingEx)

		for i := 0; i < len(deltaTerms); i++ {
			bigDeltas[i], _ = bigDeltas[i].Add(deltaTerms[i].Mul(activations[i].Transpose()))
		}
	}

	for i, _ := range gradients {
		gradients[i] = (bigDeltas[i].Add((thetas[i].Mul(matrix.Eye(thetas[i].Cols()).Set(1, 1, 0))).Scale(lambda))).Scale(1 / float64(len(trainingSet)))
	}

	return gradients
}

// Helper functions

func sigmoidMatrix(index int, value float64) float64 {
	return sigmoid(value)
}

func sigmoid(z float64) float64 {
	return 1 / (1 + math.Pow(math.E, -z))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"testing"
)

var h host

var filetests = []string{
	"http://localhost:6060/",
	"localhost",
}

func init() {
	h = host{name: "localhost"}
	s := make([]string, 0)
	h.addrs = append(s, "127.0.0.1")
}

func TestReadHosts(t *testing.T) {
	readHosts()
	for i, f := range filetests {
		if hosts[i].name != f {
			t.Errorf("hosts[%d]: got host{name: %s}, want host{name: %s}", i, hosts[i].name, f)
		}
	}
}

func TestPing(t *testing.T) {
	// TODO: Improve this when ping() has output.
	want := resp{
		seq:  0,
		code: 0,
		to:   "127.0.0.1",
		from: "127.0.0.1",
		body: fmt.Sprintf("Ranger-Chooch-%s", h.name),
	}
	h.ping()
	if h.resps[0].seq != want.seq {
		t.Errorf("h.resps[0].seq got %d, want %d", h.resps[0].seq, want.seq)
	}
	if h.resps[0].code != want.code {
		t.Errorf("h.resps[0].code got %d, want %d", h.resps[0].code, want.code)
	}
	if h.resps[0].to != want.to {
		t.Errorf("h.resps[0].to got %s, want %s", h.resps[0].to, want.to)
	}
	if h.resps[0].from != want.from {
		t.Errorf("h.resps[0].from got %s, want %s", h.resps[0].from, want.from)
	}
	if h.resps[0].body != want.body {
		t.Errorf("h.resps[0].body got %s, want %s", h.resps[0].body, want.body)
	}
}

func TestHtoI(t *testing.T) {
	h := host{name: "localhost"}
	want := "127.0.0.1"
	err := h.htoi()
	if err != nil {
		t.Fatal(err)
	}
	got := h.addrs[0]
	if got != want {
		t.Errorf("h.addrs: got %s, want %s", got, want)
	}
}

func TestAddResp(t *testing.T) {

}
<commit_msg>Remove useless test TestAddResp()<commit_after>package main

import (
	"fmt"
	"testing"
)

var h host

var filetests = []string{
	"http://localhost:6060/",
	"localhost",
}

func init() {
	h = host{name: "localhost"}
	s := make([]string, 0)
	h.addrs = append(s, "127.0.0.1")
}

func TestReadHosts(t *testing.T) {
	readHosts()
	for i, f := range filetests {
		if hosts[i].name != f {
			t.Errorf("hosts[%d]: got host{name: %s}, want host{name: %s}", i, hosts[i].name, f)
		}
	}
}

func TestPing(t *testing.T) {
	// TODO: Improve this when ping() has output.
	want := resp{
		seq:  0,
		code: 0,
		to:   "127.0.0.1",
		from: "127.0.0.1",
		body: fmt.Sprintf("Ranger-Chooch-%s", h.name),
	}
	h.ping()
	if h.resps[0].seq != want.seq {
		t.Errorf("h.resps[0].seq got %d, want %d", h.resps[0].seq, want.seq)
	}
	if h.resps[0].code != want.code {
		t.Errorf("h.resps[0].code got %d, want %d", h.resps[0].code, want.code)
	}
	if h.resps[0].to != want.to {
		t.Errorf("h.resps[0].to got %s, want %s", h.resps[0].to, want.to)
	}
	if h.resps[0].from != want.from {
		t.Errorf("h.resps[0].from got %s, want %s", h.resps[0].from, want.from)
	}
	if h.resps[0].body != want.body {
		t.Errorf("h.resps[0].body got %s, want %s", h.resps[0].body, want.body)
	}
}

func TestHtoI(t *testing.T) {
	h := host{name: "localhost"}
	want := "127.0.0.1"
	err := h.htoi()
	if err != nil {
		t.Fatal(err)
	}
	got := h.addrs[0]
	if got != want {
		t.Errorf("h.addrs: got %s, want %s", got, want)
	}
}
<|endoftext|>


<commit_before>package tests

import (
	"github.com/bozaro/tech-db-forum/generated/client"
	"github.com/bozaro/tech-db-forum/generated/models"
	"log"
	"math/rand"
	"net/url"
	"sync"
	"sync/atomic"
)

func FillUsers(c *client.Forum, parallel int, count int) []*models.User {
	results := make(chan *models.User, 64)
	var need int32 = int32(count)

	// spawn four worker goroutines
	var wg sync.WaitGroup
	for i := 0; i < parallel; i++ {
		wg.Add(1)
		go func() {
			for atomic.AddInt32(&need, -1) >= 0 {
				results <- CreateUser(c, nil)
			}
			wg.Done()
		}()
	}

	// get result
	result := make([]*models.User, count)
	for i := 0; i < count; i++ {
		result[i] = <-results
	}
	close(results)

	// wait for the workers to finish
	wg.Wait()
	return result
}

func Fill(url *url.URL) int {

	transport := CreateTransport(url)
	c := client.New(transport, nil)
	_, err := c.Operations.Clear(nil)
	CheckNil(err)

	log.Println("Creating users")
	/*users := []*models.User{}
	for i := 0; i < 1000; i++ {
		users = append(users, CreateUser(c, nil))
	}*/
	users := FillUsers(c, 8, 1000)

	log.Println("Creating forums")
	forums := []*models.Forum{}
	for i := 0; i < 20; i++ {
		forums = append(forums, CreateForum(c, nil, users[rand.Intn(len(users))]))
	}

	log.Println("Creating threads")
	threads := []*models.Thread{}
	for i := 0; i < 1000; i++ {
		thread := RandomThread()
		if rand.Intn(100) >= 5 {
			thread.Slug = ""
		}
		threads = append(threads, CreateThread(c, thread, forums[rand.Intn(len(forums))], users[rand.Intn(len(users))]))
	}

	log.Println("Creating posts")
	posts := []*models.Post{}
	for i := 0; i < 10000; i++ {
		post := RandomPost()
		post.Author = users[rand.Intn(len(users))].Nickname
		post.Thread = threads[rand.Intn(len(users))].ID
		posts = append(posts, CreatePost(c, post, nil))
	}

	log.Println("Done")
	return 0
}
<commit_msg>Добавил создание пользоватетей по очереди/в несколько потоков для сравнения<commit_after>package tests

import (
	"github.com/bozaro/tech-db-forum/generated/client"
	"github.com/bozaro/tech-db-forum/generated/models"
	"log"
	"math/rand"
	"net/url"
	"sync"
	"sync/atomic"
)

func FillUsers(c *client.Forum, parallel int, count int) []*models.User {
	results := make(chan *models.User, 64)
	var need int32 = int32(count)

	// spawn four worker goroutines
	var wg sync.WaitGroup
	for i := 0; i < parallel; i++ {
		wg.Add(1)
		go func() {
			for atomic.AddInt32(&need, -1) >= 0 {
				results <- CreateUser(c, nil)
			}
			wg.Done()
		}()
	}

	// get result
	result := make([]*models.User, count)
	for i := 0; i < count; i++ {
		result[i] = <-results
	}
	close(results)

	// wait for the workers to finish
	wg.Wait()
	return result
}

func Fill(url *url.URL) int {

	transport := CreateTransport(url)
	c := client.New(transport, nil)
	_, err := c.Operations.Clear(nil)
	CheckNil(err)

	log.Println("Creating users (single thread)")
	users := []*models.User{}
	for i := 0; i < 10000; i++ {
		users = append(users, CreateUser(c, nil))
	}
	log.Println("Creating users (multiple threads)")
	users = FillUsers(c, 8, 10000)

	log.Println("Creating forums")
	forums := []*models.Forum{}
	for i := 0; i < 20; i++ {
		forums = append(forums, CreateForum(c, nil, users[rand.Intn(len(users))]))
	}

	log.Println("Creating threads")
	threads := []*models.Thread{}
	for i := 0; i < 1000; i++ {
		thread := RandomThread()
		if rand.Intn(100) >= 5 {
			thread.Slug = ""
		}
		threads = append(threads, CreateThread(c, thread, forums[rand.Intn(len(forums))], users[rand.Intn(len(users))]))
	}

	log.Println("Creating posts")
	posts := []*models.Post{}
	for i := 0; i < 10000; i++ {
		post := RandomPost()
		post.Author = users[rand.Intn(len(users))].Nickname
		post.Thread = threads[rand.Intn(len(threads))].ID
		posts = append(posts, CreatePost(c, post, nil))
	}

	log.Println("Done")
	return 0
}
<|endoftext|>


<commit_before>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package proxy

import (
	"net/http"
	"strings"

	"github.com/apex/log"
)

type tokenProxier struct {
	handler http.Handler
}

func (p *tokenProxier) ServeHTTP(res http.ResponseWriter, req *http.Request) {
	if authorization := req.Header.Get("authorization"); authorization != "" {
		if len(authorization) >= 7 && strings.ToLower(authorization[0:7]) == "bearer " {
			req.Header.Set("Grpc-Metadata-Token", authorization[7:])
		}
		if len(authorization) >= 4 && strings.ToLower(authorization[0:4]) == "key " {
			req.Header.Set("Grpc-Metadata-Key", authorization[4:])
		}
	}
	p.handler.ServeHTTP(res, req)
}

// WithToken wraps the handler so that each request gets the Bearer token attached
func WithToken(handler http.Handler) http.Handler {
	return &tokenProxier{handler}
}

type logProxier struct {
	ctx     log.Interface
	handler http.Handler
}

func (p *logProxier) ServeHTTP(res http.ResponseWriter, req *http.Request) {
	p.ctx.WithFields(log.Fields{
		"RemoteAddress": req.RemoteAddr,
		"Method":        req.Method,
		"URI":           req.RequestURI,
	}).Info("Proxy HTTP request")
	p.handler.ServeHTTP(res, req)
}

// WithLogger wraps the handler so that each request gets logged
func WithLogger(handler http.Handler, ctx log.Interface) http.Handler {
	return &logProxier{ctx, handler}
}
<commit_msg>core/proxy: WithPagination<commit_after>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package proxy

import (
	"net/http"
	"strconv"
	"strings"

	"github.com/apex/log"
)

type tokenProxier struct {
	handler http.Handler
}

func (p *tokenProxier) ServeHTTP(res http.ResponseWriter, req *http.Request) {
	if authorization := req.Header.Get("authorization"); authorization != "" {
		if len(authorization) >= 7 && strings.ToLower(authorization[0:7]) == "bearer " {
			req.Header.Set("Grpc-Metadata-Token", authorization[7:])
		}
		if len(authorization) >= 4 && strings.ToLower(authorization[0:4]) == "key " {
			req.Header.Set("Grpc-Metadata-Key", authorization[4:])
		}
	}
	p.handler.ServeHTTP(res, req)
}

// WithToken wraps the handler so that each request gets the Bearer token attached
func WithToken(handler http.Handler) http.Handler {
	return &tokenProxier{handler}
}

type logProxier struct {
	ctx     log.Interface
	handler http.Handler
}

func (p *logProxier) ServeHTTP(res http.ResponseWriter, req *http.Request) {
	p.ctx.WithFields(log.Fields{
		"RemoteAddress": req.RemoteAddr,
		"Method":        req.Method,
		"URI":           req.RequestURI,
	}).Info("Proxy HTTP request")
	p.handler.ServeHTTP(res, req)
}

// WithLogger wraps the handler so that each request gets logged
func WithLogger(handler http.Handler, ctx log.Interface) http.Handler {
	return &logProxier{ctx, handler}
}

type paginatedHandler struct {
	handler http.Handler
}

func (h *paginatedHandler) ServeHTTP(res http.ResponseWriter, req *http.Request) {
	var b struct {
		pageQuery string
		page      int
		err       error
	}

	if b.pageQuery = req.URL.Query().Get("page"); b.pageQuery != "" {
		if b.page, b.err = strconv.Atoi(b.pageQuery); b.err != nil {
			http.Error(res, b.err.Error(), http.StatusBadRequest)
			return
		}
	}
	req.Header.Set("Grpc-Metadata-Page", strconv.Itoa(b.page))
	h.handler.ServeHTTP(res, req)
}

// WithPagination wraps the handler so that each request gets the Page value attached
func WithPagination(h http.Handler) http.Handler {
	return &paginatedHandler{h}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"strings"

	"github.com/gorilla/websocket"
	"github.com/hacsoc/slacksoc/api"
)

const (
	TOKEN_VAR = "SLACKSOC_TOKEN"
	NO_TOKEN_ERROR = "You must have the SLACKSOC_TOKEN variable to run the" +
					 " slacksoc bot"
	VERSION = "0.0.6"
)

type Bot struct {
	Token string
	Channels map[string]string
	WebSocketURL string
}

func NewBot(token string) *Bot {
	return &Bot{Token: token}
}

func (bot *Bot) Call(method string, data url.Values) (*http.Response, error) {
	data.Set("token", bot.Token)
	return api.Call(method, data)
}

func (bot *Bot) Start() error {
	payload, err := httpToJSON(bot.Call("rtm.start", url.Values{}))
	if err != nil {
		return err
	}
	ok, present := payload["ok"].(bool)
	if !present || ok != true {
		return &SlacksocError{"could not connect to RTM API"}
	}
	bot.GetChannelInfo()
	websocketURL, _ := payload["url"].(string)
	bot.WebSocketURL = websocketURL
	return nil
}

func (bot *Bot) Loop() error {
	dialer := websocket.Dialer{}
	conn, _, err := dialer.Dial(bot.WebSocketURL, http.Header{})
	if err != nil {
		return err
	}
	for {
		messageType, bytes, err := conn.ReadMessage()
		if err != nil {
			// NextReader returns an error if the connection is closed
			conn.Close()
			return nil
		}
		if messageType == websocket.BinaryMessage {
			continue // ignore binary messages
		}
		var message map[string]interface{}
		if err = json.Unmarshal(bytes, &message); err != nil {
			continue
		}
		fmt.Println(message)
		if _, ok := message["type"]; !ok {
			continue
		}
		switch message["type"].(string) {
		case "message":
			bot.ReceiveMessage(conn, message)
		default:
			continue
		}
	}
}

func (bot *Bot) ReceiveMessage(conn *websocket.Conn, message map[string]interface{}) {
	subtype, _ := message["subtype"]
	hiddenSubtype, ok := message["hidden"]
	hidden := ok && hiddenSubtype.(bool)
	reply := bot.ConstructReply(message, subtype, hidden)
	if reply != nil {
		conn.WriteJSON(reply)
	}
}

func (bot *Bot) ConstructReply(message map[string]interface{}, subtype interface{}, hidden bool) interface{} {
	if subtype != nil {
		switch subtype.(string) {
		case "bot_message":
			// don't reply to other bots
			return nil
		case "channel_join":
			return bot.SetRealNameFields(message)
		default:
			return nil
		}
	} else {
		text := message["text"].(string)
		if strings.Contains(text, "hi slacksoc") {
			return Mention(message["user"].(string), message["channel"].(string), "hi ", "")
		} else if text == "slacksoc: pm me" {
			return bot.DirectMessage(message["user"].(string), "hi")
		} else if strings.Contains(text, "gentoo") || strings.Contains(text, "Gentoo") {
			go bot.React(message, "funroll-loops")
			return nil
		}
		return nil
	}
}

func (bot *Bot) React(message map[string]interface{}, reaction string) {
	channel := message["channel"].(string)
	timestamp := message["ts"].(string)
	parameters := url.Values{}
	parameters.Set("channel", channel)
	parameters.Set("timestamp", timestamp)
	parameters.Set("name", reaction)
	bot.Call("reactions.add", parameters)
}

func (bot *Bot) SetRealNameFields(message map[string]interface{}) interface{} {
	channel := message["channel"].(string)
	if channel != bot.Channels["general"] {
		return nil
	}
	userID := message["user"].(string)
	dmChan := make(chan string)
	userChan := make(chan interface{})
	go func() {
		dm, _ := bot.OpenDirectMessage(userID)
		dmChan <- dm
	}()
	go func() {
		resp, err := bot.Call("users.info", url.Values{"user": []string{userID}})
		payload, err := httpToJSON(resp, err)
		userChan <- payload
	}()
	payload := (<- userChan).(map[string]interface{})
	success := payload["ok"].(bool)
	if !success {
		fmt.Println(payload)
		return nil
	}
	user := payload["user"].(map[string]interface{})
	nick := user["name"].(string)
	text := "Please set your real name fields. https://hacsoc.slack.com/team/%s."
	text += " Then click \"Edit\"."
	text = fmt.Sprintf(text, nick)
	dm := <- dmChan
	return NewMessage(text, dm).ToMap()
}

func main() {
	token := os.Getenv(TOKEN_VAR)
	if token == "" {
		fmt.Println(NO_TOKEN_ERROR)
		os.Exit(1)
	}

	bot := NewBot(token)
	fmt.Println("Starting bot")
	if err := bot.Start(); err != nil {
		fmt.Println(err)
	}
	fmt.Println("Looping")
	if err := bot.Loop(); err != nil {
		fmt.Println(err)
	}
}
<commit_msg>entire bot now integrates with the slack library; bump version number<commit_after>package main

import (
	"fmt"
	"net/url"
	"os"

	"github.com/ajm188/slack"
)

const (
	TOKEN_VAR = "SLACKSOC_TOKEN"
	NO_TOKEN_ERROR = "You must have the SLACKSOC_TOKEN variable to run the" +
					 " slacksoc bot"
	VERSION = "0.1.0"
)

func setRealNameFields(bot *slack.Bot, event map[string]interface{}) (*slack.Message, slack.Status) {
	channel := event["channel"].(string)
	if channel != bot.Channels["general"] {
		return nil, slack.CONTINUE
	}
	userID := event["user"].(string)
	dmChan := make(chan string)
	userChan := make(chan interface{})
	go func() {
		dm, _ := bot.OpenDirectMessage(userID)
		dmChan <- dm
	}()
	go func() {
		payload, _ := bot.Call("users.info", url.Values{"user": []string{userID}})
		userChan <- payload
	}()
	payload := (<- userChan).(map[string]interface{})
	success := payload["ok"].(bool)
	if !success {
		fmt.Println(payload)
		return nil, slack.CONTINUE
	}
	user := payload["user"].(map[string]interface{})
	nick := user["name"].(string)
	text := "Please set your real name fields. https://hacsoc.slack.com/team/%s."
	text += " Then click \"Edit\"."
	text = fmt.Sprintf(text, nick)
	dm := <- dmChan
	return slack.NewMessage(text, dm), slack.CONTINUE
}

func sendDM(bot *slack.Bot, event map[string]interface{}) (*slack.Message, slack.Status) {
	user := event["user"].(string)
	return bot.DirectMessage(user, "hi"), slack.CONTINUE
}

func main() {
	token := os.Getenv(TOKEN_VAR)
	if token == "" {
		fmt.Println(NO_TOKEN_ERROR)
		os.Exit(1)
	}

	bot := slack.NewBot(token)
	bot.Respond("hi", slack.Respond("hi there!"))
	bot.Respond("pm me", sendDM)
	bot.Listen("gentoo", slack.React("funroll-loops"))
	bot.OnEventWithSubtype("message", "channel_join", setRealNameFields)
	fmt.Println("Starting bot")
	if err := bot.Start(); err != nil {
		fmt.Println(err)
	}
}
<|endoftext|>