<commit_before>package client

import (
	"context"
	"crypto/hmac"
	"crypto/sha512"
	"encoding/hex"
	"encoding/json"
	"errors"
	"net"

	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/event"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/mux/msg"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/utils"
	"github.com/gobwas/ws"
	"github.com/gobwas/ws/wsutil"
)

type Client struct {
	id        int
	conn      net.Conn
	nonceGen  *utils.EpochNonceGenerator
	subsLimit int
	subs      map[event.Subscribe]bool
}

// New returns pointer to Client instance
func New() *Client {
	return &Client{
		subs:     make(map[event.Subscribe]bool),
		nonceGen: utils.NewEpochNonceGenerator(),
	}
}

// WithID assigns clinet ID
func (c *Client) WithID(ID int) *Client {
	c.id = ID
	return c
}

// WithSubsLimit sets limit of subscriptions on the instance
func (c *Client) WithSubsLimit(limit int) *Client {
	c.subsLimit = limit
	return c
}

// Public creates and returns client to interact with public channels
func (c *Client) Public(url string) (*Client, error) {
	conn, _, _, err := ws.DefaultDialer.Dial(context.Background(), url)
	if err != nil {
		return nil, err
	}
	c.conn = conn
	return c, nil
}

// Private creates and returns client to interact with private channels
func (c *Client) Private(key, sec, url string, dms int) (*Client, error) {
	nonce := c.nonceGen.GetNonce()
	conn, _, _, err := ws.DefaultDialer.Dial(context.Background(), url)
	if err != nil {
		return nil, err
	}

	c.conn = conn
	payload := "AUTH" + nonce
	sig := hmac.New(sha512.New384, []byte(sec))
	if _, err := sig.Write([]byte(payload)); err != nil {
		return nil, err
	}

	pldSign := hex.EncodeToString(sig.Sum(nil))
	sub := event.Subscribe{
		Event:       "auth",
		APIKEY:      key,
		AuthSig:     pldSign,
		AuthPayload: payload,
		AuthNonce:   nonce,
		DMS:         dms,
	}

	if err := c.Subscribe(sub); err != nil {
		return nil, err
	}
	return c, nil
}

// Subscribe takes subscription payload as per docs and subscribes client to it.
// We keep track of subscriptions so that when client failes, we can resubscribe.
func (c *Client) Subscribe(sub event.Subscribe) error {
	if err := c.Send(sub); err != nil {
		return err
	}

	c.AddSub(sub)
	return nil
}

// Send takes payload in form of interface and sends it to api
func (c *Client) Send(pld interface{}) error {
	b, err := json.Marshal(pld)
	if err != nil {
		return err
	}

	return wsutil.WriteClientBinary(c.conn, b)
}

// Read starts consuming data stream
func (c *Client) Read(ch chan<- msg.Msg) {
	defer c.conn.Close()

	for {
		ms, opCode, err := wsutil.ReadServerData(c.conn)
		m := msg.Msg{Data: ms, CID: c.id}

		if err != nil {
			m.Err = err
			ch <- m
			return
		}

		if opCode == ws.OpClose {
			m.Err = errors.New("client has closed unexpectedly")
			ch <- m
			return
		}

		ch <- m
	}
}

// SubsLimitReached returns true if number of subs > subsLimit
func (c *Client) SubsLimitReached() bool {
	if c.subsLimit == 0 {
		return false
	}
	return len(c.subs) == c.subsLimit
}

// SubAdded checks if given subscription is already added. Used to
// avoid duplicate subscriptions per client
func (c *Client) SubAdded(sub event.Subscribe) (isAdded bool) {
	_, isAdded = c.subs[sub]
	return
}

// AddSub adds new subscription to the list
func (c *Client) AddSub(sub event.Subscribe) {
	c.subs[sub] = true
}

// RemoveSub adds new subscription to the list
func (c *Client) RemoveSub(sub event.Subscribe) {
	delete(c.subs, sub)
}

// GetAllSubs returns all subscriptions
func (c *Client) GetAllSubs() (res []event.Subscribe) {
	for sub := range c.subs {
		res = append(res, sub)
	}
	return
}
<commit_msg>extra ws client functionality to close connection<commit_after>package client

import (
	"context"
	"crypto/hmac"
	"crypto/sha512"
	"encoding/hex"
	"encoding/json"
	"errors"
	"net"

	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/event"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/mux/msg"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/utils"
	"github.com/gobwas/ws"
	"github.com/gobwas/ws/wsutil"
)

type Client struct {
	id        int
	conn      net.Conn
	nonceGen  *utils.EpochNonceGenerator
	subsLimit int
	subs      map[event.Subscribe]bool
}

// New returns pointer to Client instance
func New() *Client {
	return &Client{
		subs:     make(map[event.Subscribe]bool),
		nonceGen: utils.NewEpochNonceGenerator(),
	}
}

// WithID assigns clinet ID
func (c *Client) WithID(ID int) *Client {
	c.id = ID
	return c
}

// WithSubsLimit sets limit of subscriptions on the instance
func (c *Client) WithSubsLimit(limit int) *Client {
	c.subsLimit = limit
	return c
}

// Public creates and returns client to interact with public channels
func (c *Client) Public(url string) (*Client, error) {
	conn, _, _, err := ws.DefaultDialer.Dial(context.Background(), url)
	if err != nil {
		return nil, err
	}
	c.conn = conn
	return c, nil
}

// Private creates and returns client to interact with private channels
func (c *Client) Private(key, sec, url string, dms int) (*Client, error) {
	nonce := c.nonceGen.GetNonce()
	conn, _, _, err := ws.DefaultDialer.Dial(context.Background(), url)
	if err != nil {
		return nil, err
	}

	c.conn = conn
	payload := "AUTH" + nonce
	sig := hmac.New(sha512.New384, []byte(sec))
	if _, err := sig.Write([]byte(payload)); err != nil {
		return nil, err
	}

	pldSign := hex.EncodeToString(sig.Sum(nil))
	sub := event.Subscribe{
		Event:       "auth",
		APIKEY:      key,
		AuthSig:     pldSign,
		AuthPayload: payload,
		AuthNonce:   nonce,
		DMS:         dms,
	}

	if err := c.Subscribe(sub); err != nil {
		return nil, err
	}
	return c, nil
}

// Subscribe takes subscription payload as per docs and subscribes client to it.
// We keep track of subscriptions so that when client failes, we can resubscribe.
func (c *Client) Subscribe(sub event.Subscribe) error {
	if err := c.Send(sub); err != nil {
		return err
	}

	c.AddSub(sub)
	return nil
}

// Send takes payload in form of interface and sends it to api
func (c *Client) Send(pld interface{}) error {
	b, err := json.Marshal(pld)
	if err != nil {
		return err
	}

	return wsutil.WriteClientBinary(c.conn, b)
}

// Close closes the socket connection
func (c *Client) Close() error {
	return c.conn.Close()
}

// Read starts consuming data stream
func (c *Client) Read(ch chan<- msg.Msg) {
	defer c.conn.Close()

	for {
		ms, opCode, err := wsutil.ReadServerData(c.conn)
		m := msg.Msg{Data: ms, CID: c.id}

		if err != nil {
			m.Err = err
			ch <- m
			return
		}

		if opCode == ws.OpClose {
			m.Err = errors.New("client has closed unexpectedly")
			ch <- m
			return
		}

		ch <- m
	}
}

// SubsLimitReached returns true if number of subs > subsLimit
func (c *Client) SubsLimitReached() bool {
	if c.subsLimit == 0 {
		return false
	}
	return len(c.subs) == c.subsLimit
}

// SubAdded checks if given subscription is already added. Used to
// avoid duplicate subscriptions per client
func (c *Client) SubAdded(sub event.Subscribe) (isAdded bool) {
	_, isAdded = c.subs[sub]
	return
}

// AddSub adds new subscription to the list
func (c *Client) AddSub(sub event.Subscribe) {
	c.subs[sub] = true
}

// RemoveSub adds new subscription to the list
func (c *Client) RemoveSub(sub event.Subscribe) {
	delete(c.subs, sub)
}

// GetAllSubs returns all subscriptions
func (c *Client) GetAllSubs() (res []event.Subscribe) {
	for sub := range c.subs {
		res = append(res, sub)
	}
	return
}
<|endoftext|>


<commit_before>package gocaffe

/*

#cgo CXXFLAGS: -I/usr/local/include/ -I/usr/local/opt/openblas/include/ -I./deps/ -O3 -Wall
#cgo LDFLAGS: -lcaffe -lboost_system -lstdc++ -lopencv_core -lopencv_highgui -lopencv_imgproc -lboost_filesystem
#include <stdlib.h>
#include "caffe.h"

*/
import "C"

import "unsafe"

type CaffePredictor struct {
	predictor C.CaffePredictor
}

func NewCaffePredictor(model, trained string) *CaffePredictor {
	modelpath := C.CString(model)
	defer C.free(unsafe.Pointer(modelpath))

	trainedpath := C.CString(trained)
	defer C.free(unsafe.Pointer(trainedpath))

	return &CaffePredictor{
		C.NewCaffePredictor(modelpath, trainedpath),
	}
}

func doubleToFloats(in *C.double, size int) []float64 {
	defer C.free(unsafe.Pointer(in))
	out := (*[1 << 30]float64)(unsafe.Pointer(in))[:size:size]
	return out
}

func (p *CaffePredictor) NClass() int {
	return int(C.NClass(p.predictor))
}

func (p *CaffePredictor) Predict(imgfile string) []float64 {
	imgpath := C.CString(imgfile)
	defer C.free(unsafe.Pointer(imgpath))

	ret := C.Predict(p.predictor, imgpath)
	return doubleToFloats(ret, p.NClass())
}
<commit_msg>opencv<commit_after>package gocaffe

/*

#cgo CXXFLAGS: -I/usr/local/include/ -I/usr/local/opt/openblas/include/ -I./deps/ -O3 -Wall
#cgo LDFLAGS: -lcaffe -lboost_system -lstdc++ -lopencv_core -lopencv_highgui -lopencv_imgproc -lboost_filesystem -lopencv_imgcodecs
#include <stdlib.h>
#include "caffe.h"

*/
import "C"

import "unsafe"

type CaffePredictor struct {
	predictor C.CaffePredictor
}

func NewCaffePredictor(model, trained string) *CaffePredictor {
	modelpath := C.CString(model)
	defer C.free(unsafe.Pointer(modelpath))

	trainedpath := C.CString(trained)
	defer C.free(unsafe.Pointer(trainedpath))

	return &CaffePredictor{
		C.NewCaffePredictor(modelpath, trainedpath),
	}
}

func doubleToFloats(in *C.double, size int) []float64 {
	defer C.free(unsafe.Pointer(in))
	out := (*[1 << 30]float64)(unsafe.Pointer(in))[:size:size]
	return out
}

func (p *CaffePredictor) NClass() int {
	return int(C.NClass(p.predictor))
}

func (p *CaffePredictor) Predict(imgfile string) []float64 {
	imgpath := C.CString(imgfile)
	defer C.free(unsafe.Pointer(imgpath))

	ret := C.Predict(p.predictor, imgpath)
	return doubleToFloats(ret, p.NClass())
}
<|endoftext|>


<commit_before>// Package tailfile provides helper functions to read the nth lines of any
// ReadSeeker.
package tailfile // import "github.com/docker/docker/pkg/tailfile"

import (
	"bufio"
	"bytes"
	"context"
	"errors"
	"io"
	"os"
)

const blockSize = 1024

var eol = []byte("\n")

// ErrNonPositiveLinesNumber is an error returned if the lines number was negative.
var ErrNonPositiveLinesNumber = errors.New("The number of lines to extract from the file must be positive")

//TailFile returns last n lines of the passed in file.
func TailFile(f *os.File, n int) ([][]byte, error) {
	size, err := f.Seek(0, io.SeekEnd)
	if err != nil {
		return nil, err
	}

	rAt := io.NewSectionReader(f, 0, size)
	r, nLines, err := NewTailReader(context.Background(), rAt, n)
	if err != nil {
		return nil, err
	}

	buf := make([][]byte, 0, nLines)
	scanner := bufio.NewScanner(r)

	for scanner.Scan() {
		buf = append(buf, scanner.Bytes())
	}
	return buf, nil
}

// SizeReaderAt is an interface used to get a ReaderAt as well as the size of the underlying reader.
// Note that the size of the underlying reader should not change when using this interface.
type SizeReaderAt interface {
	io.ReaderAt
	Size() int64
}

// NewTailReader scopes the passed in reader to just the last N lines passed in
func NewTailReader(ctx context.Context, r SizeReaderAt, reqLines int) (io.Reader, int, error) {
	return NewTailReaderWithDelimiter(ctx, r, reqLines, eol)
}

// NewTailReaderWithDelimiter scopes the passed in reader to just the last N lines passed in
// In this case a "line" is defined by the passed in delimiter.
//
// Delimiter lengths should be generally small, no more than 12 bytes
func NewTailReaderWithDelimiter(ctx context.Context, r SizeReaderAt, reqLines int, delimiter []byte) (io.Reader, int, error) {
	if reqLines < 1 {
		return nil, 0, ErrNonPositiveLinesNumber
	}
	if len(delimiter) == 0 {
		return nil, 0, errors.New("must provide a delimiter")
	}
	var (
		size      = r.Size()
		tailStart int64
		tailEnd   = size
		found     int
	)

	if int64(len(delimiter)) >= size {
		return bytes.NewReader(nil), 0, nil
	}

	scanner := newScanner(r, delimiter)
	for scanner.Scan(ctx) {
		if err := scanner.Err(); err != nil {
			return nil, 0, scanner.Err()
		}

		found++
		if found == 1 {
			tailEnd = scanner.End()
		}
		if found == reqLines {
			break
		}
	}

	tailStart = scanner.Start(ctx)

	if found == 0 {
		return bytes.NewReader(nil), 0, nil
	}

	if found < reqLines && tailStart != 0 {
		tailStart = 0
	}
	return io.NewSectionReader(r, tailStart, tailEnd-tailStart), found, nil
}

func newScanner(r SizeReaderAt, delim []byte) *scanner {
	size := r.Size()
	readSize := blockSize
	if readSize > int(size) {
		readSize = int(size)
	}
	// silly case...
	if len(delim) >= readSize/2 {
		readSize = len(delim)*2 + 2
	}

	return &scanner{
		r:     r,
		pos:   size,
		buf:   make([]byte, readSize),
		delim: delim,
	}
}

type scanner struct {
	r     SizeReaderAt
	pos   int64
	buf   []byte
	delim []byte
	err   error
	idx   int
}

func (s *scanner) Start(ctx context.Context) int64 {
	if s.idx > 0 {
		idx := bytes.LastIndex(s.buf[:s.idx], s.delim)
		if idx >= 0 {
			return s.pos + int64(idx) + int64(len(s.delim))
		}
	}

	// slow path
	buf := make([]byte, len(s.buf))
	copy(buf, s.buf)

	readAhead := &scanner{
		r:     s.r,
		pos:   s.pos,
		delim: s.delim,
		idx:   s.idx,
		buf:   buf,
	}

	if !readAhead.Scan(ctx) {
		return 0
	}
	return readAhead.End()
}

func (s *scanner) End() int64 {
	return s.pos + int64(s.idx) + int64(len(s.delim))
}

func (s *scanner) Err() error {
	return s.err
}

func (s *scanner) Scan(ctx context.Context) bool {
	if s.err != nil {
		return false
	}

	for {
		select {
		case <-ctx.Done():
			s.err = ctx.Err()
			return false
		default:
		}

		idx := s.idx - len(s.delim)
		if idx < 0 {
			readSize := int(s.pos)
			if readSize > len(s.buf) {
				readSize = len(s.buf)
			}

			if readSize < len(s.delim) {
				return false
			}

			offset := s.pos - int64(readSize)
			n, err := s.r.ReadAt(s.buf[:readSize], offset)
			if err != nil && err != io.EOF {
				s.err = err
				return false
			}

			s.pos -= int64(n)
			idx = n
		}

		s.idx = bytes.LastIndex(s.buf[:idx], s.delim)
		if s.idx >= 0 {
			return true
		}

		if len(s.delim) > 1 && s.pos > 0 {
			// in this case, there may be a partial delimiter at the front of the buffer, so set the position forward
			// up to the maximum size partial that could be there so it can be read again in the next iteration with any
			// potential remainder.
			// An example where delimiter is `####`:
			// [##asdfqwerty]
			//    ^
			// This resets the position to where the arrow is pointing.
			// It could actually check if a partial exists and at the front, but that is pretty similar to the indexing
			// code above though a bit more complex since each byte has to be checked (`len(delimiter)-1`) factorial).
			// It's much simpler and cleaner to just re-read `len(delimiter)-1` bytes again.
			s.pos += int64(len(s.delim)) - 1
		}

	}
}
<commit_msg>pkg/tailfile: normalize comment formatting<commit_after>// Package tailfile provides helper functions to read the nth lines of any
// ReadSeeker.
package tailfile // import "github.com/docker/docker/pkg/tailfile"

import (
	"bufio"
	"bytes"
	"context"
	"errors"
	"io"
	"os"
)

const blockSize = 1024

var eol = []byte("\n")

// ErrNonPositiveLinesNumber is an error returned if the lines number was negative.
var ErrNonPositiveLinesNumber = errors.New("The number of lines to extract from the file must be positive")

// TailFile returns last n lines of the passed in file.
func TailFile(f *os.File, n int) ([][]byte, error) {
	size, err := f.Seek(0, io.SeekEnd)
	if err != nil {
		return nil, err
	}

	rAt := io.NewSectionReader(f, 0, size)
	r, nLines, err := NewTailReader(context.Background(), rAt, n)
	if err != nil {
		return nil, err
	}

	buf := make([][]byte, 0, nLines)
	scanner := bufio.NewScanner(r)

	for scanner.Scan() {
		buf = append(buf, scanner.Bytes())
	}
	return buf, nil
}

// SizeReaderAt is an interface used to get a ReaderAt as well as the size of the underlying reader.
// Note that the size of the underlying reader should not change when using this interface.
type SizeReaderAt interface {
	io.ReaderAt
	Size() int64
}

// NewTailReader scopes the passed in reader to just the last N lines passed in
func NewTailReader(ctx context.Context, r SizeReaderAt, reqLines int) (io.Reader, int, error) {
	return NewTailReaderWithDelimiter(ctx, r, reqLines, eol)
}

// NewTailReaderWithDelimiter scopes the passed in reader to just the last N lines passed in
// In this case a "line" is defined by the passed in delimiter.
//
// Delimiter lengths should be generally small, no more than 12 bytes
func NewTailReaderWithDelimiter(ctx context.Context, r SizeReaderAt, reqLines int, delimiter []byte) (io.Reader, int, error) {
	if reqLines < 1 {
		return nil, 0, ErrNonPositiveLinesNumber
	}
	if len(delimiter) == 0 {
		return nil, 0, errors.New("must provide a delimiter")
	}
	var (
		size      = r.Size()
		tailStart int64
		tailEnd   = size
		found     int
	)

	if int64(len(delimiter)) >= size {
		return bytes.NewReader(nil), 0, nil
	}

	scanner := newScanner(r, delimiter)
	for scanner.Scan(ctx) {
		if err := scanner.Err(); err != nil {
			return nil, 0, scanner.Err()
		}

		found++
		if found == 1 {
			tailEnd = scanner.End()
		}
		if found == reqLines {
			break
		}
	}

	tailStart = scanner.Start(ctx)

	if found == 0 {
		return bytes.NewReader(nil), 0, nil
	}

	if found < reqLines && tailStart != 0 {
		tailStart = 0
	}
	return io.NewSectionReader(r, tailStart, tailEnd-tailStart), found, nil
}

func newScanner(r SizeReaderAt, delim []byte) *scanner {
	size := r.Size()
	readSize := blockSize
	if readSize > int(size) {
		readSize = int(size)
	}
	// silly case...
	if len(delim) >= readSize/2 {
		readSize = len(delim)*2 + 2
	}

	return &scanner{
		r:     r,
		pos:   size,
		buf:   make([]byte, readSize),
		delim: delim,
	}
}

type scanner struct {
	r     SizeReaderAt
	pos   int64
	buf   []byte
	delim []byte
	err   error
	idx   int
}

func (s *scanner) Start(ctx context.Context) int64 {
	if s.idx > 0 {
		idx := bytes.LastIndex(s.buf[:s.idx], s.delim)
		if idx >= 0 {
			return s.pos + int64(idx) + int64(len(s.delim))
		}
	}

	// slow path
	buf := make([]byte, len(s.buf))
	copy(buf, s.buf)

	readAhead := &scanner{
		r:     s.r,
		pos:   s.pos,
		delim: s.delim,
		idx:   s.idx,
		buf:   buf,
	}

	if !readAhead.Scan(ctx) {
		return 0
	}
	return readAhead.End()
}

func (s *scanner) End() int64 {
	return s.pos + int64(s.idx) + int64(len(s.delim))
}

func (s *scanner) Err() error {
	return s.err
}

func (s *scanner) Scan(ctx context.Context) bool {
	if s.err != nil {
		return false
	}

	for {
		select {
		case <-ctx.Done():
			s.err = ctx.Err()
			return false
		default:
		}

		idx := s.idx - len(s.delim)
		if idx < 0 {
			readSize := int(s.pos)
			if readSize > len(s.buf) {
				readSize = len(s.buf)
			}

			if readSize < len(s.delim) {
				return false
			}

			offset := s.pos - int64(readSize)
			n, err := s.r.ReadAt(s.buf[:readSize], offset)
			if err != nil && err != io.EOF {
				s.err = err
				return false
			}

			s.pos -= int64(n)
			idx = n
		}

		s.idx = bytes.LastIndex(s.buf[:idx], s.delim)
		if s.idx >= 0 {
			return true
		}

		if len(s.delim) > 1 && s.pos > 0 {
			// in this case, there may be a partial delimiter at the front of the buffer, so set the position forward
			// up to the maximum size partial that could be there so it can be read again in the next iteration with any
			// potential remainder.
			// An example where delimiter is `####`:
			// [##asdfqwerty]
			//    ^
			// This resets the position to where the arrow is pointing.
			// It could actually check if a partial exists and at the front, but that is pretty similar to the indexing
			// code above though a bit more complex since each byte has to be checked (`len(delimiter)-1`) factorial).
			// It's much simpler and cleaner to just re-read `len(delimiter)-1` bytes again.
			s.pos += int64(len(s.delim)) - 1
		}

	}
}
<|endoftext|>


<commit_before>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

// Plan is a description of a sequence of steps to produce a correct
// result for a query.

package simple

import (
	"fmt"

	"github.com/couchbaselabs/tuqtng/ast"
	"github.com/couchbaselabs/tuqtng/catalog"
	"github.com/couchbaselabs/tuqtng/catalog/system"
	"github.com/couchbaselabs/tuqtng/plan"
	"github.com/couchbaselabs/tuqtng/query"
)

type SimplePlanner struct {
	site        catalog.Site
	defaultPool string
}

func NewSimplePlanner(site catalog.Site, defaultPool string) *SimplePlanner {
	return &SimplePlanner{
		site:        site,
		defaultPool: defaultPool,
	}
}

func (this *SimplePlanner) Plan(stmt ast.Statement) (plan.PlanChannel, query.ErrorChannel) {
	pc := make(plan.PlanChannel)
	ec := make(query.ErrorChannel)
	go this.buildPlans(stmt, pc, ec)
	return pc, ec
}

func (this *SimplePlanner) buildSelectStatementPlans(stmt *ast.SelectStatement, pc plan.PlanChannel, ec query.ErrorChannel) {

	var planHeads []plan.PlanElement

	from := stmt.GetFrom()
	if from == nil {
		// point to :system.dual
		from = &ast.From{Pool: system.POOL_NAME, Bucket: system.BUCKET_NAME_DUAL}
	}

	// get the pool
	poolName := from.Pool
	if poolName == "" {
		poolName = this.defaultPool
	}

	pool, err := this.site.PoolByName(poolName)
	if err != nil {
		ec <- query.NewPoolDoesNotExist(poolName)
		return
	}

	bucket, err := pool.BucketByName(from.Bucket)
	if err != nil {
		ec <- query.NewBucketDoesNotExist(from.Bucket)
		return
	}

	// find all docs index
	indexes, err := bucket.Indexes()
	if err != nil {
		ec <- query.NewError(err, fmt.Sprintf("No indexes found for bucket %v", from.Bucket))
		return
	}

	foundUsableIndex := false
	for _, index := range indexes {
		var lastStep plan.PlanElement
		switch index.(type) {
		case catalog.PrimaryIndex:
			lastStep = plan.NewScan(pool.Name(), bucket.Name(), index.Name())
			lastStep = plan.NewFetch(lastStep, pool.Name(), bucket.Name(), from.Projection, from.As)
			nextFrom := from.Over
			for nextFrom != nil {
				// add document joins
				lastStep = plan.NewDocumentJoin(lastStep, nextFrom.Projection, nextFrom.As)
				nextFrom = nextFrom.Over
			}
			planHeads = append(planHeads, lastStep)
			foundUsableIndex = true
		}

	}

	if !foundUsableIndex {
		ec <- query.NewError(nil, fmt.Sprintf("No usable indexes found for bucket %v", from.Bucket))
		return
	}

	// now for all the plan heads, create a full plan
	for _, lastStep := range planHeads {

		if stmt.GetWhere() != nil {
			lastStep = plan.NewFilter(lastStep, stmt.GetWhere())
		}

		if stmt.GetGroupBy() != nil {
			lastStep = plan.NewGroup(lastStep, stmt.GetGroupBy(), stmt.GetAggregateReferences())
		}

		if stmt.GetHaving() != nil {
			lastStep = plan.NewFilter(lastStep, stmt.GetHaving())
		}

		lastStep = plan.NewProjector(lastStep, stmt.GetResultExpressionList(), true)

		if stmt.IsDistinct() {
			lastStep = plan.NewEliminateDuplicates(lastStep)
		}

		if stmt.GetOrderBy() != nil {
			explicitAliases := stmt.GetExplicitProjectionAliases()
			lastStep = plan.NewOrder(lastStep, stmt.GetOrderBy(), explicitAliases)
		}

		if stmt.GetOffset() != 0 {
			lastStep = plan.NewOffset(lastStep, stmt.GetOffset())
		}

		if stmt.GetLimit() >= 0 {
			lastStep = plan.NewLimit(lastStep, stmt.GetLimit())
		}

		if stmt.ExplainOnly {
			lastStep = plan.NewExplain(lastStep)
		}

		pc <- plan.Plan{Root: lastStep}

	}

}

func (this *SimplePlanner) buildCreateIndexStatementPlans(stmt *ast.CreateIndexStatement, pc plan.PlanChannel, ec query.ErrorChannel) {

	poolName := stmt.Pool
	if poolName == "" {
		poolName = this.defaultPool
	}

	pool, err := this.site.PoolByName(poolName)
	if err != nil {
		ec <- query.NewPoolDoesNotExist(this.defaultPool)
		return
	}

	bucket, err := pool.BucketByName(stmt.Bucket)
	if err != nil {
		ec <- query.NewBucketDoesNotExist(stmt.Bucket)
		return
	}

	var lastStep plan.PlanElement

	lastStep = plan.NewCreateIndex(pool.Name(), bucket.Name(), stmt.Name, stmt.Method, stmt.On)
	if stmt.ExplainOnly {
		lastStep = plan.NewExplain(lastStep)
	}

	pc <- plan.Plan{Root: lastStep}
	return
}

func (this *SimplePlanner) buildDropIndexStatementPlans(stmt *ast.DropIndexStatement, pc plan.PlanChannel, ec query.ErrorChannel) {

	poolName := stmt.Pool
	if poolName == "" {
		poolName = this.defaultPool
	}

	pool, err := this.site.PoolByName(poolName)
	if err != nil {
		ec <- query.NewPoolDoesNotExist(this.defaultPool)
		return
	}

	bucket, err := pool.BucketByName(stmt.Bucket)
	if err != nil {
		ec <- query.NewBucketDoesNotExist(stmt.Bucket)
		return
	}

	var lastStep plan.PlanElement

	lastStep = plan.NewDropIndex(pool.Name(), bucket.Name(), stmt.Name)
	if stmt.ExplainOnly {
		lastStep = plan.NewExplain(lastStep)
	}

	pc <- plan.Plan{Root: lastStep}
	return
}

func (this *SimplePlanner) buildPlans(stmt ast.Statement, pc plan.PlanChannel, ec query.ErrorChannel) {
	defer close(pc)
	defer close(ec)
	switch stmt := stmt.(type) {
	case *ast.SelectStatement:
		this.buildSelectStatementPlans(stmt, pc, ec)
	case *ast.CreateIndexStatement:
		this.buildCreateIndexStatementPlans(stmt, pc, ec)
	case *ast.DropIndexStatement:
		this.buildDropIndexStatementPlans(stmt, pc, ec)
	}

}
<commit_msg>remove useless variable<commit_after>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

// Plan is a description of a sequence of steps to produce a correct
// result for a query.

package simple

import (
	"fmt"

	"github.com/couchbaselabs/tuqtng/ast"
	"github.com/couchbaselabs/tuqtng/catalog"
	"github.com/couchbaselabs/tuqtng/catalog/system"
	"github.com/couchbaselabs/tuqtng/plan"
	"github.com/couchbaselabs/tuqtng/query"
)

type SimplePlanner struct {
	site        catalog.Site
	defaultPool string
}

func NewSimplePlanner(site catalog.Site, defaultPool string) *SimplePlanner {
	return &SimplePlanner{
		site:        site,
		defaultPool: defaultPool,
	}
}

func (this *SimplePlanner) Plan(stmt ast.Statement) (plan.PlanChannel, query.ErrorChannel) {
	pc := make(plan.PlanChannel)
	ec := make(query.ErrorChannel)
	go this.buildPlans(stmt, pc, ec)
	return pc, ec
}

func (this *SimplePlanner) buildSelectStatementPlans(stmt *ast.SelectStatement, pc plan.PlanChannel, ec query.ErrorChannel) {

	var planHeads []plan.PlanElement

	from := stmt.GetFrom()
	if from == nil {
		// point to :system.dual
		from = &ast.From{Pool: system.POOL_NAME, Bucket: system.BUCKET_NAME_DUAL}
	}

	// get the pool
	poolName := from.Pool
	if poolName == "" {
		poolName = this.defaultPool
	}

	pool, err := this.site.PoolByName(poolName)
	if err != nil {
		ec <- query.NewPoolDoesNotExist(poolName)
		return
	}

	bucket, err := pool.BucketByName(from.Bucket)
	if err != nil {
		ec <- query.NewBucketDoesNotExist(from.Bucket)
		return
	}

	// find all docs index
	indexes, err := bucket.Indexes()
	if err != nil {
		ec <- query.NewError(err, fmt.Sprintf("No indexes found for bucket %v", from.Bucket))
		return
	}

	for _, index := range indexes {
		var lastStep plan.PlanElement
		switch index.(type) {
		case catalog.PrimaryIndex:
			lastStep = plan.NewScan(pool.Name(), bucket.Name(), index.Name())
			lastStep = plan.NewFetch(lastStep, pool.Name(), bucket.Name(), from.Projection, from.As)
			nextFrom := from.Over
			for nextFrom != nil {
				// add document joins
				lastStep = plan.NewDocumentJoin(lastStep, nextFrom.Projection, nextFrom.As)
				nextFrom = nextFrom.Over
			}
			planHeads = append(planHeads, lastStep)
		}

	}

	if len(planHeads) == 0 {
		ec <- query.NewError(nil, fmt.Sprintf("No usable indexes found for bucket %v", from.Bucket))
		return
	}

	// now for all the plan heads, create a full plan
	for _, lastStep := range planHeads {

		if stmt.GetWhere() != nil {
			lastStep = plan.NewFilter(lastStep, stmt.GetWhere())
		}

		if stmt.GetGroupBy() != nil {
			lastStep = plan.NewGroup(lastStep, stmt.GetGroupBy(), stmt.GetAggregateReferences())
		}

		if stmt.GetHaving() != nil {
			lastStep = plan.NewFilter(lastStep, stmt.GetHaving())
		}

		lastStep = plan.NewProjector(lastStep, stmt.GetResultExpressionList(), true)

		if stmt.IsDistinct() {
			lastStep = plan.NewEliminateDuplicates(lastStep)
		}

		if stmt.GetOrderBy() != nil {
			explicitAliases := stmt.GetExplicitProjectionAliases()
			lastStep = plan.NewOrder(lastStep, stmt.GetOrderBy(), explicitAliases)
		}

		if stmt.GetOffset() != 0 {
			lastStep = plan.NewOffset(lastStep, stmt.GetOffset())
		}

		if stmt.GetLimit() >= 0 {
			lastStep = plan.NewLimit(lastStep, stmt.GetLimit())
		}

		if stmt.ExplainOnly {
			lastStep = plan.NewExplain(lastStep)
		}

		pc <- plan.Plan{Root: lastStep}

	}

}

func (this *SimplePlanner) buildCreateIndexStatementPlans(stmt *ast.CreateIndexStatement, pc plan.PlanChannel, ec query.ErrorChannel) {

	poolName := stmt.Pool
	if poolName == "" {
		poolName = this.defaultPool
	}

	pool, err := this.site.PoolByName(poolName)
	if err != nil {
		ec <- query.NewPoolDoesNotExist(this.defaultPool)
		return
	}

	bucket, err := pool.BucketByName(stmt.Bucket)
	if err != nil {
		ec <- query.NewBucketDoesNotExist(stmt.Bucket)
		return
	}

	var lastStep plan.PlanElement

	lastStep = plan.NewCreateIndex(pool.Name(), bucket.Name(), stmt.Name, stmt.Method, stmt.On)
	if stmt.ExplainOnly {
		lastStep = plan.NewExplain(lastStep)
	}

	pc <- plan.Plan{Root: lastStep}
	return
}

func (this *SimplePlanner) buildDropIndexStatementPlans(stmt *ast.DropIndexStatement, pc plan.PlanChannel, ec query.ErrorChannel) {

	poolName := stmt.Pool
	if poolName == "" {
		poolName = this.defaultPool
	}

	pool, err := this.site.PoolByName(poolName)
	if err != nil {
		ec <- query.NewPoolDoesNotExist(this.defaultPool)
		return
	}

	bucket, err := pool.BucketByName(stmt.Bucket)
	if err != nil {
		ec <- query.NewBucketDoesNotExist(stmt.Bucket)
		return
	}

	var lastStep plan.PlanElement

	lastStep = plan.NewDropIndex(pool.Name(), bucket.Name(), stmt.Name)
	if stmt.ExplainOnly {
		lastStep = plan.NewExplain(lastStep)
	}

	pc <- plan.Plan{Root: lastStep}
	return
}

func (this *SimplePlanner) buildPlans(stmt ast.Statement, pc plan.PlanChannel, ec query.ErrorChannel) {
	defer close(pc)
	defer close(ec)
	switch stmt := stmt.(type) {
	case *ast.SelectStatement:
		this.buildSelectStatementPlans(stmt, pc, ec)
	case *ast.CreateIndexStatement:
		this.buildCreateIndexStatementPlans(stmt, pc, ec)
	case *ast.DropIndexStatement:
		this.buildDropIndexStatementPlans(stmt, pc, ec)
	}

}
<|endoftext|>


<commit_before>package rx

/*Observable is a representation of any set of values over any amount of time*/
type Observable interface {
	Subscribe(InObserver)
}
<commit_msg>fix git commit<commit_after>package rx

/*Observable is a representation of any set of values over any amount of time*/
type Observable interface {
	Subscribe(InObserver)
	Map(MapFunc) Observable
}

/*MapFunc used by Map*/
type MapFunc func(interface{}) interface{}
<|endoftext|>


<commit_before>// Copyright 2015 LinkedIn Corp. Licensed under the Apache License,
// Version 2.0 (the "License"); you may not use this file except in
// compliance with the License.
 You may obtain a copy of the License
// at http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
//
distributed under the License is distributed on an "AS IS" BASIS,
//
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache
// License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License.
 You may obtain a copy of
// the License at http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
//
distributed under the License is distributed on an "AS IS" BASIS,
//
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.

package goavro

import (
	"bufio"
	"bytes"
	"compress/flate"
	"fmt"
	"github.com/golang/snappy"
	"io"
	"io/ioutil"
)

// ErrReaderInit is returned when the encoder encounters an error.
type ErrReaderInit struct {
	Message string
	Err     error
}

func (e ErrReaderInit) Error() string {
	if e.Err == nil {
		return "cannot build " + e.Message
	}
	return "cannot build " + e.Message + ": " + e.Err.Error()
}

func newReaderInitError(a ...interface{}) *ErrReaderInit {
	var err error
	var format, message string
	var ok bool
	if len(a) == 0 {
		return &ErrReaderInit{"cannot create reader: no reason given", nil}
	}
	// if last item is error: save it
	if err, ok = a[len(a)-1].(error); ok {
		a = a[:len(a)-1] // pop it
	}
	// if items left, first ought to be format string
	if len(a) > 0 {
		if format, ok = a[0].(string); ok {
			a = a[1:] // unshift
			message = fmt.Sprintf(format, a...)
		}
	}
	return &ErrReaderInit{message, err}
}

// ErrReaderBlockCount is returned when a reader detects an error
// while attempting to read the block count and block size.
type ErrReaderBlockCount struct {
	Err error
}

func (e *ErrReaderBlockCount) Error() string {
	return "cannot read block count and size: " + e.Err.Error()
}

// ReaderSetter functions are those those which are used to instantiate
// a new Reader.
type ReaderSetter func(*Reader) error

// BufferFromReader wraps the specified `io.Reader` using a
// `bufio.Reader` to read from a file.
func BufferFromReader(r io.Reader) ReaderSetter {
	return func(fr *Reader) error {
		fr.r = bufio.NewReader(r)
		return nil
	}
}

// FromReader specifies the `io.Reader` to use when reading a file.
func FromReader(r io.Reader) ReaderSetter {
	return func(fr *Reader) error {
		fr.r = r
		return nil
	}
}

// Reader structure contains data necessary to read Avro files.
type Reader struct {
	CompressionCodec string
	DataSchema       string
	Sync             []byte
	dataCodec        Codec
	datum            Datum
	deblocked        chan Datum
	done             bool
	err              error
	r                io.Reader
}

// NewReader returns a object to read data from an io.Reader using the
// Avro Object Container Files format.
//
//     func main() {
//         conn, err := net.Dial("tcp", "127.0.0.1:8080")
//         if err != nil {
//             log.Fatal(err)
//         }
//         fr, err := goavro.NewReader(goavro.FromReader(conn))
//         if err != nil {
//             log.Fatal("cannot create Reader: ", err)
//         }
//         defer func() {
//             if err := fr.Close(); err != nil {
//                 log.Fatal(err)
//             }
//         }()
//
//         for fr.Scan() {
//             datum, err := fr.Read()
//             if err != nil {
//                 log.Println("cannot read datum: ", err)
//                 continue
//             }
//             fmt.Println("RECORD: ", datum)
//         }
//     }
func NewReader(setters ...ReaderSetter) (*Reader, error) {
	var err error
	fr := &Reader{}
	for _, setter := range setters {
		err = setter(fr)
		if err != nil {
			return nil, newReaderInitError(err)
		}
	}
	if fr.r == nil {
		return nil, newReaderInitError("must specify io.Reader")
	}
	// read in header information and use it to initialize Reader
	magic := make([]byte, 4)
	_, err = fr.r.Read(magic)
	if err != nil {
		return nil, newReaderInitError("cannot read magic number", err)
	}
	if bytes.Compare(magic, []byte(magicBytes)) != 0 {
		return nil, &ErrReaderInit{Message: "invalid magic number: " + string(magic)}
	}
	meta, err := decodeHeaderMetadata(fr.r)
	if err != nil {
		return nil, newReaderInitError("cannot read header metadata", err)
	}
	fr.CompressionCodec, err = getHeaderString("avro.codec", meta)
	if err != nil {
		return nil, newReaderInitError("cannot read header metadata", err)
	}
	if !IsCompressionCodecSupported(fr.CompressionCodec) {
		return nil, newReaderInitError("unsupported codec: %s", fr.CompressionCodec)
	}
	fr.DataSchema, err = getHeaderString("avro.schema", meta)
	if err != nil {
		return nil, newReaderInitError("cannot read header metadata", err)
	}
	if fr.dataCodec, err = NewCodec(fr.DataSchema); err != nil {
		return nil, newReaderInitError("cannot compile schema", err)
	}
	fr.Sync = make([]byte, syncLength)
	if _, err = fr.r.Read(fr.Sync); err != nil {
		return nil, newReaderInitError("cannot read sync marker", err)
	}
	// setup reading pipeline
	toDecompress := make(chan *readerBlock)
	toDecode := make(chan *readerBlock)
	fr.deblocked = make(chan Datum)
	go read(fr, toDecompress)
	go decompress(fr, toDecompress, toDecode)
	go decode(fr, toDecode)
	return fr, nil
}

// Close releases resources and returns any Reader errors.
func (fr *Reader) Close() error {
	return fr.err
}

// Scan returns true if more data is ready to be read.
func (fr *Reader) Scan() bool {
	fr.datum = <-fr.deblocked
	return !fr.done
}

// Read returns the next element from the Reader.
func (fr *Reader) Read() (interface{}, error) {
	return fr.datum.Value, fr.datum.Err
}

func decodeHeaderMetadata(r io.Reader) (map[string]interface{}, error) {
	md, err := metadataCodec.Decode(r)
	if err != nil {
		return nil, err
	}
	return md.(map[string]interface{}), nil
}

func getHeaderString(someKey string, header map[string]interface{}) (string, error) {
	v, ok := header[someKey]
	if !ok {
		return "", fmt.Errorf("header ought to have %v key", someKey)
	}
	return string(v.([]byte)), nil
}

type readerBlock struct {
	datumCount int
	err        error
	r          io.Reader
}

// ErrReader is returned when the reader encounters an error.
type ErrReader struct {
	Message string
	Err     error
}

func (e ErrReader) Error() string {
	if e.Err == nil {
		return "cannot read from reader: " + e.Message
	}
	return "cannot read from reader: " + e.Message + ": " + e.Err.Error()
}

func newReaderError(a ...interface{}) *ErrReader {
	var err error
	var format, message string
	var ok bool
	if len(a) == 0 {
		return &ErrReader{"no reason given", nil}
	}
	// if last item is error: save it
	if err, ok = a[len(a)-1].(error); ok {
		a = a[:len(a)-1] // pop it
	}
	// if items left, first ought to be format string
	if len(a) > 0 {
		if format, ok = a[0].(string); ok {
			a = a[1:] // unshift
			message = fmt.Sprintf(format, a...)
		}
	}
	return &ErrReader{message, err}
}

func read(fr *Reader, toDecompress chan<- *readerBlock) {
	// NOTE: these variables created outside loop to reduce churn
	var lr io.Reader
	var bits []byte
	sync := make([]byte, syncLength)

	blockCount, blockSize, err := readBlockCountAndSize(fr.r)
	if err != nil {
		blockCount = 0
	}
	for blockCount != 0 {
		lr = io.LimitReader(fr.r, int64(blockSize))
		if bits, err = ioutil.ReadAll(lr); err != nil {
			err = newReaderError("cannot read block", err)
			break
		}
		toDecompress <- &readerBlock{datumCount: blockCount, r: bytes.NewReader(bits)}
		if _, err = fr.r.Read(sync); err != nil {
			err = newReaderError("cannot read sync marker", err)
			break
		}
		if bytes.Compare(fr.Sync, sync) != 0 {
			err = newReaderError(fmt.Sprintf("sync marker mismatch: %#v != %#v", sync, fr.Sync))
			break
		}
		if blockCount, blockSize, err = readBlockCountAndSize(fr.r); err != nil {
			break
		}
	}
	if err != nil {
		fr.err = err
	}
	close(toDecompress)
}

func readBlockCountAndSize(r io.Reader) (blockCount, blockSize int, err error) {
	bc, err := longCodec.Decode(r)
	if err != nil {
		if ed, ok := err.(*ErrDecoder); ok && ed.Err.Error() == "EOF" {
			return 0, 0, nil // we're done
		}
		return 0, 0, &ErrReaderBlockCount{err}
	}
	bs, err := longCodec.Decode(r)
	if err != nil {
		return 0, 0, &ErrReaderBlockCount{err}
	}
	return int(bc.(int64)), int(bs.(int64)), nil
}

func decompress(fr *Reader, toDecompress <-chan *readerBlock, toDecode chan<- *readerBlock) {
	switch fr.CompressionCodec {
	case CompressionDeflate:
		var rc io.ReadCloser
		var bits []byte
		for block := range toDecompress {
			rc = flate.NewReader(block.r)
			bits, block.err = ioutil.ReadAll(rc)
			if block.err != nil {
				block.err = newReaderError("cannot read from deflate", block.err)
				toDecode <- block
				_ = rc.Close() // already have the read error; ignore the close error
				continue
			}
			block.err = rc.Close()
			if block.err != nil {
				block.err = newReaderError("cannot close deflate", block.err)
				toDecode <- block
				continue
			}
			block.r = bytes.NewReader(bits)
			toDecode <- block
		}
	case CompressionNull:
		for block := range toDecompress {
			toDecode <- block
		}
	case CompressionSnappy:
		var src, dst []byte
		for block := range toDecompress {
			src, block.err = ioutil.ReadAll(block.r)
			if block.err != nil {
				block.err = newReaderError("cannot read", block.err)
				toDecode <- block
				continue
			}
			dst, block.err = snappy.Decode(dst, src)
			if block.err != nil {
				block.err = newReaderError("cannot decompress", block.err)
				toDecode <- block
				continue
			}
			block.r = bytes.NewReader(dst)
			toDecode <- block
		}
	}
	close(toDecode)
}

func decode(fr *Reader, toDecode <-chan *readerBlock) {
decodeLoop:
	for block := range toDecode {
		for i := 0; i < block.datumCount; i++ {
			var datum Datum
			datum.Value, datum.Err = fr.dataCodec.Decode(block.r)
			if datum.Value == nil && datum.Err == nil {
				break decodeLoop
			}
			fr.deblocked <- datum
		}
	}
	fr.done = true
	close(fr.deblocked)
}
<commit_msg>goavro.Reader Scan() and Read() loop now emit final datum from stream<commit_after>// Copyright 2015 LinkedIn Corp. Licensed under the Apache License,
// Version 2.0 (the "License"); you may not use this file except in
// compliance with the License.
 You may obtain a copy of the License
// at http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
//
distributed under the License is distributed on an "AS IS" BASIS,
//
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache
// License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License.
 You may obtain a copy of
// the License at http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
//
distributed under the License is distributed on an "AS IS" BASIS,
//
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.

package goavro

import (
	"bufio"
	"bytes"
	"compress/flate"
	"fmt"
	"github.com/golang/snappy"
	"io"
	"io/ioutil"
)

// ErrReaderInit is returned when the encoder encounters an error.
type ErrReaderInit struct {
	Message string
	Err     error
}

func (e ErrReaderInit) Error() string {
	if e.Err == nil {
		return "cannot build " + e.Message
	}
	return "cannot build " + e.Message + ": " + e.Err.Error()
}

func newReaderInitError(a ...interface{}) *ErrReaderInit {
	var err error
	var format, message string
	var ok bool
	if len(a) == 0 {
		return &ErrReaderInit{"cannot create reader: no reason given", nil}
	}
	// if last item is error: save it
	if err, ok = a[len(a)-1].(error); ok {
		a = a[:len(a)-1] // pop it
	}
	// if items left, first ought to be format string
	if len(a) > 0 {
		if format, ok = a[0].(string); ok {
			a = a[1:] // unshift
			message = fmt.Sprintf(format, a...)
		}
	}
	return &ErrReaderInit{message, err}
}

// ErrReaderBlockCount is returned when a reader detects an error
// while attempting to read the block count and block size.
type ErrReaderBlockCount struct {
	Err error
}

func (e *ErrReaderBlockCount) Error() string {
	return "cannot read block count and size: " + e.Err.Error()
}

// ReaderSetter functions are those those which are used to instantiate
// a new Reader.
type ReaderSetter func(*Reader) error

// BufferFromReader wraps the specified `io.Reader` using a
// `bufio.Reader` to read from a file.
func BufferFromReader(r io.Reader) ReaderSetter {
	return func(fr *Reader) error {
		fr.r = bufio.NewReader(r)
		return nil
	}
}

// FromReader specifies the `io.Reader` to use when reading a file.
func FromReader(r io.Reader) ReaderSetter {
	return func(fr *Reader) error {
		fr.r = r
		return nil
	}
}

// Reader structure contains data necessary to read Avro files.
type Reader struct {
	CompressionCodec string
	DataSchema       string
	Sync             []byte
	dataCodec        Codec
	datum            Datum
	deblocked        chan Datum
	err              error
	r                io.Reader
}

// NewReader returns a object to read data from an io.Reader using the
// Avro Object Container Files format.
//
//     func main() {
//         conn, err := net.Dial("tcp", "127.0.0.1:8080")
//         if err != nil {
//             log.Fatal(err)
//         }
//         fr, err := goavro.NewReader(goavro.FromReader(conn))
//         if err != nil {
//             log.Fatal("cannot create Reader: ", err)
//         }
//         defer func() {
//             if err := fr.Close(); err != nil {
//                 log.Fatal(err)
//             }
//         }()
//
//         for fr.Scan() {
//             datum, err := fr.Read()
//             if err != nil {
//                 log.Println("cannot read datum: ", err)
//                 continue
//             }
//             fmt.Println("RECORD: ", datum)
//         }
//     }
func NewReader(setters ...ReaderSetter) (*Reader, error) {
	var err error
	fr := &Reader{}
	for _, setter := range setters {
		err = setter(fr)
		if err != nil {
			return nil, newReaderInitError(err)
		}
	}
	if fr.r == nil {
		return nil, newReaderInitError("must specify io.Reader")
	}
	// read in header information and use it to initialize Reader
	magic := make([]byte, 4)
	_, err = fr.r.Read(magic)
	if err != nil {
		return nil, newReaderInitError("cannot read magic number", err)
	}
	if bytes.Compare(magic, []byte(magicBytes)) != 0 {
		return nil, &ErrReaderInit{Message: "invalid magic number: " + string(magic)}
	}
	meta, err := decodeHeaderMetadata(fr.r)
	if err != nil {
		return nil, newReaderInitError("cannot read header metadata", err)
	}
	fr.CompressionCodec, err = getHeaderString("avro.codec", meta)
	if err != nil {
		return nil, newReaderInitError("cannot read header metadata", err)
	}
	if !IsCompressionCodecSupported(fr.CompressionCodec) {
		return nil, newReaderInitError("unsupported codec: %s", fr.CompressionCodec)
	}
	fr.DataSchema, err = getHeaderString("avro.schema", meta)
	if err != nil {
		return nil, newReaderInitError("cannot read header metadata", err)
	}
	if fr.dataCodec, err = NewCodec(fr.DataSchema); err != nil {
		return nil, newReaderInitError("cannot compile schema", err)
	}
	fr.Sync = make([]byte, syncLength)
	if _, err = fr.r.Read(fr.Sync); err != nil {
		return nil, newReaderInitError("cannot read sync marker", err)
	}
	// setup reading pipeline
	toDecompress := make(chan *readerBlock)
	toDecode := make(chan *readerBlock)
	fr.deblocked = make(chan Datum)
	go read(fr, toDecompress)
	go decompress(fr, toDecompress, toDecode)
	go decode(fr, toDecode)
	return fr, nil
}

// Close releases resources and returns any Reader errors.
func (fr *Reader) Close() error {
	return fr.err
}

// Scan returns true if more data is ready to be read.
func (fr *Reader) Scan() bool {
	var ok bool
	fr.datum, ok = <-fr.deblocked
	return ok
}

// Read returns the next element from the Reader.
func (fr *Reader) Read() (interface{}, error) {
	return fr.datum.Value, fr.datum.Err
}

func decodeHeaderMetadata(r io.Reader) (map[string]interface{}, error) {
	md, err := metadataCodec.Decode(r)
	if err != nil {
		return nil, err
	}
	return md.(map[string]interface{}), nil
}

func getHeaderString(someKey string, header map[string]interface{}) (string, error) {
	v, ok := header[someKey]
	if !ok {
		return "", fmt.Errorf("header ought to have %v key", someKey)
	}
	return string(v.([]byte)), nil
}

type readerBlock struct {
	datumCount int
	err        error
	r          io.Reader
}

// ErrReader is returned when the reader encounters an error.
type ErrReader struct {
	Message string
	Err     error
}

func (e ErrReader) Error() string {
	if e.Err == nil {
		return "cannot read from reader: " + e.Message
	}
	return "cannot read from reader: " + e.Message + ": " + e.Err.Error()
}

func newReaderError(a ...interface{}) *ErrReader {
	var err error
	var format, message string
	var ok bool
	if len(a) == 0 {
		return &ErrReader{"no reason given", nil}
	}
	// if last item is error: save it
	if err, ok = a[len(a)-1].(error); ok {
		a = a[:len(a)-1] // pop it
	}
	// if items left, first ought to be format string
	if len(a) > 0 {
		if format, ok = a[0].(string); ok {
			a = a[1:] // unshift
			message = fmt.Sprintf(format, a...)
		}
	}
	return &ErrReader{message, err}
}

func read(fr *Reader, toDecompress chan<- *readerBlock) {
	// NOTE: these variables created outside loop to reduce churn
	var lr io.Reader
	var bits []byte
	sync := make([]byte, syncLength)

	blockCount, blockSize, err := readBlockCountAndSize(fr.r)
	if err != nil {
		fr.err = err
		blockCount = 0
	}
	for blockCount != 0 {
		lr = io.LimitReader(fr.r, int64(blockSize))
		if bits, err = ioutil.ReadAll(lr); err != nil {
			err = newReaderError("cannot read block", err)
			break
		}
		toDecompress <- &readerBlock{datumCount: blockCount, r: bytes.NewReader(bits)}
		if _, fr.err = fr.r.Read(sync); fr.err != nil {
			err = newReaderError("cannot read sync marker", fr.err)
			break
		}
		if !bytes.Equal(fr.Sync, sync) {
			fr.err = newReaderError(fmt.Sprintf("sync marker mismatch: %#v != %#v", sync, fr.Sync))
			break
		}
		if blockCount, blockSize, fr.err = readBlockCountAndSize(fr.r); fr.err != nil {
			break
		}
	}
	close(toDecompress)
}

func readBlockCountAndSize(r io.Reader) (int, int, error) {
	bc, err := longCodec.Decode(r)
	if err != nil {
		if ed, ok := err.(*ErrDecoder); ok && ed.Err == io.EOF {
			return 0, 0, nil // we're done
		}
		return 0, 0, &ErrReaderBlockCount{err}
	}
	bs, err := longCodec.Decode(r)
	if err != nil {
		return 0, 0, &ErrReaderBlockCount{err}
	}
	return int(bc.(int64)), int(bs.(int64)), nil
}

func decompress(fr *Reader, toDecompress <-chan *readerBlock, toDecode chan<- *readerBlock) {
	switch fr.CompressionCodec {
	case CompressionDeflate:
		var rc io.ReadCloser
		var bits []byte
		for block := range toDecompress {
			rc = flate.NewReader(block.r)
			bits, block.err = ioutil.ReadAll(rc)
			if block.err != nil {
				block.err = newReaderError("cannot read from deflate", block.err)
				toDecode <- block
				_ = rc.Close() // already have the read error; ignore the close error
				continue
			}
			block.err = rc.Close()
			if block.err != nil {
				block.err = newReaderError("cannot close deflate", block.err)
				toDecode <- block
				continue
			}
			block.r = bytes.NewReader(bits)
			toDecode <- block
		}
	case CompressionNull:
		for block := range toDecompress {
			toDecode <- block
		}
	case CompressionSnappy:
		var src, dst []byte
		for block := range toDecompress {
			src, block.err = ioutil.ReadAll(block.r)
			if block.err != nil {
				block.err = newReaderError("cannot read", block.err)
				toDecode <- block
				continue
			}
			dst, block.err = snappy.Decode(dst, src)
			if block.err != nil {
				block.err = newReaderError("cannot decompress", block.err)
				toDecode <- block
				continue
			}
			block.r = bytes.NewReader(dst)
			toDecode <- block
		}
	}
	close(toDecode)
}

func decode(fr *Reader, toDecode <-chan *readerBlock) {
decodeLoop:
	for block := range toDecode {
		for i := 0; i < block.datumCount; i++ {
			var datum Datum
			datum.Value, datum.Err = fr.dataCodec.Decode(block.r)
			if datum.Value == nil && datum.Err == nil {
				break decodeLoop
			}
			fr.deblocked <- datum
		}
	}
	close(fr.deblocked)
}
<|endoftext|>


<commit_before>package kcp

import (
	"io"
	"sync"
	"time"

	"github.com/v2ray/v2ray-core/common/alloc"
)

type ReceivingWindow struct {
	start uint32
	size  uint32
	list  []*DataSegment
}

func NewReceivingWindow(size uint32) *ReceivingWindow {
	return &ReceivingWindow{
		start: 0,
		size:  size,
		list:  make([]*DataSegment, size),
	}
}

func (this *ReceivingWindow) Size() uint32 {
	return this.size
}

func (this *ReceivingWindow) Position(idx uint32) uint32 {
	return (idx + this.start) % this.size
}

func (this *ReceivingWindow) Set(idx uint32, value *DataSegment) bool {
	pos := this.Position(idx)
	if this.list[pos] != nil {
		return false
	}
	this.list[pos] = value
	return true
}

func (this *ReceivingWindow) Remove(idx uint32) *DataSegment {
	pos := this.Position(idx)
	e := this.list[pos]
	this.list[pos] = nil
	return e
}

func (this *ReceivingWindow) RemoveFirst() *DataSegment {
	return this.Remove(0)
}

func (this *ReceivingWindow) Advance() {
	this.start++
	if this.start == this.size {
		this.start = 0
	}
}

type ReceivingQueue struct {
	sync.RWMutex
	closed  bool
	cache   *alloc.Buffer
	queue   chan *alloc.Buffer
	timeout time.Time
}

func NewReceivingQueue() *ReceivingQueue {
	return &ReceivingQueue{
		queue: make(chan *alloc.Buffer, effectiveConfig.ReadBuffer/effectiveConfig.Mtu),
	}
}

func (this *ReceivingQueue) Read(buf []byte) (int, error) {
	if this.cache.Len() > 0 {
		nBytes, err := this.cache.Read(buf)
		if this.cache.IsEmpty() {
			this.cache.Release()
			this.cache = nil
		}
		return nBytes, err
	}

	var totalBytes int

L:
	for totalBytes < len(buf) {
		timeToSleep := time.Millisecond
		select {
		case payload, open := <-this.queue:
			if !open {
				return totalBytes, io.EOF
			}
			nBytes, err := payload.Read(buf)
			totalBytes += nBytes
			if err != nil {
				return totalBytes, err
			}
			if !payload.IsEmpty() {
				this.cache = payload
			}
			buf = buf[nBytes:]
		case <-time.After(timeToSleep):
			if totalBytes > 0 {
				break L
			}
			this.RLock()
			if !this.timeout.IsZero() && this.timeout.Before(time.Now()) {
				this.RUnlock()
				return totalBytes, errTimeout
			}
			this.RUnlock()
			timeToSleep += 500 * time.Millisecond
		}
	}

	return totalBytes, nil
}

func (this *ReceivingQueue) Put(payload *alloc.Buffer) {
	this.RLock()
	defer this.RUnlock()

	if this.closed {
		payload.Release()
		return
	}

	this.queue <- payload
}

func (this *ReceivingQueue) SetReadDeadline(t time.Time) error {
	this.Lock()
	defer this.Unlock()

	this.timeout = t
	return nil
}

func (this *ReceivingQueue) Close() {
	this.Lock()
	defer this.Unlock()

	if this.closed {
		return
	}
	this.closed = true
	close(this.queue)
}

type ACKList struct {
	kcp        *KCP
	timestamps []uint32
	numbers    []uint32
	nextFlush  []uint32
}

func NewACKList(kcp *KCP) *ACKList {
	return &ACKList{
		kcp:        kcp,
		timestamps: make([]uint32, 0, 32),
		numbers:    make([]uint32, 0, 32),
		nextFlush:  make([]uint32, 0, 32),
	}
}

func (this *ACKList) Add(number uint32, timestamp uint32) {
	this.timestamps = append(this.timestamps, timestamp)
	this.numbers = append(this.numbers, number)
	this.nextFlush = append(this.nextFlush, 0)
}

func (this *ACKList) Clear(una uint32) {
	count := 0
	for i := 0; i < len(this.numbers); i++ {
		if this.numbers[i] >= una {
			if i != count {
				this.numbers[count] = this.numbers[i]
				this.timestamps[count] = this.timestamps[i]
				this.nextFlush[count] = this.nextFlush[i]
			}
			count++
		}
	}
	if count < len(this.numbers) {
		this.numbers = this.numbers[:count]
		this.timestamps = this.timestamps[:count]
		this.nextFlush = this.nextFlush[:count]
	}
}

func (this *ACKList) Flush() bool {
	seg := &ACKSegment{
		Conv:            this.kcp.conv,
		ReceivingNext:   this.kcp.rcv_nxt,
		ReceivingWindow: this.kcp.rcv_nxt + this.kcp.rcv_wnd,
	}
	if this.kcp.state == StateReadyToClose {
		seg.Opt = SegmentOptionClose
	}
	current := this.kcp.current
	for i := 0; i < len(this.numbers); i++ {
		if this.nextFlush[i] <= current {
			seg.Count++
			seg.NumberList = append(seg.NumberList, this.numbers[i])
			seg.TimestampList = append(seg.TimestampList, this.timestamps[i])
			this.nextFlush[i] = current + 50
			if seg.Count == 128 {
				break
			}
		}
	}
	if seg.Count > 0 {
		this.kcp.output.Write(seg)
		return true
	}
	return false
}
<commit_msg>extend ack timeout<commit_after>package kcp

import (
	"io"
	"sync"
	"time"

	"github.com/v2ray/v2ray-core/common/alloc"
)

type ReceivingWindow struct {
	start uint32
	size  uint32
	list  []*DataSegment
}

func NewReceivingWindow(size uint32) *ReceivingWindow {
	return &ReceivingWindow{
		start: 0,
		size:  size,
		list:  make([]*DataSegment, size),
	}
}

func (this *ReceivingWindow) Size() uint32 {
	return this.size
}

func (this *ReceivingWindow) Position(idx uint32) uint32 {
	return (idx + this.start) % this.size
}

func (this *ReceivingWindow) Set(idx uint32, value *DataSegment) bool {
	pos := this.Position(idx)
	if this.list[pos] != nil {
		return false
	}
	this.list[pos] = value
	return true
}

func (this *ReceivingWindow) Remove(idx uint32) *DataSegment {
	pos := this.Position(idx)
	e := this.list[pos]
	this.list[pos] = nil
	return e
}

func (this *ReceivingWindow) RemoveFirst() *DataSegment {
	return this.Remove(0)
}

func (this *ReceivingWindow) Advance() {
	this.start++
	if this.start == this.size {
		this.start = 0
	}
}

type ReceivingQueue struct {
	sync.RWMutex
	closed  bool
	cache   *alloc.Buffer
	queue   chan *alloc.Buffer
	timeout time.Time
}

func NewReceivingQueue() *ReceivingQueue {
	return &ReceivingQueue{
		queue: make(chan *alloc.Buffer, effectiveConfig.ReadBuffer/effectiveConfig.Mtu),
	}
}

func (this *ReceivingQueue) Read(buf []byte) (int, error) {
	if this.cache.Len() > 0 {
		nBytes, err := this.cache.Read(buf)
		if this.cache.IsEmpty() {
			this.cache.Release()
			this.cache = nil
		}
		return nBytes, err
	}

	var totalBytes int

L:
	for totalBytes < len(buf) {
		timeToSleep := time.Millisecond
		select {
		case payload, open := <-this.queue:
			if !open {
				return totalBytes, io.EOF
			}
			nBytes, err := payload.Read(buf)
			totalBytes += nBytes
			if err != nil {
				return totalBytes, err
			}
			if !payload.IsEmpty() {
				this.cache = payload
			}
			buf = buf[nBytes:]
		case <-time.After(timeToSleep):
			if totalBytes > 0 {
				break L
			}
			this.RLock()
			if !this.timeout.IsZero() && this.timeout.Before(time.Now()) {
				this.RUnlock()
				return totalBytes, errTimeout
			}
			this.RUnlock()
			timeToSleep += 500 * time.Millisecond
		}
	}

	return totalBytes, nil
}

func (this *ReceivingQueue) Put(payload *alloc.Buffer) {
	this.RLock()
	defer this.RUnlock()

	if this.closed {
		payload.Release()
		return
	}

	this.queue <- payload
}

func (this *ReceivingQueue) SetReadDeadline(t time.Time) error {
	this.Lock()
	defer this.Unlock()

	this.timeout = t
	return nil
}

func (this *ReceivingQueue) Close() {
	this.Lock()
	defer this.Unlock()

	if this.closed {
		return
	}
	this.closed = true
	close(this.queue)
}

type ACKList struct {
	kcp        *KCP
	timestamps []uint32
	numbers    []uint32
	nextFlush  []uint32
}

func NewACKList(kcp *KCP) *ACKList {
	return &ACKList{
		kcp:        kcp,
		timestamps: make([]uint32, 0, 32),
		numbers:    make([]uint32, 0, 32),
		nextFlush:  make([]uint32, 0, 32),
	}
}

func (this *ACKList) Add(number uint32, timestamp uint32) {
	this.timestamps = append(this.timestamps, timestamp)
	this.numbers = append(this.numbers, number)
	this.nextFlush = append(this.nextFlush, 0)
}

func (this *ACKList) Clear(una uint32) {
	count := 0
	for i := 0; i < len(this.numbers); i++ {
		if this.numbers[i] >= una {
			if i != count {
				this.numbers[count] = this.numbers[i]
				this.timestamps[count] = this.timestamps[i]
				this.nextFlush[count] = this.nextFlush[i]
			}
			count++
		}
	}
	if count < len(this.numbers) {
		this.numbers = this.numbers[:count]
		this.timestamps = this.timestamps[:count]
		this.nextFlush = this.nextFlush[:count]
	}
}

func (this *ACKList) Flush() bool {
	seg := &ACKSegment{
		Conv:            this.kcp.conv,
		ReceivingNext:   this.kcp.rcv_nxt,
		ReceivingWindow: this.kcp.rcv_nxt + this.kcp.rcv_wnd,
	}
	if this.kcp.state == StateReadyToClose {
		seg.Opt = SegmentOptionClose
	}
	current := this.kcp.current
	for i := 0; i < len(this.numbers); i++ {
		if this.nextFlush[i] <= current {
			seg.Count++
			seg.NumberList = append(seg.NumberList, this.numbers[i])
			seg.TimestampList = append(seg.TimestampList, this.timestamps[i])
			this.nextFlush[i] = current + 100
			if seg.Count == 128 {
				break
			}
		}
	}
	if seg.Count > 0 {
		this.kcp.output.Write(seg)
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package oneview

import (
	"sort"
	"sync"
)

// ICCol a list of Interconnect objects
type ICCol struct {
	Type        string `json:"type"`
	Total       int    `json:"total,omitempty"`       // "total": 1,
	Count       int    `json:"count,omitempty"`       // "count": 1,
	Start       int    `json:"start,omitempty"`       // "start": 0,
	PrevPageURI string `json:"prevPageUri,omitempty"` // "prevPageUri": null,
	NextPageURI string //`json:"nextPageUri,omitempty"` // "nextPageUri": null,
	URI         string `json:"uri,omitempty"`     // "uri": "/rest/server-profiles?filter=serialNumber%20matches%20%272M25090RMW%27&sort=name:asc"
	Members     []IC   `json:"members,omitempty"` // "members":[]
}

// Interconnect object
type IC struct {
	Type                   string        `json:"type"`
	LogicalInterconnectURI string        `json:"logicalInterconnectUri"`
	PartNumber             string        `json:"partNumber"`
	Ports                  []Port        `json:"ports"`
	IPAddressList          []interface{} `json:"ipAddressList"`
	// SnmpConfiguration struct {
	// 	Type             string        `json:"type"`
	// 	ReadCommunity    string        `json:"readCommunity"`
	// 	TrapDestinations []interface{} `json:"trapDestinations"`
	// 	SystemContact    string        `json:"systemContact"`
	// 	SnmpAccess       []interface{} `json:"snmpAccess"`
	// 	Enabled          bool          `json:"enabled"`
	// 	Description      interface{}   `json:"description"`
	// 	Status           interface{}   `json:"status"`
	// 	Name             interface{}   `json:"name"`
	// 	State            interface{}   `json:"state"`
	// 	Created          string     `json:"created"`
	// 	ETag             interface{}   `json:"eTag"`
	// 	Modified         string     `json:"modified"`
	// 	Category         string        `json:"category"`
	// 	URI              interface{}   `json:"uri"`
	// } `json:"snmpConfiguration"`
	// InterconnectLocation struct {
	// 	LocationEntries []struct {
	// 		Value string `json:"value"`
	// 		Type  string `json:"type"`
	// 	} `json:"locationEntries"`
	// } `json:"interconnectLocation"`
	// InterconnectTypeURI string      `json:"interconnectTypeUri"`
	// MgmtInterface       string      `json:"mgmtInterface"`
	// StackingMemberID    int         `json:"stackingMemberId"`
	// FirmwareVersion     string      `json:"firmwareVersion"`
	// PortCount           int         `json:"portCount"`
	// Model               string      `json:"model"`
	// EnclosureURI        string      `json:"enclosureUri"`
	// StackingDomainID    int         `json:"stackingDomainId"`
	// EnclosureType       string      `json:"enclosureType"`
	// PowerState          string      `json:"powerState"`
	// StackingDomainRole  string      `json:"stackingDomainRole"`
	// MigrationState      interface{} `json:"migrationState"`
	// QosConfiguration    struct {
	// 	Type            string `json:"type"`
	// 	ActiveQosConfig struct {
	// 		Type                       string        `json:"type"`
	// 		ConfigType                 string        `json:"configType"`
	// 		DownlinkClassificationType interface{}   `json:"downlinkClassificationType"`
	// 		UplinkClassificationType   interface{}   `json:"uplinkClassificationType"`
	// 		QosTrafficClassifiers      []interface{} `json:"qosTrafficClassifiers"`
	// 		Description                interface{}   `json:"description"`
	// 		Status                     interface{}   `json:"status"`
	// 		Name                       interface{}   `json:"name"`
	// 		State                      interface{}   `json:"state"`
	// 		Created                    interface{}   `json:"created"`
	// 		ETag                       interface{}   `json:"eTag"`
	// 		Modified                   interface{}   `json:"modified"`
	// 		Category                   string        `json:"category"`
	// 		URI                        interface{}   `json:"uri"`
	// 	} `json:"activeQosConfig"`
	// 	InactiveFCoEQosConfig    interface{} `json:"inactiveFCoEQosConfig"`
	// 	InactiveNonFCoEQosConfig interface{} `json:"inactiveNonFCoEQosConfig"`
	// 	Description              interface{} `json:"description"`
	// 	Status                   interface{} `json:"status"`
	// 	Name                     interface{} `json:"name"`
	// 	State                    interface{} `json:"state"`
	// 	Created                  string   `json:"created"`
	// 	ETag                     interface{} `json:"eTag"`
	// 	Modified                 string   `json:"modified"`
	// 	Category                 string      `json:"category"`
	// 	URI                      interface{} `json:"uri"`
	// } `json:"qosConfiguration"`
	// LldpIpv4Address            string `json:"lldpIpv4Address"`
	// LldpIpv6Address            string `json:"lldpIpv6Address"`
	// MaxBandwidth               string `json:"maxBandwidth"`
	// IgmpIdleTimeoutInterval    int    `json:"igmpIdleTimeoutInterval"`
	// EnablePauseFloodProtection bool   `json:"enablePauseFloodProtection"`
	ProductName string `json:"productName"`
	// IcmLicenses                struct {
	// 	License []struct {
	// 		RequiredCount int    `json:"requiredCount"`
	// 		LicenseType   string `json:"licenseType"`
	// 		ConsumedCount int    `json:"consumedCount"`
	// 		State         string `json:"state"`
	// 	} `json:"license"`
	// } `json:"icmLicenses"`
	UIDState                      string        `json:"uidState"`
	EnclosureName                 string        `json:"enclosureName"`
	BaseWWN                       string        `json:"baseWWN"`
	InterconnectMAC               string        `json:"interconnectMAC"`
	UnsupportedCapabilities       interface{}   `json:"unsupportedCapabilities"`
	InterconnectIP                interface{}   `json:"interconnectIP"`
	EnableTaggedLldp              bool          `json:"enableTaggedLldp"`
	SubPortCount                  int           `json:"subPortCount"`
	EdgeVirtualBridgingAvailable  bool          `json:"edgeVirtualBridgingAvailable"`
	EnableIgmpSnooping            bool          `json:"enableIgmpSnooping"`
	NetworkLoopProtectionInterval int           `json:"networkLoopProtectionInterval"`
	EnableFastMacCacheFailover    bool          `json:"enableFastMacCacheFailover"`
	EnableNetworkLoopProtection   bool          `json:"enableNetworkLoopProtection"`
	EnableRichTLV                 bool          `json:"enableRichTLV"`
	DeviceResetState              string        `json:"deviceResetState"`
	SerialNumber                  string        `json:"serialNumber"`
	Roles                         []string      `json:"roles"`
	HostName                      string        `json:"hostName"`
	ScopeUris                     []interface{} `json:"scopeUris"`
	Description                   interface{}   `json:"description"`
	Status                        string        `json:"status"`
	Name                          string        `json:"name"`
	State                         string        `json:"state"`
	Created                       string        `json:"created"`
	ETag                          string        `json:"eTag"`
	Modified                      string        `json:"modified"`
	Category                      string        `json:"category"`
	URI                           string        `json:"uri"`
	LIName                        string
	SFPList                       []SFP
}

type Port struct {
	Type                      string      `json:"type"`
	LagID                     int         `json:"lagId"`
	PortName                  string      `json:"portName"`
	PortStatus                string      `json:"portStatus"`
	FcPortProperties          interface{} `json:"fcPortProperties"`
	PortID                    string      `json:"portId"`
	InterconnectName          string      `json:"interconnectName"`
	PortHealthStatus          string      `json:"portHealthStatus"`
	Enabled                   bool        `json:"enabled"`
	PortStatusReason          string      `json:"portStatusReason"`
	PortType                  string      `json:"portType"`
	Vlans                     interface{} `json:"vlans"`
	DcbxInfo                  interface{} `json:"dcbxInfo"`
	BayNumber                 int         `json:"bayNumber"`
	Subports                  interface{} `json:"subports"`
	LagStates                 interface{} `json:"lagStates"`
	PortRunningCapabilityType interface{} `json:"portRunningCapabilityType"`
	PortMonitorConfigInfo     string      `json:"portMonitorConfigInfo"`
	PairedPortName            interface{} `json:"pairedPortName"`
	VendorSpecificPortName    interface{} `json:"vendorSpecificPortName"`
	Neighbor                  Neighbor    `json:"neighbor"`
	ConnectorType             string      `json:"connectorType"`
	AssociatedUplinkSetURI    interface{} `json:"associatedUplinkSetUri"`
	OperationalSpeed          interface{} `json:"operationalSpeed"`
	Available                 bool        `json:"available"`
	Capability                []string    `json:"capability"`
	PortTypeExtended          string      `json:"portTypeExtended"`
	ConfigPortTypes           []string    `json:"configPortTypes"`
	PortSplitMode             string      `json:"portSplitMode"`
	Description               interface{} `json:"description"`
	Status                    string      `json:"status"`
	Name                      string      `json:"name"`
	State                     interface{} `json:"state"`
	Created                   interface{} `json:"created"`
	ETag                      interface{} `json:"eTag"`
	Modified                  interface{} `json:"modified"`
	Category                  string      `json:"category"`
	URI                       string      `json:"uri"`
	TransceiverPN             string
}

type Neighbor struct {
	RemoteMgmtAddress        interface{} `json:"remoteMgmtAddress"`
	RemotePortID             string      `json:"remotePortId"`
	RemoteChassisID          string      `json:"remoteChassisId"`
	RemoteChassisIDType      interface{} `json:"remoteChassisIdType"`
	RemotePortIDType         interface{} `json:"remotePortIdType"`
	RemotePortDescription    interface{} `json:"remotePortDescription"`
	RemoteSystemDescription  interface{} `json:"remoteSystemDescription"`
	RemoteMgmtAddressType    interface{} `json:"remoteMgmtAddressType"`
	RemoteSystemCapabilities interface{} `json:"remoteSystemCapabilities"`
	RemoteSystemName         string      `json:"remoteSystemName"`
	RemoteType               interface{} `json:"remoteType"`
	LinkURI                  interface{} `json:"linkUri"`
	LinkLabel                interface{} `json:"linkLabel"`
}

type SFP struct {
	Identifier       string      `json:"identifier"`
	SerialNumber     string      `json:"serialNumber"`
	VendorName       string      `json:"vendorName"`
	VendorPartNumber string      `json:"vendorPartNumber"`
	VendorRevision   string      `json:"vendorRevision"`
	Speed            string      `json:"speed"`
	PortName         string      `json:"portName"`
	Connector        interface{} `json:"connector"`
	ExtIdentifier    string      `json:"extIdentifier"`
	VendorOui        string      `json:"vendorOui"`
}

func (c *CLIOVClient) GetIC() []IC {

	var wg sync.WaitGroup

	resourcelist := []string{"IC", "LI"}

	for _, v := range resourcelist {
		localv := v
		wg.Add(1)

		go func() {
			defer wg.Done()
			c.GetResourceLists(localv, "")
		}()
	}

	wg.Wait()

	icList := *(rmap["IC"].listptr.(*[]IC))
	liList := *(rmap["LI"].listptr.(*[]LI))

	liMap := make(map[string]LI)

	for _, v := range liList {
		liMap[v.URI] = v
	}

	for i, v := range icList {
		icList[i].LIName = liMap[v.LogicalInterconnectURI].Name
	}

	sort.Slice(icList, func(i, j int) bool { return icList[i].Name < icList[j].Name })

	return icList
}

func GetICPort() []IC {

	var wg sync.WaitGroup

	resourcelist := []string{"IC"}

	for _, v := range resourcelist {
		localv := v
		wg.Add(1)

		go func() {
			defer wg.Done()
			c := NewCLIOVClient()
			c.GetResourceLists(localv, "")
		}()
	}

	wg.Wait()

	icList := *(rmap["IC"].listptr.(*[]IC))

	return icList
}

func GetSFP() []IC {

	var wg sync.WaitGroup

	resourcelist := []string{"IC"}

	for _, v := range resourcelist {
		localv := v
		wg.Add(1)

		go func() {
			defer wg.Done()
			c := NewCLIOVClient()
			c.GetResourceLists(localv, "")
		}()
	}

	wg.Wait()

	icList := *(rmap["IC"].listptr.(*[]IC))

	for i := range icList {
		locallist := &icList[i]
		wg.Add(1)
		go func() {
			defer wg.Done()
			SFPGetURI(locallist)
		}()
	}

	wg.Wait()

	return icList

}

func SFPGetURI(ic *IC) {

	// log.Println("[DEBUG] Rest Get SFP", ic.Name)
	// defer timeTrack(time.Now(), "Rest Get SFP")

	// icID := strings.Replace(ic.URI, "/rest/interconnects/", "", -1)

	// c := NewCLIOVClient()

	// var sfpuri string
	// switch c.APIVersion {
	// case 500:
	// 	sfpuri = ICURL + "/" + icID + SFPURL //  /rest/interconnects/{id}/pluggableModuleInformation
	// case 300:
	// 	sfpuri = ICURL + SFPURL + "/" + icID //  /rest/interconnects/pluggableModuleInformation/{ID}
	// }

	// data, err := c.GetURI("", "", sfpuri)

	// if err != nil {
	// 	fmt.Println("getSFP URL err: ", err)
	// 	os.Exit(1)
	// }

	// var list []SFP
	// if err := json.Unmarshal(data, &list); err != nil {
	// 	fmt.Println("unmarshal SFP json err: ", err)
	// 	os.Exit(1)
	// }

	// sort.Slice(list, func(i, j int) bool { return list[i].PortName < list[j].PortName })

	// (*ic).SFPList = list

}
<commit_msg>quick add<commit_after>package oneview

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"sort"
	"strings"
	"sync"
	"time"
)

// ICCol a list of Interconnect objects
type ICCol struct {
	Type        string `json:"type"`
	Total       int    `json:"total,omitempty"`       // "total": 1,
	Count       int    `json:"count,omitempty"`       // "count": 1,
	Start       int    `json:"start,omitempty"`       // "start": 0,
	PrevPageURI string `json:"prevPageUri,omitempty"` // "prevPageUri": null,
	NextPageURI string //`json:"nextPageUri,omitempty"` // "nextPageUri": null,
	URI         string `json:"uri,omitempty"`     // "uri": "/rest/server-profiles?filter=serialNumber%20matches%20%272M25090RMW%27&sort=name:asc"
	Members     []IC   `json:"members,omitempty"` // "members":[]
}

// Interconnect object
type IC struct {
	Type                   string        `json:"type"`
	LogicalInterconnectURI string        `json:"logicalInterconnectUri"`
	PartNumber             string        `json:"partNumber"`
	Ports                  []Port        `json:"ports"`
	IPAddressList          []interface{} `json:"ipAddressList"`
	// SnmpConfiguration struct {
	// 	Type             string        `json:"type"`
	// 	ReadCommunity    string        `json:"readCommunity"`
	// 	TrapDestinations []interface{} `json:"trapDestinations"`
	// 	SystemContact    string        `json:"systemContact"`
	// 	SnmpAccess       []interface{} `json:"snmpAccess"`
	// 	Enabled          bool          `json:"enabled"`
	// 	Description      interface{}   `json:"description"`
	// 	Status           interface{}   `json:"status"`
	// 	Name             interface{}   `json:"name"`
	// 	State            interface{}   `json:"state"`
	// 	Created          string     `json:"created"`
	// 	ETag             interface{}   `json:"eTag"`
	// 	Modified         string     `json:"modified"`
	// 	Category         string        `json:"category"`
	// 	URI              interface{}   `json:"uri"`
	// } `json:"snmpConfiguration"`
	// InterconnectLocation struct {
	// 	LocationEntries []struct {
	// 		Value string `json:"value"`
	// 		Type  string `json:"type"`
	// 	} `json:"locationEntries"`
	// } `json:"interconnectLocation"`
	// InterconnectTypeURI string      `json:"interconnectTypeUri"`
	// MgmtInterface       string      `json:"mgmtInterface"`
	// StackingMemberID    int         `json:"stackingMemberId"`
	// FirmwareVersion     string      `json:"firmwareVersion"`
	// PortCount           int         `json:"portCount"`
	// Model               string      `json:"model"`
	// EnclosureURI        string      `json:"enclosureUri"`
	// StackingDomainID    int         `json:"stackingDomainId"`
	// EnclosureType       string      `json:"enclosureType"`
	// PowerState          string      `json:"powerState"`
	// StackingDomainRole  string      `json:"stackingDomainRole"`
	// MigrationState      interface{} `json:"migrationState"`
	// QosConfiguration    struct {
	// 	Type            string `json:"type"`
	// 	ActiveQosConfig struct {
	// 		Type                       string        `json:"type"`
	// 		ConfigType                 string        `json:"configType"`
	// 		DownlinkClassificationType interface{}   `json:"downlinkClassificationType"`
	// 		UplinkClassificationType   interface{}   `json:"uplinkClassificationType"`
	// 		QosTrafficClassifiers      []interface{} `json:"qosTrafficClassifiers"`
	// 		Description                interface{}   `json:"description"`
	// 		Status                     interface{}   `json:"status"`
	// 		Name                       interface{}   `json:"name"`
	// 		State                      interface{}   `json:"state"`
	// 		Created                    interface{}   `json:"created"`
	// 		ETag                       interface{}   `json:"eTag"`
	// 		Modified                   interface{}   `json:"modified"`
	// 		Category                   string        `json:"category"`
	// 		URI                        interface{}   `json:"uri"`
	// 	} `json:"activeQosConfig"`
	// 	InactiveFCoEQosConfig    interface{} `json:"inactiveFCoEQosConfig"`
	// 	InactiveNonFCoEQosConfig interface{} `json:"inactiveNonFCoEQosConfig"`
	// 	Description              interface{} `json:"description"`
	// 	Status                   interface{} `json:"status"`
	// 	Name                     interface{} `json:"name"`
	// 	State                    interface{} `json:"state"`
	// 	Created                  string   `json:"created"`
	// 	ETag                     interface{} `json:"eTag"`
	// 	Modified                 string   `json:"modified"`
	// 	Category                 string      `json:"category"`
	// 	URI                      interface{} `json:"uri"`
	// } `json:"qosConfiguration"`
	// LldpIpv4Address            string `json:"lldpIpv4Address"`
	// LldpIpv6Address            string `json:"lldpIpv6Address"`
	// MaxBandwidth               string `json:"maxBandwidth"`
	// IgmpIdleTimeoutInterval    int    `json:"igmpIdleTimeoutInterval"`
	// EnablePauseFloodProtection bool   `json:"enablePauseFloodProtection"`
	ProductName string `json:"productName"`
	// IcmLicenses                struct {
	// 	License []struct {
	// 		RequiredCount int    `json:"requiredCount"`
	// 		LicenseType   string `json:"licenseType"`
	// 		ConsumedCount int    `json:"consumedCount"`
	// 		State         string `json:"state"`
	// 	} `json:"license"`
	// } `json:"icmLicenses"`
	UIDState                      string        `json:"uidState"`
	EnclosureName                 string        `json:"enclosureName"`
	BaseWWN                       string        `json:"baseWWN"`
	InterconnectMAC               string        `json:"interconnectMAC"`
	UnsupportedCapabilities       interface{}   `json:"unsupportedCapabilities"`
	InterconnectIP                interface{}   `json:"interconnectIP"`
	EnableTaggedLldp              bool          `json:"enableTaggedLldp"`
	SubPortCount                  int           `json:"subPortCount"`
	EdgeVirtualBridgingAvailable  bool          `json:"edgeVirtualBridgingAvailable"`
	EnableIgmpSnooping            bool          `json:"enableIgmpSnooping"`
	NetworkLoopProtectionInterval int           `json:"networkLoopProtectionInterval"`
	EnableFastMacCacheFailover    bool          `json:"enableFastMacCacheFailover"`
	EnableNetworkLoopProtection   bool          `json:"enableNetworkLoopProtection"`
	EnableRichTLV                 bool          `json:"enableRichTLV"`
	DeviceResetState              string        `json:"deviceResetState"`
	SerialNumber                  string        `json:"serialNumber"`
	Roles                         []string      `json:"roles"`
	HostName                      string        `json:"hostName"`
	ScopeUris                     []interface{} `json:"scopeUris"`
	Description                   interface{}   `json:"description"`
	Status                        string        `json:"status"`
	Name                          string        `json:"name"`
	State                         string        `json:"state"`
	Created                       string        `json:"created"`
	ETag                          string        `json:"eTag"`
	Modified                      string        `json:"modified"`
	Category                      string        `json:"category"`
	URI                           string        `json:"uri"`
	LIName                        string
	SFPList                       []SFP
}

type Port struct {
	Type                      string      `json:"type"`
	LagID                     int         `json:"lagId"`
	PortName                  string      `json:"portName"`
	PortStatus                string      `json:"portStatus"`
	FcPortProperties          interface{} `json:"fcPortProperties"`
	PortID                    string      `json:"portId"`
	InterconnectName          string      `json:"interconnectName"`
	PortHealthStatus          string      `json:"portHealthStatus"`
	Enabled                   bool        `json:"enabled"`
	PortStatusReason          string      `json:"portStatusReason"`
	PortType                  string      `json:"portType"`
	Vlans                     interface{} `json:"vlans"`
	DcbxInfo                  interface{} `json:"dcbxInfo"`
	BayNumber                 int         `json:"bayNumber"`
	Subports                  interface{} `json:"subports"`
	LagStates                 interface{} `json:"lagStates"`
	PortRunningCapabilityType interface{} `json:"portRunningCapabilityType"`
	PortMonitorConfigInfo     string      `json:"portMonitorConfigInfo"`
	PairedPortName            interface{} `json:"pairedPortName"`
	VendorSpecificPortName    interface{} `json:"vendorSpecificPortName"`
	Neighbor                  Neighbor    `json:"neighbor"`
	ConnectorType             string      `json:"connectorType"`
	AssociatedUplinkSetURI    interface{} `json:"associatedUplinkSetUri"`
	OperationalSpeed          interface{} `json:"operationalSpeed"`
	Available                 bool        `json:"available"`
	Capability                []string    `json:"capability"`
	PortTypeExtended          string      `json:"portTypeExtended"`
	ConfigPortTypes           []string    `json:"configPortTypes"`
	PortSplitMode             string      `json:"portSplitMode"`
	Description               interface{} `json:"description"`
	Status                    string      `json:"status"`
	Name                      string      `json:"name"`
	State                     interface{} `json:"state"`
	Created                   interface{} `json:"created"`
	ETag                      interface{} `json:"eTag"`
	Modified                  interface{} `json:"modified"`
	Category                  string      `json:"category"`
	URI                       string      `json:"uri"`
	TransceiverPN             string
}

type Neighbor struct {
	RemoteMgmtAddress        interface{} `json:"remoteMgmtAddress"`
	RemotePortID             string      `json:"remotePortId"`
	RemoteChassisID          string      `json:"remoteChassisId"`
	RemoteChassisIDType      interface{} `json:"remoteChassisIdType"`
	RemotePortIDType         interface{} `json:"remotePortIdType"`
	RemotePortDescription    interface{} `json:"remotePortDescription"`
	RemoteSystemDescription  interface{} `json:"remoteSystemDescription"`
	RemoteMgmtAddressType    interface{} `json:"remoteMgmtAddressType"`
	RemoteSystemCapabilities interface{} `json:"remoteSystemCapabilities"`
	RemoteSystemName         string      `json:"remoteSystemName"`
	RemoteType               interface{} `json:"remoteType"`
	LinkURI                  interface{} `json:"linkUri"`
	LinkLabel                interface{} `json:"linkLabel"`
}

type SFP struct {
	Identifier       string      `json:"identifier"`
	SerialNumber     string      `json:"serialNumber"`
	VendorName       string      `json:"vendorName"`
	VendorPartNumber string      `json:"vendorPartNumber"`
	VendorRevision   string      `json:"vendorRevision"`
	Speed            string      `json:"speed"`
	PortName         string      `json:"portName"`
	Connector        interface{} `json:"connector"`
	ExtIdentifier    string      `json:"extIdentifier"`
	VendorOui        string      `json:"vendorOui"`
}

func (c *CLIOVClient) GetIC() []IC {

	var wg sync.WaitGroup

	resourcelist := []string{"IC", "LI"}

	for _, v := range resourcelist {
		localv := v
		wg.Add(1)

		go func() {
			defer wg.Done()
			c.GetResourceLists(localv, "")
		}()
	}

	wg.Wait()

	icList := *(rmap["IC"].listptr.(*[]IC))
	liList := *(rmap["LI"].listptr.(*[]LI))

	liMap := make(map[string]LI)

	for _, v := range liList {
		liMap[v.URI] = v
	}

	for i, v := range icList {
		icList[i].LIName = liMap[v.LogicalInterconnectURI].Name
	}

	sort.Slice(icList, func(i, j int) bool { return icList[i].Name < icList[j].Name })

	return icList
}

func GetICPort() []IC {

	var wg sync.WaitGroup

	resourcelist := []string{"IC"}

	for _, v := range resourcelist {
		localv := v
		wg.Add(1)

		go func() {
			defer wg.Done()
			c := NewCLIOVClient()
			c.GetResourceLists(localv, "")
		}()
	}

	wg.Wait()

	icList := *(rmap["IC"].listptr.(*[]IC))

	return icList
}

func GetSFP() []IC {

	var wg sync.WaitGroup

	resourcelist := []string{"IC"}

	for _, v := range resourcelist {
		localv := v
		wg.Add(1)

		go func() {
			defer wg.Done()
			c := NewCLIOVClient()
			c.GetResourceLists(localv, "")
		}()
	}

	wg.Wait()

	icList := *(rmap["IC"].listptr.(*[]IC))

	for i := range icList {
		locallist := &icList[i]
		wg.Add(1)
		go func() {
			defer wg.Done()
			SFPGetURI(locallist)
		}()
	}

	wg.Wait()

	return icList

}

func SFPGetURI(ic *IC) {

	log.Println("[DEBUG] Rest Get SFP", ic.Name)
	defer timeTrack(time.Now(), "Rest Get SFP")

	icID := strings.Replace(ic.URI, "/rest/interconnects/", "", -1)

	c := NewCLIOVClient()

	var sfpuri string
	switch c.APIVersion {
	case 500:
		sfpuri = ICURL + "/" + icID + SFPURL //  /rest/interconnects/{id}/pluggableModuleInformation
	case 300:
		sfpuri = ICURL + SFPURL + "/" + icID //  /rest/interconnects/pluggableModuleInformation/{ID}
	}

	data, err := c.SendHTTPRequest("Get", sfpuri, "", "", nil)

	if err != nil {
		fmt.Println("getSFP URL err: ", err)
		os.Exit(1)
	}

	var list []SFP
	if err := json.Unmarshal(data, &list); err != nil {
		fmt.Println("unmarshal SFP json err: ", err)
		os.Exit(1)
	}

	sort.Slice(list, func(i, j int) bool { return list[i].PortName < list[j].PortName })

	(*ic).SFPList = list

}
<|endoftext|>