<commit_before>package nodes

import (
	"fmt"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/stampzilla/gozwave/commands"
	"github.com/stampzilla/gozwave/commands/reports"
)

func (n *Node) RequestEndpoints() error {
	// Todo: Send raw messages here
	n.RLock()
	if n.Device == nil || n.Device.CommandClasses == nil {
		n.RUnlock()
		n.Lock()
		n.Endpoints = make([]*Endpoint, 0)
		n.Unlock()
		return fmt.Errorf("Failed 'RequestEndpoints', no commandclasses exists")
	}

	//for _, v := range n.Device.CommandClasses {
	//logrus.Errorf("Request endpoint %d:%x - %s", n.Id, byte(v.ID), v.ID)
	//}
	n.RUnlock()

	cmd := commands.NewRaw(
		[]byte{
			commands.MultiInstance, // Command class
			0x07, // Command: MultiInstanceCmd_Get
			//byte(v.ID),
			0x25, // TransmitOptions?
			//0x23, // Callback?
		})
	cmd.SetNode(n.Id)

	fmt.Println("")
	fmt.Println("")
	logrus.Debugf("Request endpoint %d", n.Id)

	t, _ := n.connection.WriteAndWaitForReport(cmd, time.Second*2, 0x08) // Request node information
	report := <-t

	logrus.Debugf("Request endpoint %d report: %#v", n.Id, report)

	if report != nil {
		switch cmd := report.(type) {
		case *reports.MultiChannelEndPoints:
			//n.ManufacurerSpecific = cmd
			logrus.Debug(cmd.String())
			n.Lock()
			n.Endpoints = make([]*Endpoint, 0)
			for i := 1; i < cmd.Endpoints; i++ {
				n.Endpoints = append(n.Endpoints, &Endpoint{
					Id:             i,
					CommandClasses: n.Device.CommandClasses,

					StateBool:  make(map[string]bool),
					StateFloat: make(map[string]float64),

					node: n,
				})
			}
			n.Unlock()
			return nil
		default:
			logrus.Errorf("Wrong type: %t", cmd)
		}
	}

	//}

	return fmt.Errorf("Failed to request endpoints")
}
<commit_msg>Bugfixed failing build<commit_after>package nodes

import (
	"fmt"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/stampzilla/gozwave/commands"
	"github.com/stampzilla/gozwave/commands/reports"
)

func (n *Node) RequestEndpoints() error {
	// Todo: Send raw messages here
	n.RLock()
	if n.Device == nil || n.Device.CommandClasses == nil {
		n.RUnlock()
		n.Lock()
		n.Endpoints = make([]*Endpoint, 0)
		n.Unlock()
		return fmt.Errorf("Failed 'RequestEndpoints', no commandclasses exists")
	}

	//for _, v := range n.Device.CommandClasses {
	//logrus.Errorf("Request endpoint %d:%x - %s", n.Id, byte(v.ID), v.ID)
	//}
	n.RUnlock()

	cmd := commands.NewRaw(
		[]byte{
			commands.MultiInstance, // Command class
			0x07, // Command: MultiInstanceCmd_Get
			//byte(v.ID),
			0x25, // TransmitOptions?
			//0x23, // Callback?
		})
	cmd.SetNode(n.Id)

	fmt.Println("")
	fmt.Println("")
	logrus.Debugf("Request endpoint %d", n.Id)

	t, _ := n.connection.WriteAndWaitForReport(cmd, time.Second*2, 0x08) // Request node information
	report := <-t

	logrus.Debugf("Request endpoint %d report: %#v", n.Id, report)

	if report != nil {
		switch cmd := report.(type) {
		case *reports.MultiChannelEndpoints:
			//n.ManufacurerSpecific = cmd
			logrus.Debug(cmd.String())
			n.Lock()
			n.Endpoints = make([]*Endpoint, 0)
			for i := 1; i < cmd.Endpoints; i++ {
				n.Endpoints = append(n.Endpoints, &Endpoint{
					Id:             i,
					CommandClasses: n.Device.CommandClasses,

					StateBool:  make(map[string]bool),
					StateFloat: make(map[string]float64),

					node: n,
				})
			}
			n.Unlock()
			return nil
		default:
			logrus.Errorf("Wrong type: %t", cmd)
		}
	}

	//}

	return fmt.Errorf("Failed to request endpoints")
}
<|endoftext|>


<commit_before>package core

import (
	"io"
	"bytes"
	"errors"
	"fmt"
	"encoding/json"
	"sort"
	"strings"
	"encoding/hex"
	"github.com/s3git/s3git-go/internal/cas"
	"github.com/s3git/s3git-go/internal/kv"
)

type treeObject struct {
	coreObject
	S3gitAdded   []string `json:"s3gitAdded"`
	S3gitRemoved []string `json:"s3gitRemoved"`
	S3gitPadding string   `json:"s3gitPadding"`
}

func makeTreeObject(added <-chan []byte, removed []string) *treeObject {
	to := treeObject{coreObject: coreObject{S3gitVersion: 1, S3gitType: kv.TREE}}

	addedArray := []string{}

	for k := range added {
		addedArray = append(addedArray, hex.EncodeToString(k))
	}

	sort.Strings(addedArray)
	to.S3gitAdded = addedArray

	sort.Strings(removed)
	to.S3gitRemoved = removed

	return &to
}

func (to *treeObject) empty() bool {
	return len(to.S3gitAdded) == 0 && len(to.S3gitRemoved) == 0
}

func (to *treeObject) writeToDisk() (string, error) {

	buf := new(bytes.Buffer)

	encoder := json.NewEncoder(buf)
	if err := encoder.Encode(to); err != nil {
		return "", err
	}

	return to.write(buf, kv.TREE)
}

// Return tree object based on hash
func GetTreeObject(hash string) (*treeObject, error) {

	cr := cas.MakeReader(hash)
	if cr == nil {
		return nil, errors.New(fmt.Sprint("Failed to read hash %s", hash))
	}

	buf := bytes.NewBuffer(nil)
	// TODO: Find out why io.Copy does not read whole file from cas (truncated for 50 MB tree files)
	// io.Copy(buf, cr)

	size := 0
	array := make([]byte, cas.ChunkSize)
	for {
		read, err := cr.Read(array)
		size += read
		if read > 0 {
			_, err := buf.Write(array[:read])
			if err != nil {
				panic(err)
			}
		}
		if err == io.EOF {
			break
		}
	}

	s := string(buf.Bytes())

	return GetTreeObjectFromString(s)
}

// Return tree object from string contents
func GetTreeObjectFromString(s string) (*treeObject, error) {

	dec := json.NewDecoder(strings.NewReader(s))
	var to treeObject
	if err := dec.Decode(&to); err != nil {
		return nil, err
	}

	return &to, nil
}
<commit_msg>Formatting change<commit_after>package core

import (
	"bytes"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/s3git/s3git-go/internal/cas"
	"github.com/s3git/s3git-go/internal/kv"
	"io"
	"sort"
	"strings"
)

type treeObject struct {
	coreObject
	S3gitAdded      []string `json:"s3gitAdded"`
	S3gitRemoved    []string `json:"s3gitRemoved"`
	S3gitPadding    string   `json:"s3gitPadding"`
	S3gitBlobUrl    string   `json:"s3gitBlobUrl"`    // URL to access blob (when different from tree object itself)
	S3gitBlobRegion string   `json:"s3gitBlobRegion"` // URL to access blob (when different from tree object itself)
}

func makeTreeObject(added <-chan []byte, removed []string) *treeObject {
	to := treeObject{coreObject: coreObject{S3gitVersion: 1, S3gitType: kv.TREE}}

	addedArray := []string{}

	for k := range added {
		addedArray = append(addedArray, hex.EncodeToString(k))
	}

	sort.Strings(addedArray)
	to.S3gitAdded = addedArray

	sort.Strings(removed)
	to.S3gitRemoved = removed

	return &to
}

func (to *treeObject) empty() bool {
	return len(to.S3gitAdded) == 0 && len(to.S3gitRemoved) == 0
}

func (to *treeObject) writeToDisk() (string, error) {

	buf := new(bytes.Buffer)

	encoder := json.NewEncoder(buf)
	if err := encoder.Encode(to); err != nil {
		return "", err
	}

	return to.write(buf, kv.TREE)
}

// Return tree object based on hash
func GetTreeObject(hash string) (*treeObject, error) {

	cr := cas.MakeReader(hash)
	if cr == nil {
		return nil, errors.New(fmt.Sprint("Failed to read hash %s", hash))
	}

	buf := bytes.NewBuffer(nil)
	// TODO: Find out why io.Copy does not read whole file from cas (truncated for 50 MB tree files)
	// io.Copy(buf, cr)

	size := 0
	array := make([]byte, cas.ChunkSize)
	for {
		read, err := cr.Read(array)
		size += read
		if read > 0 {
			_, err := buf.Write(array[:read])
			if err != nil {
				panic(err)
			}
		}
		if err == io.EOF {
			break
		}
	}

	s := string(buf.Bytes())

	return GetTreeObjectFromString(s)
}

// Return tree object from string contents
func GetTreeObjectFromString(s string) (*treeObject, error) {

	dec := json.NewDecoder(strings.NewReader(s))
	var to treeObject
	if err := dec.Decode(&to); err != nil {
		return nil, err
	}

	return &to, nil
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the LGPLv3, see LICENCE file for details.

package v4

import (
	"net/http"
	"strconv"
	"sync/atomic"

	"github.com/juju/utils/parallel"
	"gopkg.in/errgo.v1"

	"github.com/juju/charmstore/internal/charmstore"
	"github.com/juju/charmstore/internal/router"
	"github.com/juju/charmstore/params"
)

const maxConcurrency = 20

// GET search[?text=text][&autocomplete=1][&filter=value…][&limit=limit][&include=meta]
// http://tinyurl.com/qzobc69
func (h *Handler) serveSearch(_ http.Header, req *http.Request) (interface{}, error) {
	sp, err := parseSearchParams(req)
	if err != nil {
		return "", err
	}
	// perform query
	results, err := h.store.Search(sp)
	if err != nil {
		return nil, errgo.Notef(err, "error performing search")
	}
	response := params.SearchResponse{
		SearchTime: results.SearchTime,
		Total:      results.Total,
		Results:    make([]params.SearchResult, len(results.Results)),
	}
	run := parallel.NewRun(maxConcurrency)
	var missing int32
	for i, ref := range results.Results {
		i, ref := i, ref
		run.Do(func() error {
			meta, err := h.Router.GetMetadata(ref, sp.Include)
			if err != nil {
				// Unfortunately it is possible to get errors here due to
				// internal inconsistency, so rather than throwing away
				// all the search results, we just log the error and move on.
				logger.Errorf("cannot retrieve metadata for %v: %v", ref, err)
				atomic.AddInt32(&missing, 1)
				return nil
			}
			response.Results[i] = params.SearchResult{
				Id:   ref,
				Meta: meta,
			}
			return nil
		})
	}
	run.Wait()
	if missing == 0 {
		return response, nil
	}
	// We're missing some results - shuffle all the results down to
	// fill the gaps.
	j := 0
	for _, result := range response.Results {
		if result.Id != nil {
			response.Results[j] = result
			j++
		}
	}
	response.Results = response.Results[0:j]
	return response, nil
}

// GET search/interesting[?limit=limit][&include=meta]
// http://tinyurl.com/ntmdrg8
func (h *Handler) serveSearchInteresting(w http.ResponseWriter, req *http.Request) {
	router.WriteError(w, errNotImplemented)
}

// parseSearchParms extracts the search paramaters from the request
func parseSearchParams(req *http.Request) (charmstore.SearchParams, error) {
	sp := charmstore.SearchParams{}
	var err error
	for k, v := range req.Form {
		switch k {
		case "text":
			sp.Text = v[0]
		case "autocomplete":
			sp.AutoComplete, err = parseBool(v[0])
			if err != nil {
				return charmstore.SearchParams{}, badRequestf(err, "invalid autocomplete parameter")
			}
		case "limit":
			sp.Limit, err = strconv.Atoi(v[0])
			if err != nil {
				return charmstore.SearchParams{}, badRequestf(err, "invalid limit parameter: could not parse integer")
			}
			if sp.Limit < 1 {
				return charmstore.SearchParams{}, badRequestf(nil, "invalid limit parameter: expected integer greater than zero")
			}
		case "include":
			for _, s := range v {
				if s != "" {
					sp.Include = append(sp.Include, s)
				}
			}
		case "description", "name", "owner", "provides", "requires", "series", "summary", "tags", "type":
			if sp.Filters == nil {
				sp.Filters = make(map[string][]string)
			}
			sp.Filters[k] = v
		case "skip":
			sp.Skip, err = strconv.Atoi(v[0])
			if err != nil {
				return charmstore.SearchParams{}, badRequestf(err, "invalid skip parameter: could not parse integer")
			}
			if sp.Skip < 0 {
				return charmstore.SearchParams{}, badRequestf(nil, "invalid skip parameter: expected non-negative integer")
			}
		default:
			return charmstore.SearchParams{}, badRequestf(nil, "invalid parameter: %s", k)
		}
	}

	return sp, nil
}
<commit_msg>internal/v4: add forgotten comment from PR #210<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the LGPLv3, see LICENCE file for details.

package v4

import (
	"net/http"
	"strconv"
	"sync/atomic"

	"github.com/juju/utils/parallel"
	"gopkg.in/errgo.v1"

	"github.com/juju/charmstore/internal/charmstore"
	"github.com/juju/charmstore/internal/router"
	"github.com/juju/charmstore/params"
)

const maxConcurrency = 20

// GET search[?text=text][&autocomplete=1][&filter=value…][&limit=limit][&include=meta]
// http://tinyurl.com/qzobc69
func (h *Handler) serveSearch(_ http.Header, req *http.Request) (interface{}, error) {
	sp, err := parseSearchParams(req)
	if err != nil {
		return "", err
	}
	// perform query
	results, err := h.store.Search(sp)
	if err != nil {
		return nil, errgo.Notef(err, "error performing search")
	}
	response := params.SearchResponse{
		SearchTime: results.SearchTime,
		Total:      results.Total,
		Results:    make([]params.SearchResult, len(results.Results)),
	}
	run := parallel.NewRun(maxConcurrency)
	var missing int32
	for i, ref := range results.Results {
		i, ref := i, ref
		run.Do(func() error {
			meta, err := h.Router.GetMetadata(ref, sp.Include)
			if err != nil {
				// Unfortunately it is possible to get errors here due to
				// internal inconsistency, so rather than throwing away
				// all the search results, we just log the error and move on.
				logger.Errorf("cannot retrieve metadata for %v: %v", ref, err)
				atomic.AddInt32(&missing, 1)
				return nil
			}
			response.Results[i] = params.SearchResult{
				Id:   ref,
				Meta: meta,
			}
			return nil
		})
	}
	// We never return an error from the Do function above, so no need to
	// check the error here.
	run.Wait()
	if missing == 0 {
		return response, nil
	}
	// We're missing some results - shuffle all the results down to
	// fill the gaps.
	j := 0
	for _, result := range response.Results {
		if result.Id != nil {
			response.Results[j] = result
			j++
		}
	}
	response.Results = response.Results[0:j]
	return response, nil
}

// GET search/interesting[?limit=limit][&include=meta]
// http://tinyurl.com/ntmdrg8
func (h *Handler) serveSearchInteresting(w http.ResponseWriter, req *http.Request) {
	router.WriteError(w, errNotImplemented)
}

// parseSearchParms extracts the search paramaters from the request
func parseSearchParams(req *http.Request) (charmstore.SearchParams, error) {
	sp := charmstore.SearchParams{}
	var err error
	for k, v := range req.Form {
		switch k {
		case "text":
			sp.Text = v[0]
		case "autocomplete":
			sp.AutoComplete, err = parseBool(v[0])
			if err != nil {
				return charmstore.SearchParams{}, badRequestf(err, "invalid autocomplete parameter")
			}
		case "limit":
			sp.Limit, err = strconv.Atoi(v[0])
			if err != nil {
				return charmstore.SearchParams{}, badRequestf(err, "invalid limit parameter: could not parse integer")
			}
			if sp.Limit < 1 {
				return charmstore.SearchParams{}, badRequestf(nil, "invalid limit parameter: expected integer greater than zero")
			}
		case "include":
			for _, s := range v {
				if s != "" {
					sp.Include = append(sp.Include, s)
				}
			}
		case "description", "name", "owner", "provides", "requires", "series", "summary", "tags", "type":
			if sp.Filters == nil {
				sp.Filters = make(map[string][]string)
			}
			sp.Filters[k] = v
		case "skip":
			sp.Skip, err = strconv.Atoi(v[0])
			if err != nil {
				return charmstore.SearchParams{}, badRequestf(err, "invalid skip parameter: could not parse integer")
			}
			if sp.Skip < 0 {
				return charmstore.SearchParams{}, badRequestf(nil, "invalid skip parameter: expected non-negative integer")
			}
		default:
			return charmstore.SearchParams{}, badRequestf(nil, "invalid parameter: %s", k)
		}
	}

	return sp, nil
}
<|endoftext|>


<commit_before>package algebra

import (
	"testing"
)

func TestNumbers(t *testing.T) {
	testSum(t, "1", 1)
	testSum(t, "-1", -1)
}

func TestAddition(t *testing.T) {
	testSum(t, "1 + 1", 2)
	testSum(t, "100 + 1", 101)

	testSum(t, "1 + -1", 0)
}

func TestSubtraction(t *testing.T) {
	testSum(t, "1 - 1", 0)
	testSum(t, "100 - 1", 99)

	testSum(t, "100 - -1", 101)
}

func TestMultiplication(t *testing.T) {
	testSum(t, "1 * 1", 1)
	testSum(t, "10 * 10", 100)
	testSum(t, "-10 * 10", -100)
}

func TestDivision(t *testing.T) {
	testSum(t, "2 / 1", 2)
	testSum(t, "10 / 2", 5)
	testSum(t, "-100 / 5", -20)
}

func TestBrackets(t *testing.T) {
	testSum(t, "(2 + 2) * 2", 8)
	testSum(t, "(2 - 2) * 1000", 0)
}

func testSum(t *testing.T, source string, expected int) {
	got, err := Evaluate(source)
	if err != nil {
		t.Error("error:", err)
		return
	}

	if got != expected {
		t.Errorf("number error: %v should equal %d, got %d", source, expected, got)
	}
}
<commit_msg>parser_test: simple power parse tests<commit_after>package algebra

import (
	"testing"
)

func TestNumbers(t *testing.T) {
	testSum(t, "1", 1)
	testSum(t, "-1", -1)
}

func TestAddition(t *testing.T) {
	testSum(t, "1 + 1", 2)
	testSum(t, "100 + 1", 101)

	testSum(t, "1 + -1", 0)
}

func TestSubtraction(t *testing.T) {
	testSum(t, "1 - 1", 0)
	testSum(t, "100 - 1", 99)

	testSum(t, "100 - -1", 101)
}

func TestMultiplication(t *testing.T) {
	testSum(t, "1 * 1", 1)
	testSum(t, "10 * 10", 100)
	testSum(t, "-10 * 10", -100)
}

func TestDivision(t *testing.T) {
	testSum(t, "2 / 1", 2)
	testSum(t, "10 / 2", 5)
	testSum(t, "-100 / 5", -20)
}

func TestBrackets(t *testing.T) {
	testSum(t, "(2 + 2) * 2", 8)
	testSum(t, "(2 - 2) * 1000", 0)
}

func TestPowers(t *testing.T) {
	testSum(t, "4^2", 16)
	testSum(t, "4^4", 256)
	testSum(t, "9^2", 81)
	testSum(t, "9^-2", 0)
}

func testSum(t *testing.T, source string, expected int) {
	got, err := Evaluate(source)
	if err != nil {
		t.Error("error:", err)
		return
	}

	if got != expected {
		t.Errorf("number error: %v should equal %d, got %d", source, expected, got)
	}
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package indexer

import (
	"errors"
	"sync/atomic"
	"time"
	"unsafe"
)

//MutationQueue interface specifies methods which a mutation queue for indexer
//needs to implement
type MutationQueue interface {

	//enqueue a mutation reference based on vbucket
	Enqueue(mutation *MutationKeys, vbucket Vbucket) error

	//dequeue a vbucket's mutation and keep sending on a channel until stop signal
	Dequeue(vbucket Vbucket) (<-chan *MutationKeys, chan<- bool, error)
	//dequeue a vbucket's mutation upto seqno(wait if not available)
	DequeueUptoSeqno(vbucket Vbucket, seqno Seqno) (<-chan *MutationKeys, error)
	//dequeue single element for a vbucket and return
	DequeueSingleElement(vbucket Vbucket) *MutationKeys

	//return reference to a vbucket's mutation at Tail of queue without dequeue
	PeekTail(vbucket Vbucket) *MutationKeys
	//return reference to a vbucket's mutation at Head of queue without dequeue
	PeekHead(vbucket Vbucket) *MutationKeys

	//return size of queue per vbucket
	GetSize(vbucket Vbucket) int64

	//returns the numbers of vbuckets for the queue
	GetNumVbuckets() uint16
}

//AtomicMutationQueue is a lock-free multi-queue with internal queue per
//vbucket for storing mutation references. This is loosely based on
//http://www.drdobbs.com/parallel/writing-lock-free-code-a-corrected-queue/210604448?pgno=1
//with the main difference being that free nodes are being reused here to reduce GC.
//
//It doesn't copy the mutation and its caller's responsiblity
//to allocate/deallocate KeyVersions struct. A mutation which is currently in queue
//shouldn't be freed.
//
//This implementation uses Go "atomic" pkg to provide safe concurrent access
//for a single reader and writer per vbucket queue without using mutex locks.
//
//It provides safe concurrent read/write access across vbucket queues.

type atomicMutationQueue struct {
	head        []unsafe.Pointer //head pointer per vbucket queue
	tail        []unsafe.Pointer //tail pointer per vbucket queue
	free        []*node          //free pointer per vbucket queue
	numVbuckets uint16           //num vbuckets for the queue
	size        []int64          //size of queue per vbucket
}

//NewAtomicMutationQueue allocates a new Atomic Mutation Queue and initializes it
func NewAtomicMutationQueue(numVbuckets uint16) *atomicMutationQueue {

	q := &atomicMutationQueue{head: make([]unsafe.Pointer, numVbuckets),
		tail:        make([]unsafe.Pointer, numVbuckets),
		free:        make([]*node, numVbuckets),
		size:        make([]int64, numVbuckets),
		numVbuckets: numVbuckets,
	}

	var x uint16
	for x = 0; x < numVbuckets; x++ {
		node := &node{} //sentinel node for the queue
		q.head[x] = unsafe.Pointer(node)
		q.tail[x] = unsafe.Pointer(node)
		q.free[x] = node
	}

	return q

}

//Node represents a single element in the queue
type node struct {
	mutation *MutationKeys
	next     *node
}

//Poll Interval for dequeue thread
const DEQUEUE_POLL_INTERVAL = 5

//Enqueue will enqueue the mutation reference for given vbucket.
//Caller should not free the mutation till it is dequeued.
//Mutation will not be copied internally by the queue.
func (q *atomicMutationQueue) Enqueue(mutation *MutationKeys, vbucket Vbucket) error {

	if vbucket < 0 || vbucket > Vbucket(q.numVbuckets)-1 {
		return errors.New("vbucket out of range")
	}

	//create a new node
	n := q.allocNode(vbucket)
	n.mutation = mutation
	n.next = nil

	//point tail's next to new node
	tail := (*node)(atomic.LoadPointer(&q.tail[vbucket]))
	tail.next = n
	//update tail to new node
	atomic.StorePointer(&q.tail[vbucket], unsafe.Pointer(tail.next))

	atomic.AddInt64(&q.size[vbucket], 1)

	return nil

}

//DequeueUptoSeqno returns a channel on which it will return mutation reference
//for specified vbucket upto the sequence number specified.
//This function will keep polling till mutations upto seqno are available
//to be sent. It terminates when it finds a mutation with seqno higher than
//the one specified as argument. This allow for multiple mutations with same
//seqno (e.g. in case of multiple indexes)
//It closes the mutation channel to indicate its done.
func (q *atomicMutationQueue) DequeueUptoSeqno(vbucket Vbucket, seqno Seqno) (
	<-chan *MutationKeys, error) {

	datach := make(chan *MutationKeys)

	go q.dequeueUptoSeqno(vbucket, seqno, datach)

	return datach, nil

}

func (q *atomicMutationQueue) dequeueUptoSeqno(vbucket Vbucket, seqno Seqno,
	datach chan *MutationKeys) {

	//every DEQUEUE_POLL_INTERVAL milliseconds, check for new mutations
	ticker := time.NewTicker(time.Millisecond * DEQUEUE_POLL_INTERVAL)

	dequeueCount := 0

	for _ = range ticker.C {
		for atomic.LoadPointer(&q.head[vbucket]) !=
			atomic.LoadPointer(&q.tail[vbucket]) { //if queue is nonempty

			head := (*node)(atomic.LoadPointer(&q.head[vbucket]))
			//copy the mutation pointer
			m := head.next.mutation
			if seqno >= m.meta.seqno {
				//free mutation pointer
				head.next.mutation = nil
				//move head to next
				atomic.StorePointer(&q.head[vbucket], unsafe.Pointer(head.next))
				atomic.AddInt64(&q.size[vbucket], -1)
				//send mutation to caller
				datach <- m
				dequeueCount++
			}

			//once the seqno is reached, close the channel
			if seqno <= m.meta.seqno {
				ticker.Stop()
				close(datach)
				return
			}
		}
		//if this list is empty, there is no seqno to compare and determine if
		//new mutations are going to arrive. For now, return assuming the queue should
		//have mutations before flush.
		//TODO: make this an input parameter if this call should block and wait in case
		//of empty queue or not
		if dequeueCount == 0 {
			ticker.Stop()
			close(datach)
			return
		}
	}
}

//Dequeue returns a channel on which it will return mutation reference for specified vbucket.
//This function will keep polling and send mutations as those become available.
//It returns a stop channel on which caller can signal it to stop.
func (q *atomicMutationQueue) Dequeue(vbucket Vbucket) (<-chan *MutationKeys,
	chan<- bool, error) {

	datach := make(chan *MutationKeys)
	stopch := make(chan bool)

	//every DEQUEUE_POLL_INTERVAL milliseconds, check for new mutations
	ticker := time.NewTicker(time.Millisecond * DEQUEUE_POLL_INTERVAL)

	go func() {
		for {
			select {
			case <-ticker.C:
				q.dequeue(vbucket, datach)
			case <-stopch:
				ticker.Stop()
				close(datach)
				return
			}
		}
	}()

	return datach, stopch, nil

}

func (q *atomicMutationQueue) dequeue(vbucket Vbucket, datach chan *MutationKeys) {

	//keep dequeuing till list is empty
	for {
		m := q.DequeueSingleElement(vbucket)
		if m == nil {
			return
		}
		//send mutation to caller
		datach <- m
	}

}

//DequeueSingleElement dequeues a single element and returns.
//Returns nil in case of empty queue.
func (q *atomicMutationQueue) DequeueSingleElement(vbucket Vbucket) *MutationKeys {

	if atomic.LoadPointer(&q.head[vbucket]) !=
		atomic.LoadPointer(&q.tail[vbucket]) { //if queue is nonempty

		head := (*node)(atomic.LoadPointer(&q.head[vbucket]))
		//copy the mutation pointer
		m := head.next.mutation
		//move head to next
		atomic.StorePointer(&q.head[vbucket], unsafe.Pointer(head.next))
		atomic.AddInt64(&q.size[vbucket], -1)
		return m
	}
	return nil
}

//PeekTail returns reference to a vbucket's mutation at tail of queue without dequeue
func (q *atomicMutationQueue) PeekTail(vbucket Vbucket) *MutationKeys {
	if atomic.LoadPointer(&q.head[vbucket]) !=
		atomic.LoadPointer(&q.tail[vbucket]) { //if queue is nonempty
		tail := (*node)(atomic.LoadPointer(&q.tail[vbucket]))
		return tail.mutation
	}
	return nil
}

//PeekHead returns reference to a vbucket's mutation at head of queue without dequeue
func (q *atomicMutationQueue) PeekHead(vbucket Vbucket) *MutationKeys {
	if atomic.LoadPointer(&q.head[vbucket]) !=
		atomic.LoadPointer(&q.tail[vbucket]) { //if queue is nonempty
		head := (*node)(atomic.LoadPointer(&q.head[vbucket]))
		return head.mutation
	}
	return nil
}

//GetSize returns the size of the vbucket queue
func (q *atomicMutationQueue) GetSize(vbucket Vbucket) int64 {
	return atomic.LoadInt64(&q.size[vbucket])
}

//GetNumVbuckets returns the numbers of vbuckets for the queue
func (q *atomicMutationQueue) GetNumVbuckets() uint16 {
	return q.numVbuckets
}

//allocNode tries to get node from freelist, otherwise allocates a new node and returns
func (q *atomicMutationQueue) allocNode(vbucket Vbucket) *node {

	//get node from freelist
	n, err := q.popFreeList(vbucket)
	if err == nil {
		return n
	} else {
		//allocate new node and return
		return &node{}
	}

}

//popFreeList removes a node from freelist and returns to caller.
//if freelist is empty, it returns nil.
func (q *atomicMutationQueue) popFreeList(vbucket Vbucket) (*node, error) {

	if q.free[vbucket] != (*node)(atomic.LoadPointer(&q.head[vbucket])) {
		n := q.free[vbucket]
		q.free[vbucket] = q.free[vbucket].next
		n.mutation = nil
		n.next = nil
		return n, nil
	} else {
		return nil, errors.New("Free List Empty")
	}

}
<commit_msg>MB-14460 Mutation Queue Dequeue Logic Change<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package indexer

import (
	"errors"
	"sync/atomic"
	"time"
	"unsafe"
)

//MutationQueue interface specifies methods which a mutation queue for indexer
//needs to implement
type MutationQueue interface {

	//enqueue a mutation reference based on vbucket
	Enqueue(mutation *MutationKeys, vbucket Vbucket) error

	//dequeue a vbucket's mutation and keep sending on a channel until stop signal
	Dequeue(vbucket Vbucket) (<-chan *MutationKeys, chan<- bool, error)
	//dequeue a vbucket's mutation upto seqno(wait if not available)
	DequeueUptoSeqno(vbucket Vbucket, seqno Seqno) (<-chan *MutationKeys, error)
	//dequeue single element for a vbucket and return
	DequeueSingleElement(vbucket Vbucket) *MutationKeys

	//return reference to a vbucket's mutation at Tail of queue without dequeue
	PeekTail(vbucket Vbucket) *MutationKeys
	//return reference to a vbucket's mutation at Head of queue without dequeue
	PeekHead(vbucket Vbucket) *MutationKeys

	//return size of queue per vbucket
	GetSize(vbucket Vbucket) int64

	//returns the numbers of vbuckets for the queue
	GetNumVbuckets() uint16
}

//AtomicMutationQueue is a lock-free multi-queue with internal queue per
//vbucket for storing mutation references. This is loosely based on
//http://www.drdobbs.com/parallel/writing-lock-free-code-a-corrected-queue/210604448?pgno=1
//with the main difference being that free nodes are being reused here to reduce GC.
//
//It doesn't copy the mutation and its caller's responsiblity
//to allocate/deallocate KeyVersions struct. A mutation which is currently in queue
//shouldn't be freed.
//
//This implementation uses Go "atomic" pkg to provide safe concurrent access
//for a single reader and writer per vbucket queue without using mutex locks.
//
//It provides safe concurrent read/write access across vbucket queues.

type atomicMutationQueue struct {
	head        []unsafe.Pointer //head pointer per vbucket queue
	tail        []unsafe.Pointer //tail pointer per vbucket queue
	free        []*node          //free pointer per vbucket queue
	numVbuckets uint16           //num vbuckets for the queue
	size        []int64          //size of queue per vbucket
}

//NewAtomicMutationQueue allocates a new Atomic Mutation Queue and initializes it
func NewAtomicMutationQueue(numVbuckets uint16) *atomicMutationQueue {

	q := &atomicMutationQueue{head: make([]unsafe.Pointer, numVbuckets),
		tail:        make([]unsafe.Pointer, numVbuckets),
		free:        make([]*node, numVbuckets),
		size:        make([]int64, numVbuckets),
		numVbuckets: numVbuckets,
	}

	var x uint16
	for x = 0; x < numVbuckets; x++ {
		node := &node{} //sentinel node for the queue
		q.head[x] = unsafe.Pointer(node)
		q.tail[x] = unsafe.Pointer(node)
		q.free[x] = node
	}

	return q

}

//Node represents a single element in the queue
type node struct {
	mutation *MutationKeys
	next     *node
}

//Poll Interval for dequeue thread
const DEQUEUE_POLL_INTERVAL = 5

//Enqueue will enqueue the mutation reference for given vbucket.
//Caller should not free the mutation till it is dequeued.
//Mutation will not be copied internally by the queue.
func (q *atomicMutationQueue) Enqueue(mutation *MutationKeys, vbucket Vbucket) error {

	if vbucket < 0 || vbucket > Vbucket(q.numVbuckets)-1 {
		return errors.New("vbucket out of range")
	}

	//create a new node
	n := q.allocNode(vbucket)
	n.mutation = mutation
	n.next = nil

	//point tail's next to new node
	tail := (*node)(atomic.LoadPointer(&q.tail[vbucket]))
	tail.next = n
	//update tail to new node
	atomic.StorePointer(&q.tail[vbucket], unsafe.Pointer(tail.next))

	atomic.AddInt64(&q.size[vbucket], 1)

	return nil

}

//DequeueUptoSeqno returns a channel on which it will return mutation reference
//for specified vbucket upto the sequence number specified.
//This function will keep polling till mutations upto seqno are available
//to be sent. It terminates when it finds a mutation with seqno higher than
//the one specified as argument. This allow for multiple mutations with same
//seqno (e.g. in case of multiple indexes)
//It closes the mutation channel to indicate its done.
func (q *atomicMutationQueue) DequeueUptoSeqno(vbucket Vbucket, seqno Seqno) (
	<-chan *MutationKeys, error) {

	datach := make(chan *MutationKeys)

	go q.dequeueUptoSeqno(vbucket, seqno, datach)

	return datach, nil

}

func (q *atomicMutationQueue) dequeueUptoSeqno(vbucket Vbucket, seqno Seqno,
	datach chan *MutationKeys) {

	//every DEQUEUE_POLL_INTERVAL milliseconds, check for new mutations
	ticker := time.NewTicker(time.Millisecond * DEQUEUE_POLL_INTERVAL)

	var dequeueSeq Seqno

	for _ = range ticker.C {
		for atomic.LoadPointer(&q.head[vbucket]) !=
			atomic.LoadPointer(&q.tail[vbucket]) { //if queue is nonempty

			head := (*node)(atomic.LoadPointer(&q.head[vbucket]))
			//copy the mutation pointer
			m := head.next.mutation
			if seqno >= m.meta.seqno {
				//free mutation pointer
				head.next.mutation = nil
				//move head to next
				atomic.StorePointer(&q.head[vbucket], unsafe.Pointer(head.next))
				atomic.AddInt64(&q.size[vbucket], -1)
				//send mutation to caller
				dequeueSeq = m.meta.seqno
				datach <- m
			}

			//once the seqno is reached, close the channel
			if seqno <= dequeueSeq {
				ticker.Stop()
				close(datach)
				return
			}
		}
	}
}

//Dequeue returns a channel on which it will return mutation reference for specified vbucket.
//This function will keep polling and send mutations as those become available.
//It returns a stop channel on which caller can signal it to stop.
func (q *atomicMutationQueue) Dequeue(vbucket Vbucket) (<-chan *MutationKeys,
	chan<- bool, error) {

	datach := make(chan *MutationKeys)
	stopch := make(chan bool)

	//every DEQUEUE_POLL_INTERVAL milliseconds, check for new mutations
	ticker := time.NewTicker(time.Millisecond * DEQUEUE_POLL_INTERVAL)

	go func() {
		for {
			select {
			case <-ticker.C:
				q.dequeue(vbucket, datach)
			case <-stopch:
				ticker.Stop()
				close(datach)
				return
			}
		}
	}()

	return datach, stopch, nil

}

func (q *atomicMutationQueue) dequeue(vbucket Vbucket, datach chan *MutationKeys) {

	//keep dequeuing till list is empty
	for {
		m := q.DequeueSingleElement(vbucket)
		if m == nil {
			return
		}
		//send mutation to caller
		datach <- m
	}

}

//DequeueSingleElement dequeues a single element and returns.
//Returns nil in case of empty queue.
func (q *atomicMutationQueue) DequeueSingleElement(vbucket Vbucket) *MutationKeys {

	if atomic.LoadPointer(&q.head[vbucket]) !=
		atomic.LoadPointer(&q.tail[vbucket]) { //if queue is nonempty

		head := (*node)(atomic.LoadPointer(&q.head[vbucket]))
		//copy the mutation pointer
		m := head.next.mutation
		//move head to next
		atomic.StorePointer(&q.head[vbucket], unsafe.Pointer(head.next))
		atomic.AddInt64(&q.size[vbucket], -1)
		return m
	}
	return nil
}

//PeekTail returns reference to a vbucket's mutation at tail of queue without dequeue
func (q *atomicMutationQueue) PeekTail(vbucket Vbucket) *MutationKeys {
	if atomic.LoadPointer(&q.head[vbucket]) !=
		atomic.LoadPointer(&q.tail[vbucket]) { //if queue is nonempty
		tail := (*node)(atomic.LoadPointer(&q.tail[vbucket]))
		return tail.mutation
	}
	return nil
}

//PeekHead returns reference to a vbucket's mutation at head of queue without dequeue
func (q *atomicMutationQueue) PeekHead(vbucket Vbucket) *MutationKeys {
	if atomic.LoadPointer(&q.head[vbucket]) !=
		atomic.LoadPointer(&q.tail[vbucket]) { //if queue is nonempty
		head := (*node)(atomic.LoadPointer(&q.head[vbucket]))
		return head.mutation
	}
	return nil
}

//GetSize returns the size of the vbucket queue
func (q *atomicMutationQueue) GetSize(vbucket Vbucket) int64 {
	return atomic.LoadInt64(&q.size[vbucket])
}

//GetNumVbuckets returns the numbers of vbuckets for the queue
func (q *atomicMutationQueue) GetNumVbuckets() uint16 {
	return q.numVbuckets
}

//allocNode tries to get node from freelist, otherwise allocates a new node and returns
func (q *atomicMutationQueue) allocNode(vbucket Vbucket) *node {

	//get node from freelist
	n, err := q.popFreeList(vbucket)
	if err == nil {
		return n
	} else {
		//allocate new node and return
		return &node{}
	}

}

//popFreeList removes a node from freelist and returns to caller.
//if freelist is empty, it returns nil.
func (q *atomicMutationQueue) popFreeList(vbucket Vbucket) (*node, error) {

	if q.free[vbucket] != (*node)(atomic.LoadPointer(&q.head[vbucket])) {
		n := q.free[vbucket]
		q.free[vbucket] = q.free[vbucket].next
		n.mutation = nil
		n.next = nil
		return n, nil
	} else {
		return nil, errors.New("Free List Empty")
	}

}
<|endoftext|>


<commit_before>/*
 * Copyright 2017 Dgraph Labs, Inc. and Contributors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package badger

import (
	"bytes"
	"io/ioutil"
	"math/rand"
	"sync/atomic"

	"github.com/dgraph-io/badger/table"
	"github.com/dgraph-io/badger/y"
	"github.com/pkg/errors"
)

// summary is produced when DB is closed. Currently it is used only for testing.
type summary struct {
	fileIDs map[uint64]bool
}

func (s *levelsController) getSummary() *summary {
	out := &summary{
		fileIDs: make(map[uint64]bool),
	}
	for _, l := range s.levels {
		l.getSummary(out)
	}
	return out
}

func (s *levelHandler) getSummary(sum *summary) {
	s.RLock()
	defer s.RUnlock()
	for _, t := range s.tables {
		sum.fileIDs[t.ID()] = true
	}
}

func (s *KV) validate() { s.lc.validate() }

func (s *levelsController) validate() error {
	for _, l := range s.levels {
		if err := l.validate(); err != nil {
			return errors.Wrap(err, "Levels Controller")
		}
	}
	return nil
}

// Check does some sanity check on one level of data or in-memory index.
func (s *levelHandler) validate() error {
	if s.level == 0 {
		return nil
	}

	s.RLock()
	defer s.RUnlock()
	numTables := len(s.tables)
	for j := 1; j < numTables; j++ {
		if j >= len(s.tables) {
			return errors.Errorf("Level %d, j=%d numTables=%d", s.level, j, numTables)
		}

		if bytes.Compare(s.tables[j-1].Biggest(), s.tables[j].Smallest()) >= 0 {
			return errors.Errorf(
				"Inter: %s vs %s: level=%d j=%d numTables=%d",
				string(s.tables[j-1].Biggest()), string(s.tables[j].Smallest()), s.level, j, numTables)
		}

		if bytes.Compare(s.tables[j].Smallest(), s.tables[j].Biggest()) > 0 {
			return errors.Errorf(
				"Intra: %q vs %q: level=%d j=%d numTables=%d",
				s.tables[j].Smallest(), s.tables[j].Biggest(), s.level, j, numTables)
		}
	}
	return nil
}

// func (s *KV) debugPrintMore() { s.lc.debugPrintMore() }

// // debugPrintMore shows key ranges of each level.
// func (s *levelsController) debugPrintMore() {
// 	s.Lock()
// 	defer s.Unlock()
// 	for i := 0; i < s.kv.opt.MaxLevels; i++ {
// 		s.levels[i].debugPrintMore()
// 	}
// }

// func (s *levelHandler) debugPrintMore() {
// 	s.RLock()
// 	defer s.RUnlock()
// 	s.elog.Printf("Level %d:", s.level)
// 	for _, t := range s.tables {
// 		y.Printf(" [%s, %s]", t.Smallest(), t.Biggest())
// 	}
// 	y.Printf("\n")
// }

// reserveFileIDs reserve k fileIDs. Returns pair is a half-interval.
// If we return [3, 6), it means use 3, 4, 5.
func (s *levelsController) reserveFileIDs(k int) (uint64, uint64) {
	id := atomic.AddUint64(&s.maxFileID, uint64(k))
	return id - uint64(k), id
}

func getIDMap(dir string) map[uint64]struct{} {
	fileInfos, err := ioutil.ReadDir(dir)
	y.Check(err)
	idMap := make(map[uint64]struct{})
	for _, info := range fileInfos {
		if info.IsDir() {
			continue
		}
		fileID, ok := table.ParseFileID(info.Name())
		if !ok {
			continue
		}
		idMap[fileID] = struct{}{}
	}
	return idMap
}

// mod65535 mods by 65535 fast.
func mod65535(a uint32) uint32 {
	a = (a >> 16) + (a & 0xFFFF) /* sum base 2**16 digits */
	if a < 65535 {
		return a
	}
	if a < (2 * 65535) {
		return a - 65535
	}
	return a - (2 * 65535)
}

func newCASCounter() uint16 {
	return uint16(1 + mod65535(rand.Uint32()))
}
<commit_msg>Seed rand, so that it generates new CAS counters every run.<commit_after>/*
 * Copyright 2017 Dgraph Labs, Inc. and Contributors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package badger

import (
	"bytes"
	"io/ioutil"
	"math/rand"
	"sync/atomic"
	"time"

	"github.com/dgraph-io/badger/table"
	"github.com/dgraph-io/badger/y"
	"github.com/pkg/errors"
)

// summary is produced when DB is closed. Currently it is used only for testing.
type summary struct {
	fileIDs map[uint64]bool
}

func (s *levelsController) getSummary() *summary {
	out := &summary{
		fileIDs: make(map[uint64]bool),
	}
	for _, l := range s.levels {
		l.getSummary(out)
	}
	return out
}

func (s *levelHandler) getSummary(sum *summary) {
	s.RLock()
	defer s.RUnlock()
	for _, t := range s.tables {
		sum.fileIDs[t.ID()] = true
	}
}

func (s *KV) validate() { s.lc.validate() }

func (s *levelsController) validate() error {
	for _, l := range s.levels {
		if err := l.validate(); err != nil {
			return errors.Wrap(err, "Levels Controller")
		}
	}
	return nil
}

// Check does some sanity check on one level of data or in-memory index.
func (s *levelHandler) validate() error {
	if s.level == 0 {
		return nil
	}

	s.RLock()
	defer s.RUnlock()
	numTables := len(s.tables)
	for j := 1; j < numTables; j++ {
		if j >= len(s.tables) {
			return errors.Errorf("Level %d, j=%d numTables=%d", s.level, j, numTables)
		}

		if bytes.Compare(s.tables[j-1].Biggest(), s.tables[j].Smallest()) >= 0 {
			return errors.Errorf(
				"Inter: %s vs %s: level=%d j=%d numTables=%d",
				string(s.tables[j-1].Biggest()), string(s.tables[j].Smallest()), s.level, j, numTables)
		}

		if bytes.Compare(s.tables[j].Smallest(), s.tables[j].Biggest()) > 0 {
			return errors.Errorf(
				"Intra: %q vs %q: level=%d j=%d numTables=%d",
				s.tables[j].Smallest(), s.tables[j].Biggest(), s.level, j, numTables)
		}
	}
	return nil
}

// func (s *KV) debugPrintMore() { s.lc.debugPrintMore() }

// // debugPrintMore shows key ranges of each level.
// func (s *levelsController) debugPrintMore() {
// 	s.Lock()
// 	defer s.Unlock()
// 	for i := 0; i < s.kv.opt.MaxLevels; i++ {
// 		s.levels[i].debugPrintMore()
// 	}
// }

// func (s *levelHandler) debugPrintMore() {
// 	s.RLock()
// 	defer s.RUnlock()
// 	s.elog.Printf("Level %d:", s.level)
// 	for _, t := range s.tables {
// 		y.Printf(" [%s, %s]", t.Smallest(), t.Biggest())
// 	}
// 	y.Printf("\n")
// }

// reserveFileIDs reserve k fileIDs. Returns pair is a half-interval.
// If we return [3, 6), it means use 3, 4, 5.
func (s *levelsController) reserveFileIDs(k int) (uint64, uint64) {
	id := atomic.AddUint64(&s.maxFileID, uint64(k))
	return id - uint64(k), id
}

func getIDMap(dir string) map[uint64]struct{} {
	fileInfos, err := ioutil.ReadDir(dir)
	y.Check(err)
	idMap := make(map[uint64]struct{})
	for _, info := range fileInfos {
		if info.IsDir() {
			continue
		}
		fileID, ok := table.ParseFileID(info.Name())
		if !ok {
			continue
		}
		idMap[fileID] = struct{}{}
	}
	return idMap
}

// mod65535 mods by 65535 fast.
func mod65535(a uint32) uint32 {
	a = (a >> 16) + (a & 0xFFFF) /* sum base 2**16 digits */
	if a < 65535 {
		return a
	}
	if a < (2 * 65535) {
		return a - 65535
	}
	return a - (2 * 65535)
}

func newCASCounter() uint16 {
	return uint16(1 + mod65535(rand.Uint32()))
}

func init() {
	rand.Seed(time.Now().UnixNano())
}
<|endoftext|>


<commit_before>package httpToFirehose

import (
	"encoding/json"
	firehosePool "github.com/gabrielperezs/streamspooler/firehose"
	"log"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/fasthttp/router"
	"github.com/gallir/smart-relayer/lib"
	"github.com/valyala/fasthttp"
)

// Server is the thread that listen for clients' connections
type Server struct {
	sync.Mutex
	config  lib.RelayerConfig
	done    chan bool
	exiting bool
	engine  *fasthttp.Server

	fh             *firehosePool.Server
	lastConnection time.Time
	lastError      time.Time
}

var (
	defaultTimeout          = 5 * time.Second
	defaultMaxResults int64 = 100
)

// New creates a new http local server
func New(c lib.RelayerConfig, done chan bool) (*Server, error) {
	srv := &Server{
		done: done,
	}

	srv.Reload(&c)

	return srv, nil
}

// Reload the configuration
func (srv *Server) Reload(c *lib.RelayerConfig) (err error) {
	srv.Lock()
	defer srv.Unlock()

	srv.config = *c

	fhConfig := firehosePool.Config{
		Profile:       srv.config.Profile,
		Region:        srv.config.Region,
		StreamName:    srv.config.StreamName,
		MaxWorkers:    srv.config.MaxConnections,
		MaxRecords:    srv.config.MaxRecords,
		Buffer:        srv.config.Buffer,
		ConcatRecords: srv.config.Concat,
		Critical:      srv.config.Critical,
	}

	if srv.fh == nil {
		srv.fh = firehosePool.New(fhConfig)
	} else {
		go srv.fh.Reload(&fhConfig)
	}

	return nil
}

// Start accepts incoming connections on the Listener
func (srv *Server) Start() (e error) {
	srv.Lock()
	defer srv.Unlock()

	if srv.engine != nil {
		return nil
	}

	r := router.New()
	r.GET("/ping", srv.ok)
	r.POST("/firehose-raw", srv.submitRaw)
	r.POST("/firehose", srv.submit)

	srv.engine = &fasthttp.Server{
		Handler: r.Handler,
	}

	go func() {
		switch {
		case strings.HasPrefix(srv.config.Listen, "tcp://"):
			e = srv.engine.ListenAndServe(srv.config.Listen[6:])
		case strings.HasPrefix(srv.config.Listen, "unix://"):
			e = srv.engine.ListenAndServeUNIX(srv.config.Listen[7:], os.FileMode(0666))
		}
	}()
	return e
}

// Exit closes the listener and send done to main
func (srv *Server) Exit() {
	srv.exiting = true

	if srv.engine != nil {
		if e := srv.engine.Shutdown(); e != nil {
			log.Println("httpToFirehose ERROR: shutting down failed", e)
		}
	}

	// finishing the server
	srv.done <- true
}

func (srv *Server) ok(ctx *fasthttp.RequestCtx) {
	ctx.SetContentType("application/json")
	ctx.SetStatusCode(fasthttp.StatusOK)
	ctx.SetBody([]byte("{\"status\": \"success\"}"))
}

func (srv *Server) submitRaw(ctx *fasthttp.RequestCtx) {
	srv.sendBytes(ctx.Request.Body()) // Does not check if the data is a valid json.

	ctx.SetContentType("application/json")
	ctx.SetStatusCode(fasthttp.StatusOK)
	ctx.SetBody([]byte("{\"status\": \"success\"}"))
}

func (srv *Server) submit(ctx *fasthttp.RequestCtx) {
	ctx.SetContentType("application/json")

	var parsedJson map[string]interface{}
	err := json.Unmarshal(ctx.Request.Body(), &parsedJson)
	if err != nil {
		ctx.SetStatusCode(fasthttp.StatusBadRequest)
		ctx.SetBody([]byte("{\"status\": \"invalid_json\"}"))
		return
	}

	var rowWithTimestamp = lib.NewInterRecord()
	rowWithTimestamp.SetData(parsedJson)
	srv.sendRecord(rowWithTimestamp)

	ctx.SetStatusCode(fasthttp.StatusOK)
	ctx.SetBody([]byte("{\"status\": \"success\"}"))
}

func (srv *Server) sendRecord(r *lib.InterRecord) {
	if srv.exiting {
		return
	}

	// It blocks until the message can be delivered, for critical logs
	if srv.config.Critical || srv.config.Mode != "smart" {
		srv.fh.C <- r.Bytes()
		return
	}

	select {
	case srv.fh.C <- r.Bytes():
	default:
		log.Printf("Firehose: channel is full, discarded. Queued messages %d", len(srv.fh.C))
	}
}

func (srv *Server) sendBytes(b []byte) {
	r := lib.NewInterRecord()
	r.Types = 1
	r.Raw = b
	srv.sendRecord(r)
}
<commit_msg>Added exit for firehose when exiting<commit_after>package httpToFirehose

import (
	"encoding/json"
	firehosePool "github.com/gabrielperezs/streamspooler/firehose"
	"log"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/fasthttp/router"
	"github.com/gallir/smart-relayer/lib"
	"github.com/valyala/fasthttp"
)

// Server is the thread that listen for clients' connections
type Server struct {
	sync.Mutex
	config  lib.RelayerConfig
	done    chan bool
	exiting bool
	engine  *fasthttp.Server

	fh             *firehosePool.Server
	lastConnection time.Time
	lastError      time.Time
}

var (
	defaultTimeout          = 5 * time.Second
	defaultMaxResults int64 = 100
)

// New creates a new http local server
func New(c lib.RelayerConfig, done chan bool) (*Server, error) {
	srv := &Server{
		done: done,
	}

	srv.Reload(&c)

	return srv, nil
}

// Reload the configuration
func (srv *Server) Reload(c *lib.RelayerConfig) (err error) {
	srv.Lock()
	defer srv.Unlock()

	srv.config = *c

	fhConfig := firehosePool.Config{
		Profile:       srv.config.Profile,
		Region:        srv.config.Region,
		StreamName:    srv.config.StreamName,
		MaxWorkers:    srv.config.MaxConnections,
		MaxRecords:    srv.config.MaxRecords,
		Buffer:        srv.config.Buffer,
		ConcatRecords: srv.config.Concat,
		Critical:      srv.config.Critical,
	}

	if srv.fh == nil {
		srv.fh = firehosePool.New(fhConfig)
	} else {
		go srv.fh.Reload(&fhConfig)
	}

	return nil
}

// Start accepts incoming connections on the Listener
func (srv *Server) Start() (e error) {
	srv.Lock()
	defer srv.Unlock()

	if srv.engine != nil {
		return nil
	}

	r := router.New()
	r.GET("/ping", srv.ok)
	r.POST("/firehose-raw", srv.submitRaw)
	r.POST("/firehose", srv.submit)

	srv.engine = &fasthttp.Server{
		Handler: r.Handler,
	}

	go func() {
		switch {
		case strings.HasPrefix(srv.config.Listen, "tcp://"):
			e = srv.engine.ListenAndServe(srv.config.Listen[6:])
		case strings.HasPrefix(srv.config.Listen, "unix://"):
			e = srv.engine.ListenAndServeUNIX(srv.config.Listen[7:], os.FileMode(0666))
		}
	}()
	return e
}

// Exit closes the listener and send done to main
func (srv *Server) Exit() {
	srv.exiting = true

	if srv.engine != nil {
		if e := srv.engine.Shutdown(); e != nil {
			log.Println("httpToFirehose ERROR: shutting down failed", e)
		}
	}

	if srv.fh != nil {
		srv.fh.Exit()
	}

	// finishing the server
	srv.done <- true
}

func (srv *Server) ok(ctx *fasthttp.RequestCtx) {
	ctx.SetContentType("application/json")
	ctx.SetStatusCode(fasthttp.StatusOK)
	ctx.SetBody([]byte("{\"status\": \"success\"}"))
}

func (srv *Server) submitRaw(ctx *fasthttp.RequestCtx) {
	srv.sendBytes(ctx.Request.Body()) // Does not check if the data is a valid json.

	ctx.SetContentType("application/json")
	ctx.SetStatusCode(fasthttp.StatusOK)
	ctx.SetBody([]byte("{\"status\": \"success\"}"))
}

func (srv *Server) submit(ctx *fasthttp.RequestCtx) {
	ctx.SetContentType("application/json")

	var parsedJson map[string]interface{}
	err := json.Unmarshal(ctx.Request.Body(), &parsedJson)
	if err != nil {
		ctx.SetStatusCode(fasthttp.StatusBadRequest)
		ctx.SetBody([]byte("{\"status\": \"invalid_json\"}"))
		return
	}

	var rowWithTimestamp = lib.NewInterRecord()
	rowWithTimestamp.SetData(parsedJson)
	srv.sendRecord(rowWithTimestamp)

	ctx.SetStatusCode(fasthttp.StatusOK)
	ctx.SetBody([]byte("{\"status\": \"success\"}"))
}

func (srv *Server) sendRecord(r *lib.InterRecord) {
	if srv.exiting {
		return
	}

	// It blocks until the message can be delivered, for critical logs
	if srv.config.Critical || srv.config.Mode != "smart" {
		srv.fh.C <- r.Bytes()
		return
	}

	select {
	case srv.fh.C <- r.Bytes():
	default:
		log.Printf("Firehose: channel is full, discarded. Queued messages %d", len(srv.fh.C))
	}
}

func (srv *Server) sendBytes(b []byte) {
	r := lib.NewInterRecord()
	r.Types = 1
	r.Raw = b
	srv.sendRecord(r)
}
<|endoftext|>


<commit_before>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build go1.8

package httpreplay_test

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"os"
	"testing"
	"time"

	"cloud.google.com/go/httpreplay"
	"cloud.google.com/go/internal/testutil"
	"cloud.google.com/go/storage"
	"golang.org/x/net/context"
	"google.golang.org/api/option"
)

func TestIntegration_RecordAndReplay(t *testing.T) {
	httpreplay.DebugHeaders()
	if testing.Short() {
		t.Skip("Integration tests skipped in short mode")
	}
	f, err := ioutil.TempFile("", "httpreplay")
	if err != nil {
		t.Fatal(err)
	}
	replayFilename := f.Name()
	if err := f.Close(); err != nil {
		t.Fatal(err)
	}
	defer os.Remove(replayFilename)
	projectID := testutil.ProjID()
	if projectID == "" {
		t.Skip("Need project ID. See CONTRIBUTING.md for details.")
	}
	ctx := context.Background()

	// Record.
	initial := time.Now()
	ibytes, err := json.Marshal(initial)
	if err != nil {
		t.Fatal(err)
	}
	rec, err := httpreplay.NewRecorder(replayFilename, ibytes)
	if err != nil {
		t.Fatal(err)
	}
	hc, err := rec.Client(ctx, option.WithTokenSource(
		testutil.TokenSource(ctx, storage.ScopeFullControl)))
	if err != nil {
		t.Fatal(err)
	}
	wanta, wantc := run(t, hc)
	testReadCRC(t, hc, "recording")
	if err := rec.Close(); err != nil {
		t.Fatalf("rec.Close: %v", err)
	}

	// Replay.
	rep, err := httpreplay.NewReplayer(replayFilename)
	if err != nil {
		t.Fatal(err)
	}
	defer rep.Close()
	hc, err = rep.Client(ctx)
	if err != nil {
		t.Fatal(err)
	}
	gota, gotc := run(t, hc)
	testReadCRC(t, hc, "replaying")

	if diff := testutil.Diff(gota, wanta); diff != "" {
		t.Error(diff)
	}
	if !bytes.Equal(gotc, wantc) {
		t.Errorf("got %q, want %q", gotc, wantc)
	}
	var gotInitial time.Time
	if err := json.Unmarshal(rep.Initial(), &gotInitial); err != nil {
		t.Fatal(err)
	}
	if !gotInitial.Equal(initial) {
		t.Errorf("initial: got %v, want %v", gotInitial, initial)
	}
}

// TODO(jba): test errors

func run(t *testing.T, hc *http.Client) (*storage.BucketAttrs, []byte) {
	ctx := context.Background()
	client, err := storage.NewClient(ctx, option.WithHTTPClient(hc))
	if err != nil {
		t.Fatal(err)
	}
	defer client.Close()
	b := client.Bucket(testutil.ProjID())
	attrs, err := b.Attrs(ctx)
	if err != nil {
		t.Fatal(err)
	}
	obj := b.Object("replay-test")
	w := obj.NewWriter(ctx)
	data := []byte{150, 151, 152}
	if _, err := w.Write(data); err != nil {
		t.Fatal(err)
	}
	if err := w.Close(); err != nil {
		t.Fatal(err)
	}

	r, err := obj.NewReader(ctx)
	if err != nil {
		t.Fatal(err)
	}
	defer r.Close()
	contents, err := ioutil.ReadAll(r)
	if err != nil {
		t.Fatal(err)
	}

	return attrs, contents
}

func testReadCRC(t *testing.T, hc *http.Client, mode string) {
	const (
		// This is an uncompressed file.
		// See https://cloud.google.com/storage/docs/public-datasets/landsat
		uncompressedBucket = "gcp-public-data-landsat"
		uncompressedObject = "LC08/PRE/044/034/LC80440342016259LGN00/LC80440342016259LGN00_MTL.txt"

		gzippedBucket   = "storage-library-test-bucket"
		gzippedObject   = "gzipped-text.txt"
		gzippedContents = "hello world" // uncompressed contents of the file
	)
	ctx := context.Background()
	client, err := storage.NewClient(ctx, option.WithHTTPClient(hc))
	if err != nil {
		t.Fatalf("%s: %v", mode, err)
	}
	defer client.Close()

	uncompressedObj := client.Bucket(uncompressedBucket).Object(uncompressedObject)
	gzippedObj := client.Bucket(gzippedBucket).Object(gzippedObject)

	for _, test := range []struct {
		desc           string
		obj            *storage.ObjectHandle
		offset, length int64
		readCompressed bool // don't decompress a gzipped file

		wantErr bool
		wantLen int // length of contents
	}{
		{
			desc:           "uncompressed, entire file",
			obj:            uncompressedObj,
			offset:         0,
			length:         -1,
			readCompressed: false,
			wantLen:        7903,
		},
		{
			desc:           "uncompressed, entire file, don't decompress",
			obj:            uncompressedObj,
			offset:         0,
			length:         -1,
			readCompressed: true,
			wantLen:        7903,
		},
		{
			desc:           "uncompressed, suffix",
			obj:            uncompressedObj,
			offset:         3,
			length:         -1,
			readCompressed: false,
			wantLen:        7900,
		},
		{
			desc:           "uncompressed, prefix",
			obj:            uncompressedObj,
			offset:         0,
			length:         18,
			readCompressed: false,
			wantLen:        18,
		},
		{
			// When a gzipped file is unzipped by GCS, we can't verify the checksum
			// because it was computed against the zipped contents. There is no
			// header that indicates that a gzipped file is being served unzipped.
			// But our CRC check only happens if there is a Content-Length header,
			// and that header is absent for this read.
			desc:           "compressed, entire file, server unzips",
			obj:            gzippedObj,
			offset:         0,
			length:         -1,
			readCompressed: false,
			wantLen:        11,
		},
		{
			// When we read a gzipped file uncompressed, it's like reading a regular file:
			// the served content and the CRC match.
			desc:           "compressed, entire file, read compressed",
			obj:            gzippedObj,
			offset:         0,
			length:         -1,
			readCompressed: true,
			wantLen:        31,
		},
		{
			desc:           "compressed, partial, server unzips",
			obj:            gzippedObj,
			offset:         1,
			length:         8,
			readCompressed: false,
			wantErr:        true, // GCS can't serve part of a gzipped object
		},
		{
			desc:           "compressed, partial, read compressed",
			obj:            gzippedObj,
			offset:         1,
			length:         8,
			readCompressed: true,
			wantLen:        8,
		},
		{
			desc:    "uncompressed, HEAD",
			obj:     uncompressedObj,
			offset:  0,
			length:  0,
			wantLen: 0,
		},
		{
			desc:    "compressed, HEAD",
			obj:     gzippedObj,
			offset:  0,
			length:  0,
			wantLen: 0,
		},
	} {
		obj := test.obj.ReadCompressed(test.readCompressed)
		r, err := obj.NewRangeReader(ctx, test.offset, test.length)
		if err != nil {
			if test.wantErr {
				continue
			}
			t.Errorf("%s: %s: %v", mode, test.desc, err)
			continue
		}
		data, err := ioutil.ReadAll(r)
		_ = r.Close()
		if err != nil {
			t.Errorf("%s: %s: %v", mode, test.desc, err)
			continue
		}
		if got, want := len(data), test.wantLen; got != want {
			t.Errorf("%s: %s: len: got %d, want %d", mode, test.desc, got, want)
		}
	}
}
<commit_msg>httpreplay: remove a test case<commit_after>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build go1.8

package httpreplay_test

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"os"
	"testing"
	"time"

	"cloud.google.com/go/httpreplay"
	"cloud.google.com/go/internal/testutil"
	"cloud.google.com/go/storage"
	"golang.org/x/net/context"
	"google.golang.org/api/option"
)

func TestIntegration_RecordAndReplay(t *testing.T) {
	httpreplay.DebugHeaders()
	if testing.Short() {
		t.Skip("Integration tests skipped in short mode")
	}
	f, err := ioutil.TempFile("", "httpreplay")
	if err != nil {
		t.Fatal(err)
	}
	replayFilename := f.Name()
	if err := f.Close(); err != nil {
		t.Fatal(err)
	}
	defer os.Remove(replayFilename)
	projectID := testutil.ProjID()
	if projectID == "" {
		t.Skip("Need project ID. See CONTRIBUTING.md for details.")
	}
	ctx := context.Background()

	// Record.
	initial := time.Now()
	ibytes, err := json.Marshal(initial)
	if err != nil {
		t.Fatal(err)
	}
	rec, err := httpreplay.NewRecorder(replayFilename, ibytes)
	if err != nil {
		t.Fatal(err)
	}
	hc, err := rec.Client(ctx, option.WithTokenSource(
		testutil.TokenSource(ctx, storage.ScopeFullControl)))
	if err != nil {
		t.Fatal(err)
	}
	wanta, wantc := run(t, hc)
	testReadCRC(t, hc, "recording")
	if err := rec.Close(); err != nil {
		t.Fatalf("rec.Close: %v", err)
	}

	// Replay.
	rep, err := httpreplay.NewReplayer(replayFilename)
	if err != nil {
		t.Fatal(err)
	}
	defer rep.Close()
	hc, err = rep.Client(ctx)
	if err != nil {
		t.Fatal(err)
	}
	gota, gotc := run(t, hc)
	testReadCRC(t, hc, "replaying")

	if diff := testutil.Diff(gota, wanta); diff != "" {
		t.Error(diff)
	}
	if !bytes.Equal(gotc, wantc) {
		t.Errorf("got %q, want %q", gotc, wantc)
	}
	var gotInitial time.Time
	if err := json.Unmarshal(rep.Initial(), &gotInitial); err != nil {
		t.Fatal(err)
	}
	if !gotInitial.Equal(initial) {
		t.Errorf("initial: got %v, want %v", gotInitial, initial)
	}
}

// TODO(jba): test errors

func run(t *testing.T, hc *http.Client) (*storage.BucketAttrs, []byte) {
	ctx := context.Background()
	client, err := storage.NewClient(ctx, option.WithHTTPClient(hc))
	if err != nil {
		t.Fatal(err)
	}
	defer client.Close()
	b := client.Bucket(testutil.ProjID())
	attrs, err := b.Attrs(ctx)
	if err != nil {
		t.Fatal(err)
	}
	obj := b.Object("replay-test")
	w := obj.NewWriter(ctx)
	data := []byte{150, 151, 152}
	if _, err := w.Write(data); err != nil {
		t.Fatal(err)
	}
	if err := w.Close(); err != nil {
		t.Fatal(err)
	}

	r, err := obj.NewReader(ctx)
	if err != nil {
		t.Fatal(err)
	}
	defer r.Close()
	contents, err := ioutil.ReadAll(r)
	if err != nil {
		t.Fatal(err)
	}

	return attrs, contents
}

func testReadCRC(t *testing.T, hc *http.Client, mode string) {
	const (
		// This is an uncompressed file.
		// See https://cloud.google.com/storage/docs/public-datasets/landsat
		uncompressedBucket = "gcp-public-data-landsat"
		uncompressedObject = "LC08/PRE/044/034/LC80440342016259LGN00/LC80440342016259LGN00_MTL.txt"

		gzippedBucket   = "storage-library-test-bucket"
		gzippedObject   = "gzipped-text.txt"
		gzippedContents = "hello world" // uncompressed contents of the file
	)
	ctx := context.Background()
	client, err := storage.NewClient(ctx, option.WithHTTPClient(hc))
	if err != nil {
		t.Fatalf("%s: %v", mode, err)
	}
	defer client.Close()

	uncompressedObj := client.Bucket(uncompressedBucket).Object(uncompressedObject)
	gzippedObj := client.Bucket(gzippedBucket).Object(gzippedObject)

	for _, test := range []struct {
		desc           string
		obj            *storage.ObjectHandle
		offset, length int64
		readCompressed bool // don't decompress a gzipped file

		wantErr bool
		wantLen int // length of contents
	}{
		{
			desc:           "uncompressed, entire file",
			obj:            uncompressedObj,
			offset:         0,
			length:         -1,
			readCompressed: false,
			wantLen:        7903,
		},
		{
			desc:           "uncompressed, entire file, don't decompress",
			obj:            uncompressedObj,
			offset:         0,
			length:         -1,
			readCompressed: true,
			wantLen:        7903,
		},
		{
			desc:           "uncompressed, suffix",
			obj:            uncompressedObj,
			offset:         3,
			length:         -1,
			readCompressed: false,
			wantLen:        7900,
		},
		{
			desc:           "uncompressed, prefix",
			obj:            uncompressedObj,
			offset:         0,
			length:         18,
			readCompressed: false,
			wantLen:        18,
		},
		{
			// When a gzipped file is unzipped by GCS, we can't verify the checksum
			// because it was computed against the zipped contents. There is no
			// header that indicates that a gzipped file is being served unzipped.
			// But our CRC check only happens if there is a Content-Length header,
			// and that header is absent for this read.
			desc:           "compressed, entire file, server unzips",
			obj:            gzippedObj,
			offset:         0,
			length:         -1,
			readCompressed: false,
			wantLen:        11,
		},
		{
			// When we read a gzipped file uncompressed, it's like reading a regular file:
			// the served content and the CRC match.
			desc:           "compressed, entire file, read compressed",
			obj:            gzippedObj,
			offset:         0,
			length:         -1,
			readCompressed: true,
			wantLen:        31,
		},
		{
			desc:           "compressed, partial, read compressed",
			obj:            gzippedObj,
			offset:         1,
			length:         8,
			readCompressed: true,
			wantLen:        8,
		},
		{
			desc:    "uncompressed, HEAD",
			obj:     uncompressedObj,
			offset:  0,
			length:  0,
			wantLen: 0,
		},
		{
			desc:    "compressed, HEAD",
			obj:     gzippedObj,
			offset:  0,
			length:  0,
			wantLen: 0,
		},
	} {
		obj := test.obj.ReadCompressed(test.readCompressed)
		r, err := obj.NewRangeReader(ctx, test.offset, test.length)
		if err != nil {
			if test.wantErr {
				continue
			}
			t.Errorf("%s: %s: %v", mode, test.desc, err)
			continue
		}
		data, err := ioutil.ReadAll(r)
		_ = r.Close()
		if err != nil {
			t.Errorf("%s: %s: %v", mode, test.desc, err)
			continue
		}
		if got, want := len(data), test.wantLen; got != want {
			t.Errorf("%s: %s: len: got %d, want %d", mode, test.desc, got, want)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright ©2015 The Gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gonum

import "gonum.org/v1/gonum/blas"

// Dlaset sets the off-diagonal elements of A to alpha, and the diagonal
// elements to beta. If uplo == blas.Upper, only the elements in the upper
// triangular part are set. If uplo == blas.Lower, only the elements in the
// lower triangular part are set. If uplo is otherwise, all of the elements of A
// are set.
//
// Dlaset is an internal routine. It is exported for testing purposes.
func (impl Implementation) Dlaset(uplo blas.Uplo, m, n int, alpha, beta float64, a []float64, lda int) {
	switch {
	case m < 0:
		panic(mLT0)
	case n < 0:
		panic(nLT0)
	case lda < max(1, n):
		panic(badLdA)
	}

	minmn := min(m, n)
	if minmn == 0 {
		return
	}

	if len(a) < (m-1)*lda+n {
		panic(shortA)
	}

	if uplo == blas.Upper {
		for i := 0; i < m; i++ {
			for j := i + 1; j < n; j++ {
				a[i*lda+j] = alpha
			}
		}
	} else if uplo == blas.Lower {
		for i := 0; i < m; i++ {
			for j := 0; j < min(i+1, n); j++ {
				a[i*lda+j] = alpha
			}
		}
	} else {
		for i := 0; i < m; i++ {
			for j := 0; j < n; j++ {
				a[i*lda+j] = alpha
			}
		}
	}
	for i := 0; i < minmn; i++ {
		a[i*lda+i] = beta
	}
}
<commit_msg>lapack: set strictly lower triangle in Dlaset<commit_after>// Copyright ©2015 The Gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gonum

import "gonum.org/v1/gonum/blas"

// Dlaset sets the off-diagonal elements of A to alpha, and the diagonal
// elements to beta. If uplo == blas.Upper, only the elements in the upper
// triangular part are set. If uplo == blas.Lower, only the elements in the
// lower triangular part are set. If uplo is otherwise, all of the elements of A
// are set.
//
// Dlaset is an internal routine. It is exported for testing purposes.
func (impl Implementation) Dlaset(uplo blas.Uplo, m, n int, alpha, beta float64, a []float64, lda int) {
	switch {
	case m < 0:
		panic(mLT0)
	case n < 0:
		panic(nLT0)
	case lda < max(1, n):
		panic(badLdA)
	}

	minmn := min(m, n)
	if minmn == 0 {
		return
	}

	if len(a) < (m-1)*lda+n {
		panic(shortA)
	}

	switch uplo {
	case blas.Upper:
		for i := 0; i < m; i++ {
			for j := i + 1; j < n; j++ {
				a[i*lda+j] = alpha
			}
		}
	case blas.Lower:
		for i := 0; i < m; i++ {
			for j := 0; j < min(i, n); j++ {
				a[i*lda+j] = alpha
			}
		}
	default:
		for i := 0; i < m; i++ {
			for j := 0; j < n; j++ {
				a[i*lda+j] = alpha
			}
		}
	}
	for i := 0; i < minmn; i++ {
		a[i*lda+i] = beta
	}
}
<|endoftext|>


<commit_before>// Copyright ©2019 The Gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gonum

import (
	"math"

	"gonum.org/v1/gonum/blas"
	"gonum.org/v1/gonum/blas/blas64"
)

// Dpbcon returns an estimate of the reciprocal of the condition number (in the
// 1-norm) of an n×n symmetric positive definite band matrix using the Cholesky
// factorization
//  A = U^T*U  if uplo == blas.Upper
//  A = L*L^T  if uplo == blas.Lower
// computed by Dpbtrf. The estimate is obtained for norm(inv(A)), and the
// reciprocal of the condition number is computed as
//  rcond = 1 / (anorm * norm(inv(A))).
//
// The length of work must be at least 3*n and the length of iwork must be at
// least n.
func (impl Implementation) Dpbcon(uplo blas.Uplo, n, kd int, ab []float64, ldab int, anorm float64, work []float64, iwork []int) (rcond float64) {
	switch {
	case uplo != blas.Upper && uplo != blas.Lower:
		panic(badUplo)
	case n < 0:
		panic(nLT0)
	case kd < 0:
		panic(kdLT0)
	case ldab < kd+1:
		panic(badLdA)
	case anorm < 0:
		panic(badNorm)
	}

	// Quick return if possible.
	if n == 0 {
		return 1
	}

	switch {
	case len(ab) < (n-1)*ldab+kd+1:
		panic(shortAB)
	case len(work) < 3*n:
		panic(shortWork)
	case len(iwork) < n:
		panic(shortIWork)
	}

	// Quick return if possible.
	if anorm == 0 {
		return 0
	}

	const smlnum = dlamchS

	var (
		ainvnm float64
		kase   int
		isave  [3]int

		normin         bool
		scaleL, scaleU float64

		// Denote work slices.
		x     = work[:n]
		v     = work[n : 2*n]
		cnorm = work[2*n : 3*n]
	)
	// Estimate the 1-norm of the inverse.
	bi := blas64.Implementation()
	for {
		ainvnm, kase = impl.Dlacn2(n, v, x, iwork, ainvnm, kase, &isave)
		if kase == 0 {
			break
		}
		if uplo == blas.Upper {
			// Multiply x by inv(U^T).
			scaleL = impl.Dlatbs(uplo, blas.Trans, blas.NonUnit, normin, n, kd, ab, ldab, x, cnorm)
			normin = true
			// Multiply x by inv(U).
			scaleU = impl.Dlatbs(uplo, blas.NoTrans, blas.NonUnit, normin, n, kd, ab, ldab, x, cnorm)
		} else {
			// Multiply x by inv(L).
			scaleL = impl.Dlatbs(uplo, blas.NoTrans, blas.NonUnit, normin, n, kd, ab, ldab, x, cnorm)
			normin = true
			// Multiply x by inv(L^T).
			scaleU = impl.Dlatbs(uplo, blas.Trans, blas.NonUnit, normin, n, kd, ab, ldab, x, cnorm)
		}
		// Multiply x by 1/scale if doing so will not cause overflow.
		scale := scaleL * scaleU
		if scale != 1 {
			ix := bi.Idamax(n, x, 1)
			if scale < math.Abs(x[ix])*smlnum || scale == 0 {
				return 0
			}
			impl.Drscl(n, scale, x, 1)
		}
	}
	if ainvnm == 0 {
		return 0
	}
	// Return the estimate of the reciprocal condition number.
	return (1 / ainvnm) / anorm
}
<commit_msg>lapack/gonum: refactor inner loop in Dpbcon<commit_after>// Copyright ©2019 The Gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gonum

import (
	"math"

	"gonum.org/v1/gonum/blas"
	"gonum.org/v1/gonum/blas/blas64"
)

// Dpbcon returns an estimate of the reciprocal of the condition number (in the
// 1-norm) of an n×n symmetric positive definite band matrix using the Cholesky
// factorization
//  A = U^T*U  if uplo == blas.Upper
//  A = L*L^T  if uplo == blas.Lower
// computed by Dpbtrf. The estimate is obtained for norm(inv(A)), and the
// reciprocal of the condition number is computed as
//  rcond = 1 / (anorm * norm(inv(A))).
//
// The length of work must be at least 3*n and the length of iwork must be at
// least n.
func (impl Implementation) Dpbcon(uplo blas.Uplo, n, kd int, ab []float64, ldab int, anorm float64, work []float64, iwork []int) (rcond float64) {
	switch {
	case uplo != blas.Upper && uplo != blas.Lower:
		panic(badUplo)
	case n < 0:
		panic(nLT0)
	case kd < 0:
		panic(kdLT0)
	case ldab < kd+1:
		panic(badLdA)
	case anorm < 0:
		panic(badNorm)
	}

	// Quick return if possible.
	if n == 0 {
		return 1
	}

	switch {
	case len(ab) < (n-1)*ldab+kd+1:
		panic(shortAB)
	case len(work) < 3*n:
		panic(shortWork)
	case len(iwork) < n:
		panic(shortIWork)
	}

	// Quick return if possible.
	if anorm == 0 {
		return 0
	}

	const smlnum = dlamchS

	var (
		ainvnm float64
		kase   int
		isave  [3]int
		normin bool

		// Denote work slices.
		x     = work[:n]
		v     = work[n : 2*n]
		cnorm = work[2*n : 3*n]
	)
	// Estimate the 1-norm of the inverse.
	bi := blas64.Implementation()
	for {
		ainvnm, kase = impl.Dlacn2(n, v, x, iwork, ainvnm, kase, &isave)
		if kase == 0 {
			break
		}
		var op1, op2 blas.Transpose
		if uplo == blas.Upper {
			// Multiply x by inv(U^T),
			op1 = blas.Trans
			// then by inv(U^T).
			op2 = blas.NoTrans
		} else {
			// Multiply x by inv(L),
			op1 = blas.NoTrans
			// then by inv(L^T).
			op2 = blas.Trans
		}
		scaleL := impl.Dlatbs(uplo, op1, blas.NonUnit, normin, n, kd, ab, ldab, x, cnorm)
		normin = true
		scaleU := impl.Dlatbs(uplo, op2, blas.NonUnit, normin, n, kd, ab, ldab, x, cnorm)
		// Multiply x by 1/scale if doing so will not cause overflow.
		scale := scaleL * scaleU
		if scale != 1 {
			ix := bi.Idamax(n, x, 1)
			if scale < math.Abs(x[ix])*smlnum || scale == 0 {
				return 0
			}
			impl.Drscl(n, scale, x, 1)
		}
	}
	if ainvnm == 0 {
		return 0
	}
	// Return the estimate of the reciprocal condition number.
	return (1 / ainvnm) / anorm
}
<|endoftext|>


<commit_before>// Copyright 2017 The ACH Authors
// Use of this source code is governed by an Apache License
// license that can be found in the LICENSE file.

package ach

import (
	"fmt"
	"strconv"
	"strings"
	"time"
)

// msgServiceClass

// BatchHeader identifies the originating entity and the type of transactions
// contained in the batch (i.e., the standard entry class, PPD for consumer, CCD
// or CTX for corporate). This record also contains the effective date, or desired
// settlement date, for all entries contained in this batch. The settlement date
// field is not entered as it is determined by the ACH operator
type BatchHeader struct {
	// RecordType defines the type of record in the block. 5
	recordType string

	// ServiceClassCode ACH Mixed Debits and Credits ‘200’
	// ACH Credits Only ‘220’
	// ACH Debits Only ‘225'
	ServiceClassCode int

	// CompanyName the company originating the entries in the batch
	CompanyName string

	// CompanyDiscretionaryData allows Originators and/or ODFIs to include codes (one or more),
	// of significance only to them, to enable specialized handling of all
	// subsequent entries in that batch. There will be no standardized
	// interpretation for the value of the field. This field must be returned
	// intact on any return entry.
	CompanyDiscretionaryData string

	// CompanyIdentification The 9 digit FEIN number (proceeded by a predetermined
	// alpha or numeric character) of the entity in the company name field
	CompanyIdentification string

	// StandardEntryClassCode PPD’ for consumer transactions, ‘CCD’ or ‘CTX’ for corporate
	StandardEntryClassCode string

	// CompanyEntryDescription A description of the entries contained in the batch
	//
	//The Originator establishes the value of this field to provide a
	// description of the purpose of the entry to be displayed back to
	// the receive For example, "GAS BILL," "REG. SALARY," "INS. PREM,"
	// "SOC. SEC.," "DTC," "TRADE PAY," "PURCHASE," etc.
	//
	// This field must contain the word "REVERSAL" (left justified) when the
	// batch contains reversing entries.
	//
	// This field must contain the word "RECLAIM" (left justified) when the
	// batch contains reclamation entries.
	//
	// This field must contain the word "NONSETTLED" (left justified) when the
	// batch contains entries which could not settle.
	CompanyEntryDescription string

	// CompanyDescriptiveDate except as otherwise noted below, the Originator establishes this field
	// as the date it would like to see displayed to the receiver for
	// descriptive purposes. This field is never used to control timing of any
	// computer or manual operation. It is solely for descriptive purposes.
	// The RDFI should not assume any specific format. Examples of possible
	// entries in this field are "011392,", "01 92," "JAN 13," "JAN 92," etc.
	CompanyDescriptiveDate string

	// EffectiveEntryDate the date on which the entries are to settle
	EffectiveEntryDate time.Time

	// SettlementDate Leave blank, this field is inserted by the ACH operator
	settlementDate string

	// OriginatorStatusCode '1'
	OriginatorStatusCode int

	//ODFIIdentification First 8 digits of the originating DFI transit routing number
	ODFIIdentification int

	// BatchNumber is assigned in ascending sequence to each batch by the ODFI
	// or its Sending Point in a given file of entries. Since the batch number
	// in the Batch Header Record and the Batch Control Record is the same,
	// the ascending sequence number should be assigned by batch and not by
	// record.
	BatchNumber int

	// validator is composed for data validation
	validator

	// converters is composed for ACH to golang Converters
	converters
}

// NewBatchHeader returns a new BatchHeader with default values for none exported fields
func NewBatchHeader(params ...BatchParam) *BatchHeader {
	bh := &BatchHeader{
		recordType:           "5",
		OriginatorStatusCode: 1,
		BatchNumber:          1,
	}
	if len(params) > 0 {
		bh.ServiceClassCode = bh.parseNumField(params[0].ServiceClassCode)
		bh.CompanyName = params[0].CompanyName
		bh.CompanyIdentification = params[0].CompanyIdentification
		bh.StandardEntryClassCode = params[0].StandardEntryClass
		bh.CompanyEntryDescription = params[0].CompanyEntryDescription
		bh.CompanyDescriptiveDate = params[0].CompanyDescriptiveDate
		bh.EffectiveEntryDate = bh.parseSimpleDate(params[0].EffectiveEntryDate)
		bh.ODFIIdentification = bh.parseNumField(params[0].ODFIIdentification)
		return bh
	}
	return bh
}

// Parse takes the input record string and parses the BatchHeader values
func (bh *BatchHeader) Parse(record string) {
	// 1-1 Always "5"
	bh.recordType = "5"
	// 2-4 If the entries are credits, always "220". If the entries are debits, always "225"
	bh.ServiceClassCode = bh.parseNumField(record[1:4])
	// 5-20 Your company's name. This name may appear on the receivers’ statements prepared by the RDFI.
	bh.CompanyName = strings.TrimSpace(record[4:20])
	// 21-40 Optional field you may use to describe the batch for internal accounting purposes
	bh.CompanyDiscretionaryData = strings.TrimSpace(record[20:40])
	// 41-50 A 10-digit number assigned to you by the ODFI once they approve you to
	// originate ACH files through them. This is the same as the "Immediate origin" field in File Header Record
	bh.CompanyIdentification = strings.TrimSpace(record[40:50])
	// 51-53 If the entries are PPD (credits/debits towards consumer account), use "PPD".
	// If the entries are CCD (credits/debits towards corporate account), use "CCD".
	// The difference between the 2 class codes are outside of the scope of this post, but generally most ACH transfers to consumer bank accounts should use "PPD"
	bh.StandardEntryClassCode = record[50:53]
	// 54-63 Your description of the transaction. This text will appear on the receivers’ bank statement.
	// For example: "Payroll   "
	bh.CompanyEntryDescription = strings.TrimSpace(record[53:63])
	// 64-69 The date you choose to identify the transactions in YYMMDD format.
	// This date may be printed on the receivers’ bank statement by the RDFI
	bh.CompanyDescriptiveDate = strings.TrimSpace(record[63:69])
	// 70-75 Date transactions are to be posted to the receivers’ account.
	// You almost always want the transaction to post as soon as possible, so put tomorrow's date in YYMMDD format
	bh.EffectiveEntryDate = bh.parseSimpleDate(record[69:75])
	// 76-79 Always blank (just fill with spaces)
	bh.settlementDate = "   "
	// 79-79 Always 1
	bh.OriginatorStatusCode = bh.parseNumField(record[78:79])
	// 80-87 Your ODFI's routing number without the last digit. The last digit is simply a
	// checksum digit, which is why it is not necessary
	bh.ODFIIdentification = bh.parseNumField(record[79:87])
	// 88-94 Sequential number of this Batch Header Record
	// For example, put "1" if this is the first Batch Header Record in the file
	bh.BatchNumber = bh.parseNumField(record[87:94])
}

// String writes the BatchHeader struct to a 94 character string.
func (bh *BatchHeader) String() string {
	return fmt.Sprintf("%v%v%v%v%v%v%v%v%v%v%v%v%v",
		bh.recordType,
		bh.ServiceClassCode,
		bh.CompanyNameField(),
		bh.CompanyDiscretionaryDataField(),
		bh.CompanyIdentificationField(),
		bh.StandardEntryClassCode,
		bh.CompanyEntryDescriptionField(),
		bh.CompanyDescriptiveDateField(),
		bh.EffectiveEntryDateField(),
		bh.settlementDateField(),
		bh.OriginatorStatusCode,
		bh.ODFIIdentificationField(),
		bh.BatchNumberField(),
	)
}

// Validate performs NACHA format rule checks on the record and returns an error if not Validated
// The first error encountered is returned and stops that parsing.
func (bh *BatchHeader) Validate() error {
	if err := bh.fieldInclusion(); err != nil {
		return err
	}
	if bh.recordType != "5" {
		msg := fmt.Sprintf(msgRecordType, 5)
		return &FieldError{FieldName: "recordType", Value: bh.recordType, Msg: msg}
	}
	if err := bh.isServiceClass(bh.ServiceClassCode); err != nil {
		return &FieldError{FieldName: "ServiceClassCode", Value: strconv.Itoa(bh.ServiceClassCode), Msg: err.Error()}
	}
	if err := bh.isSECCode(bh.StandardEntryClassCode); err != nil {
		return &FieldError{FieldName: "StandardEntryClassCode", Value: bh.StandardEntryClassCode, Msg: err.Error()}
	}
	if err := bh.isOriginatorStatusCode(bh.OriginatorStatusCode); err != nil {
		return &FieldError{FieldName: "OriginatorStatusCode", Value: strconv.Itoa(bh.OriginatorStatusCode), Msg: err.Error()}
	}
	if err := bh.isAlphanumeric(bh.CompanyName); err != nil {
		return &FieldError{FieldName: "CompanyName", Value: bh.CompanyName, Msg: err.Error()}
	}
	if err := bh.isAlphanumeric(bh.CompanyDiscretionaryData); err != nil {
		return &FieldError{FieldName: "CompanyDiscretionaryData", Value: bh.CompanyDiscretionaryData, Msg: err.Error()}
	}
	if err := bh.isAlphanumeric(bh.CompanyIdentification); err != nil {
		return &FieldError{FieldName: "CompanyIdentification", Value: bh.CompanyIdentification, Msg: err.Error()}
	}
	if err := bh.isAlphanumeric(bh.CompanyEntryDescription); err != nil {
		return &FieldError{FieldName: "CompanyEntryDescription", Value: bh.CompanyEntryDescription, Msg: err.Error()}
	}

	return nil
}

// fieldInclusion validate mandatory fields are not default values. If fields are
// invalid the ACH transfer will be returned.
func (bh *BatchHeader) fieldInclusion() error {
	if bh.recordType == "" {
		return &FieldError{FieldName: "recordType", Value: bh.recordType, Msg: msgFieldInclusion}
	}
	if bh.ServiceClassCode == 0 {
		return &FieldError{FieldName: "ServiceClassCode", Value: strconv.Itoa(bh.ServiceClassCode), Msg: msgFieldInclusion}
	}
	if bh.CompanyName == "" {
		return &FieldError{FieldName: "CompanyName", Value: bh.CompanyName, Msg: msgFieldInclusion}
	}
	if bh.CompanyIdentification == "" {
		return &FieldError{FieldName: "CompanyIdentification", Value: bh.CompanyIdentification, Msg: msgFieldInclusion}
	}
	if bh.StandardEntryClassCode == "" {
		return &FieldError{FieldName: "StandardEntryClassCode", Value: bh.StandardEntryClassCode, Msg: msgFieldInclusion}
	}
	if bh.CompanyEntryDescription == "" {
		return &FieldError{FieldName: "CompanyEntryDescription", Value: bh.CompanyEntryDescription, Msg: msgFieldInclusion}
	}
	if bh.OriginatorStatusCode == 0 {
		return &FieldError{FieldName: "OriginatorStatusCode", Value: strconv.Itoa(bh.OriginatorStatusCode), Msg: msgFieldInclusion}
	}
	if bh.ODFIIdentification == 0 {
		return &FieldError{FieldName: "ODFIIdentification", Value: bh.ODFIIdentificationField(), Msg: msgFieldInclusion}
	}

	return nil
}

// CompanyNameField get the CompanyName left padded
func (bh *BatchHeader) CompanyNameField() string {
	return bh.alphaField(bh.CompanyName, 16)
}

// CompanyDiscretionaryDataField get the CompanyDiscretionaryData left padded
func (bh *BatchHeader) CompanyDiscretionaryDataField() string {
	return bh.alphaField(bh.CompanyDiscretionaryData, 20)
}

// CompanyIdentificationField get the CompanyIdentification left padded
func (bh *BatchHeader) CompanyIdentificationField() string {
	return bh.alphaField(bh.CompanyIdentification, 10)
}

// CompanyEntryDescriptionField get the CompanyEntryDescription left padded
func (bh *BatchHeader) CompanyEntryDescriptionField() string {
	return bh.alphaField(bh.CompanyEntryDescription, 10)
}

// CompanyDescriptiveDateField get the CompanyDescriptiveDate left padded
func (bh *BatchHeader) CompanyDescriptiveDateField() string {
	return bh.alphaField(bh.CompanyDescriptiveDate, 6)
}

// EffectiveEntryDateField get the EffectiveEntryDate in YYMMDD format
func (bh *BatchHeader) EffectiveEntryDateField() string {
	return bh.formatSimpleDate(bh.EffectiveEntryDate)
}

// ODFIIdentificationField get the odfi number zero padded
func (bh *BatchHeader) ODFIIdentificationField() string {
	return bh.numericField(bh.ODFIIdentification, 8)
}

// BatchNumberField get the batch number zero padded
func (bh *BatchHeader) BatchNumberField() string {
	return bh.numericField(bh.BatchNumber, 7)
}

func (bh *BatchHeader) settlementDateField() string {
	return bh.alphaField(bh.settlementDate, 3)
}
<commit_msg>BatchHeader.OriginatorStatusCode can be zero. Validation does not need to exist for default value 0<commit_after>// Copyright 2017 The ACH Authors
// Use of this source code is governed by an Apache License
// license that can be found in the LICENSE file.

package ach

import (
	"fmt"
	"strconv"
	"strings"
	"time"
)

// msgServiceClass

// BatchHeader identifies the originating entity and the type of transactions
// contained in the batch (i.e., the standard entry class, PPD for consumer, CCD
// or CTX for corporate). This record also contains the effective date, or desired
// settlement date, for all entries contained in this batch. The settlement date
// field is not entered as it is determined by the ACH operator
type BatchHeader struct {
	// RecordType defines the type of record in the block. 5
	recordType string

	// ServiceClassCode ACH Mixed Debits and Credits ‘200’
	// ACH Credits Only ‘220’
	// ACH Debits Only ‘225'
	ServiceClassCode int

	// CompanyName the company originating the entries in the batch
	CompanyName string

	// CompanyDiscretionaryData allows Originators and/or ODFIs to include codes (one or more),
	// of significance only to them, to enable specialized handling of all
	// subsequent entries in that batch. There will be no standardized
	// interpretation for the value of the field. This field must be returned
	// intact on any return entry.
	CompanyDiscretionaryData string

	// CompanyIdentification The 9 digit FEIN number (proceeded by a predetermined
	// alpha or numeric character) of the entity in the company name field
	CompanyIdentification string

	// StandardEntryClassCode PPD’ for consumer transactions, ‘CCD’ or ‘CTX’ for corporate
	StandardEntryClassCode string

	// CompanyEntryDescription A description of the entries contained in the batch
	//
	//The Originator establishes the value of this field to provide a
	// description of the purpose of the entry to be displayed back to
	// the receive For example, "GAS BILL," "REG. SALARY," "INS. PREM,"
	// "SOC. SEC.," "DTC," "TRADE PAY," "PURCHASE," etc.
	//
	// This field must contain the word "REVERSAL" (left justified) when the
	// batch contains reversing entries.
	//
	// This field must contain the word "RECLAIM" (left justified) when the
	// batch contains reclamation entries.
	//
	// This field must contain the word "NONSETTLED" (left justified) when the
	// batch contains entries which could not settle.
	CompanyEntryDescription string

	// CompanyDescriptiveDate except as otherwise noted below, the Originator establishes this field
	// as the date it would like to see displayed to the receiver for
	// descriptive purposes. This field is never used to control timing of any
	// computer or manual operation. It is solely for descriptive purposes.
	// The RDFI should not assume any specific format. Examples of possible
	// entries in this field are "011392,", "01 92," "JAN 13," "JAN 92," etc.
	CompanyDescriptiveDate string

	// EffectiveEntryDate the date on which the entries are to settle
	EffectiveEntryDate time.Time

	// SettlementDate Leave blank, this field is inserted by the ACH operator
	settlementDate string

	// OriginatorStatusCode '1'
	OriginatorStatusCode int

	//ODFIIdentification First 8 digits of the originating DFI transit routing number
	ODFIIdentification int

	// BatchNumber is assigned in ascending sequence to each batch by the ODFI
	// or its Sending Point in a given file of entries. Since the batch number
	// in the Batch Header Record and the Batch Control Record is the same,
	// the ascending sequence number should be assigned by batch and not by
	// record.
	BatchNumber int

	// validator is composed for data validation
	validator

	// converters is composed for ACH to golang Converters
	converters
}

// NewBatchHeader returns a new BatchHeader with default values for none exported fields
func NewBatchHeader(params ...BatchParam) *BatchHeader {
	bh := &BatchHeader{
		recordType:           "5",
		OriginatorStatusCode: 0, //Prepared by an Originator
		BatchNumber:          1,
	}
	if len(params) > 0 {
		bh.ServiceClassCode = bh.parseNumField(params[0].ServiceClassCode)
		bh.CompanyName = params[0].CompanyName
		bh.CompanyIdentification = params[0].CompanyIdentification
		bh.StandardEntryClassCode = params[0].StandardEntryClass
		bh.CompanyEntryDescription = params[0].CompanyEntryDescription
		bh.CompanyDescriptiveDate = params[0].CompanyDescriptiveDate
		bh.EffectiveEntryDate = bh.parseSimpleDate(params[0].EffectiveEntryDate)
		bh.ODFIIdentification = bh.parseNumField(params[0].ODFIIdentification)
		return bh
	}
	return bh
}

// Parse takes the input record string and parses the BatchHeader values
func (bh *BatchHeader) Parse(record string) {
	// 1-1 Always "5"
	bh.recordType = "5"
	// 2-4 If the entries are credits, always "220". If the entries are debits, always "225"
	bh.ServiceClassCode = bh.parseNumField(record[1:4])
	// 5-20 Your company's name. This name may appear on the receivers’ statements prepared by the RDFI.
	bh.CompanyName = strings.TrimSpace(record[4:20])
	// 21-40 Optional field you may use to describe the batch for internal accounting purposes
	bh.CompanyDiscretionaryData = strings.TrimSpace(record[20:40])
	// 41-50 A 10-digit number assigned to you by the ODFI once they approve you to
	// originate ACH files through them. This is the same as the "Immediate origin" field in File Header Record
	bh.CompanyIdentification = strings.TrimSpace(record[40:50])
	// 51-53 If the entries are PPD (credits/debits towards consumer account), use "PPD".
	// If the entries are CCD (credits/debits towards corporate account), use "CCD".
	// The difference between the 2 class codes are outside of the scope of this post, but generally most ACH transfers to consumer bank accounts should use "PPD"
	bh.StandardEntryClassCode = record[50:53]
	// 54-63 Your description of the transaction. This text will appear on the receivers’ bank statement.
	// For example: "Payroll   "
	bh.CompanyEntryDescription = strings.TrimSpace(record[53:63])
	// 64-69 The date you choose to identify the transactions in YYMMDD format.
	// This date may be printed on the receivers’ bank statement by the RDFI
	bh.CompanyDescriptiveDate = strings.TrimSpace(record[63:69])
	// 70-75 Date transactions are to be posted to the receivers’ account.
	// You almost always want the transaction to post as soon as possible, so put tomorrow's date in YYMMDD format
	bh.EffectiveEntryDate = bh.parseSimpleDate(record[69:75])
	// 76-79 Always blank (just fill with spaces)
	bh.settlementDate = "   "
	// 79-79 Always 1
	bh.OriginatorStatusCode = bh.parseNumField(record[78:79])
	// 80-87 Your ODFI's routing number without the last digit. The last digit is simply a
	// checksum digit, which is why it is not necessary
	bh.ODFIIdentification = bh.parseNumField(record[79:87])
	// 88-94 Sequential number of this Batch Header Record
	// For example, put "1" if this is the first Batch Header Record in the file
	bh.BatchNumber = bh.parseNumField(record[87:94])
}

// String writes the BatchHeader struct to a 94 character string.
func (bh *BatchHeader) String() string {
	return fmt.Sprintf("%v%v%v%v%v%v%v%v%v%v%v%v%v",
		bh.recordType,
		bh.ServiceClassCode,
		bh.CompanyNameField(),
		bh.CompanyDiscretionaryDataField(),
		bh.CompanyIdentificationField(),
		bh.StandardEntryClassCode,
		bh.CompanyEntryDescriptionField(),
		bh.CompanyDescriptiveDateField(),
		bh.EffectiveEntryDateField(),
		bh.settlementDateField(),
		bh.OriginatorStatusCode,
		bh.ODFIIdentificationField(),
		bh.BatchNumberField(),
	)
}

// Validate performs NACHA format rule checks on the record and returns an error if not Validated
// The first error encountered is returned and stops that parsing.
func (bh *BatchHeader) Validate() error {
	if err := bh.fieldInclusion(); err != nil {
		return err
	}
	if bh.recordType != "5" {
		msg := fmt.Sprintf(msgRecordType, 5)
		return &FieldError{FieldName: "recordType", Value: bh.recordType, Msg: msg}
	}
	if err := bh.isServiceClass(bh.ServiceClassCode); err != nil {
		return &FieldError{FieldName: "ServiceClassCode", Value: strconv.Itoa(bh.ServiceClassCode), Msg: err.Error()}
	}
	if err := bh.isSECCode(bh.StandardEntryClassCode); err != nil {
		return &FieldError{FieldName: "StandardEntryClassCode", Value: bh.StandardEntryClassCode, Msg: err.Error()}
	}
	if err := bh.isOriginatorStatusCode(bh.OriginatorStatusCode); err != nil {
		return &FieldError{FieldName: "OriginatorStatusCode", Value: strconv.Itoa(bh.OriginatorStatusCode), Msg: err.Error()}
	}
	if err := bh.isAlphanumeric(bh.CompanyName); err != nil {
		return &FieldError{FieldName: "CompanyName", Value: bh.CompanyName, Msg: err.Error()}
	}
	if err := bh.isAlphanumeric(bh.CompanyDiscretionaryData); err != nil {
		return &FieldError{FieldName: "CompanyDiscretionaryData", Value: bh.CompanyDiscretionaryData, Msg: err.Error()}
	}
	if err := bh.isAlphanumeric(bh.CompanyIdentification); err != nil {
		return &FieldError{FieldName: "CompanyIdentification", Value: bh.CompanyIdentification, Msg: err.Error()}
	}
	if err := bh.isAlphanumeric(bh.CompanyEntryDescription); err != nil {
		return &FieldError{FieldName: "CompanyEntryDescription", Value: bh.CompanyEntryDescription, Msg: err.Error()}
	}

	return nil
}

// fieldInclusion validate mandatory fields are not default values. If fields are
// invalid the ACH transfer will be returned.
func (bh *BatchHeader) fieldInclusion() error {
	if bh.recordType == "" {
		return &FieldError{FieldName: "recordType", Value: bh.recordType, Msg: msgFieldInclusion}
	}
	if bh.ServiceClassCode == 0 {
		return &FieldError{FieldName: "ServiceClassCode", Value: strconv.Itoa(bh.ServiceClassCode), Msg: msgFieldInclusion}
	}
	if bh.CompanyName == "" {
		return &FieldError{FieldName: "CompanyName", Value: bh.CompanyName, Msg: msgFieldInclusion}
	}
	if bh.CompanyIdentification == "" {
		return &FieldError{FieldName: "CompanyIdentification", Value: bh.CompanyIdentification, Msg: msgFieldInclusion}
	}
	if bh.StandardEntryClassCode == "" {
		return &FieldError{FieldName: "StandardEntryClassCode", Value: bh.StandardEntryClassCode, Msg: msgFieldInclusion}
	}
	if bh.CompanyEntryDescription == "" {
		return &FieldError{FieldName: "CompanyEntryDescription", Value: bh.CompanyEntryDescription, Msg: msgFieldInclusion}
	}
	if bh.ODFIIdentification == 0 {
		return &FieldError{FieldName: "ODFIIdentification", Value: bh.ODFIIdentificationField(), Msg: msgFieldInclusion}
	}

	return nil
}

// CompanyNameField get the CompanyName left padded
func (bh *BatchHeader) CompanyNameField() string {
	return bh.alphaField(bh.CompanyName, 16)
}

// CompanyDiscretionaryDataField get the CompanyDiscretionaryData left padded
func (bh *BatchHeader) CompanyDiscretionaryDataField() string {
	return bh.alphaField(bh.CompanyDiscretionaryData, 20)
}

// CompanyIdentificationField get the CompanyIdentification left padded
func (bh *BatchHeader) CompanyIdentificationField() string {
	return bh.alphaField(bh.CompanyIdentification, 10)
}

// CompanyEntryDescriptionField get the CompanyEntryDescription left padded
func (bh *BatchHeader) CompanyEntryDescriptionField() string {
	return bh.alphaField(bh.CompanyEntryDescription, 10)
}

// CompanyDescriptiveDateField get the CompanyDescriptiveDate left padded
func (bh *BatchHeader) CompanyDescriptiveDateField() string {
	return bh.alphaField(bh.CompanyDescriptiveDate, 6)
}

// EffectiveEntryDateField get the EffectiveEntryDate in YYMMDD format
func (bh *BatchHeader) EffectiveEntryDateField() string {
	return bh.formatSimpleDate(bh.EffectiveEntryDate)
}

// ODFIIdentificationField get the odfi number zero padded
func (bh *BatchHeader) ODFIIdentificationField() string {
	return bh.numericField(bh.ODFIIdentification, 8)
}

// BatchNumberField get the batch number zero padded
func (bh *BatchHeader) BatchNumberField() string {
	return bh.numericField(bh.BatchNumber, 7)
}

func (bh *BatchHeader) settlementDateField() string {
	return bh.alphaField(bh.settlementDate, 3)
}
<|endoftext|>