<commit_before>package apdex

import (
	"math/rand"
	"testing"
	"time"

	"github.com/xunyu/common"
	"github.com/xunyu/config"
)

type node map[string]interface{}

var (
	testRequest = []string{
		"{\"host\":\"example.a.com\",\"code\":200,\"request_time\":0.034",
		"{\"host\":\"example.b.com\",\"code\":200,\"request_time\":0.035",
		"{\"host\":\"example.c.com\",\"code\":200,\"request_time\":0.071",
		"{\"host\":\"example.a.com\",\"code\":200,\"request_time\":1.133",
		"{\"host\":\"example.b.com\",\"code\":200,\"request_time\":0.334",
		"{\"host\":\"example.a.com\",\"code\":200,\"request_time\":0.216",
		"{\"host\":\"example.b.com\",\"code\":200,\"request_time\":0.603",
		"{\"host\":\"example.a.com\",\"code\":200,\"request_time\":0.401",
		"{\"host\":\"example.b.com\",\"code\":200,\"request_time\":0.316",
		"{\"host\":\"example.a.com\",\"code\":200,\"request_time\":0.316",
		"{\"host\":\"example.b.com\",\"code\":200,\"request_time\":0.099",
		"{\"host\":\"example.a.com\",\"code\":200,\"request_time\":0.216",
		"{\"host\":\"example.b.com\",\"code\":200,\"request_time\":0.016",
		"{\"host\":\"example.b.com\",\"code\":200,\"request_time\":0.616",
		"{\"host\":\"example.a.com\",\"code\":200,\"request_time\":0.216",
		"{\"host\":\"example.b.com\",\"code\":200,\"request_time\":0.116",
		"{\"host\":\"example.a.com\",\"code\":200,\"request_time\":0.216",
		"{\"host\":\"example.b.com\",\"code\":200,\"request_time\":0.116",
		"{\"host\":\"example.a.com\",\"code\":200,\"request_time\":0.016",
	}
)

func genConfig() (*config.Config, error) {
	cfg, err := config.From(node{
		"rules": []node{
			node{
				"host":       "example.a.com",
				"satisfied":  200,
				"tolerating": 300,
			},
			node{
				"host":       "example.b.com",
				"satisfied":  100,
				"tolerating": 300,
			},
		},
	})

	return cfg, err
}

func genInput() <-chan common.DataInter {
	in := make(chan common.DataInter)
	go func() {
		t := time.Now()
		for j := 0; j < len(testRequest); j++ {
			rn := rand.Intn(45) * -1
			newT := t.Add(time.Second * time.Duration(rn))
			s := testRequest[j] + ",\"time\":\"" + newT.Format(time.RFC3339) + "\"}"
			in <- s
		}
		close(in)
	}()
	return in
}

func TestApdexConfig(t *testing.T) {
	cfg, err := genConfig()
	if nil != err {
		t.Error(err)
		t.FailNow()
	}

	ap, err := New(cfg)
	if nil != err {
		t.Error(err)
		t.FailNow()
	}
	t.Logf("instance apdex is %v", ap)
}

func TestApdexFilter(t *testing.T) {
	done := make(chan bool)
	in := genInput()
	out := filter(in)

	go func() {
		for data := range out {
			t.Logf("test apdex filter message: %v", data)
		}
		done <- true
	}()
	<-done
}

func filter(in <-chan common.DataInter) <-chan common.DataStr {
	cfg, _ := genConfig()
	ap, _ := New(cfg)

	out := make(chan common.DataStr, 1)

	go func(ap common.Pluginer) {
		ap.Filter(out)
		fc := ap.GetFilterChannel()
		for data := range in {
			fc <- data
		}
		close(fc)
	}(ap)

	return out
}
<commit_msg>Get rid of done & reduce time duration<commit_after>package apdex

import (
	"math/rand"
	"testing"
	"time"

	"github.com/xunyu/common"
	"github.com/xunyu/config"
)

type node map[string]interface{}

var (
	testRequest = []string{
		"{\"host\":\"example.a.com\",\"code\":200,\"request_time\":0.034",
		"{\"host\":\"example.b.com\",\"code\":200,\"request_time\":0.035",
		"{\"host\":\"example.c.com\",\"code\":200,\"request_time\":0.071",
		"{\"host\":\"example.a.com\",\"code\":200,\"request_time\":1.133",
		"{\"host\":\"example.b.com\",\"code\":200,\"request_time\":0.334",
		"{\"host\":\"example.a.com\",\"code\":200,\"request_time\":0.216",
		"{\"host\":\"example.b.com\",\"code\":200,\"request_time\":0.603",
		"{\"host\":\"example.a.com\",\"code\":200,\"request_time\":0.401",
		"{\"host\":\"example.b.com\",\"code\":200,\"request_time\":0.316",
		"{\"host\":\"example.a.com\",\"code\":200,\"request_time\":0.316",
		"{\"host\":\"example.b.com\",\"code\":200,\"request_time\":0.099",
		"{\"host\":\"example.a.com\",\"code\":200,\"request_time\":0.216",
		"{\"host\":\"example.b.com\",\"code\":200,\"request_time\":0.016",
		"{\"host\":\"example.b.com\",\"code\":200,\"request_time\":0.616",
		"{\"host\":\"example.a.com\",\"code\":200,\"request_time\":0.216",
		"{\"host\":\"example.b.com\",\"code\":200,\"request_time\":0.116",
		"{\"host\":\"example.a.com\",\"code\":200,\"request_time\":0.216",
		"{\"host\":\"example.b.com\",\"code\":200,\"request_time\":0.116",
		"{\"host\":\"example.a.com\",\"code\":200,\"request_time\":0.016",
	}
)

func genConfig() (*config.Config, error) {
	cfg, err := config.From(node{
		"rules": []node{
			node{
				"host":       "example.a.com",
				"satisfied":  200,
				"tolerating": 300,
			},
			node{
				"host":       "example.b.com",
				"satisfied":  100,
				"tolerating": 300,
			},
		},
	})

	return cfg, err
}

func genInput() <-chan common.DataInter {
	in := make(chan common.DataInter)
	go func() {
		t := time.Now()
		for j := 0; j < len(testRequest); j++ {
			rn := rand.Intn(30) * -1
			newT := t.Add(time.Second * time.Duration(rn))
			s := testRequest[j] + ",\"time\":\"" + newT.Format(time.RFC3339) + "\"}"
			in <- s
		}
		close(in)
	}()
	return in
}

func TestApdexConfig(t *testing.T) {
	cfg, err := genConfig()
	if nil != err {
		t.Error(err)
		t.FailNow()
	}

	ap, err := New(cfg)
	if nil != err {
		t.Error(err)
		t.FailNow()
	}
	t.Logf("instance apdex is %v", ap)
}

func TestApdexFilter(t *testing.T) {
	in := genInput()
	out := filter(in)

	go func() {
		for data := range out {
			t.Logf("test apdex filter message: %v", data)
		}
	}()
	<-time.After(time.Second * 30)
}

func filter(in <-chan common.DataInter) <-chan common.DataStr {
	cfg, _ := genConfig()
	ap, _ := New(cfg)

	out := make(chan common.DataStr, 1)

	go func(ap common.Pluginer) {
		ap.Filter(out)
		fc := ap.GetFilterChannel()
		for data := range in {
			fc <- data
		}
		close(fc)
	}(ap)

	return out
}
<|endoftext|>


<commit_before>package hal

import (
	"fmt"
	"log"
	"sync"
)

// RouterCTX holds the router's context, including input/output chans.
type RouterCTX struct {
	brokers map[string]Broker
	in      chan *Evt     // messages from brokers --> plugins
	out     chan *Evt     // messages from plugins --> brokers
	update  chan struct{} // to notify the router that the instance list changed
	mut     sync.Mutex
	init    sync.Once
}

var routerSingleton RouterCTX

// Router returns the singleton router context. The router is initialized
// on the first call to this function.
func Router() *RouterCTX {
	routerSingleton.init.Do(func() {
		routerSingleton.in = make(chan *Evt, 1000)
		routerSingleton.out = make(chan *Evt, 1000)
		routerSingleton.update = make(chan struct{}, 1)
		routerSingleton.brokers = make(map[string]Broker)
	})

	return &routerSingleton
}

// forward from one (go) channel to another
// TODO: figure out if this needs to check for closed channels, etc.
func forward(from, to chan *Evt) {
	for {
		select {
		case evt := <-from:
			to <- evt
		}
	}
}

// AddBroker adds a broker to the router and starts forwarding
// events between it and the router.
func (r *RouterCTX) AddBroker(b Broker) {
	r.mut.Lock()
	defer r.mut.Unlock()

	if _, exists := r.brokers[b.Name()]; exists {
		panic(fmt.Sprintf("BUG: broker '%s' added > 1 times.", b.Name()))
	}

	b2r := make(chan *Evt, 1000) // messages from the broker to the router

	// start the broker's event stream
	go b.Stream(b2r)

	// forward events from the broker to the router's input channel
	go forward(b2r, r.in)

	r.brokers[b.Name()] = b
}

func (r *RouterCTX) GetBroker(name string) Broker {
	r.mut.Lock()
	defer r.mut.Unlock()

	if broker, exists := r.brokers[name]; exists {
		return broker
	}

	return nil
}

func (r *RouterCTX) Brokers() []Broker {
	r.mut.Lock()
	defer r.mut.Unlock()

	out := make([]Broker, 0)
	for _, b := range r.brokers {
		out = append(out, b)
	}

	return out
}

// Route is the main method for the router. It blocks and should be run in a goroutine
// exactly once.
func (r *RouterCTX) Route() {
	for {
		select {
		case evt := <-r.in:
			if evt.Broker == nil {
				panic("BUG: received event with nil Broker. This breaks all the things!")
			}

			go r.processEvent(evt)
		}
	}
}

// processEvent processes one event and is intended to run in a goroutine.
func (r *RouterCTX) processEvent(evt *Evt) {
	var pname string // must be in the recovery handler's scope

	// get a snapshot of the instance list
	// TODO: keep an eye on the cost of copying this list for every message
	pr := PluginRegistry()
	instances := pr.InstanceList()

	// if a plugin panics, catch it & log it
	// TODO: report errors to a channel?
	defer func() {
		if r := recover(); r != nil {
			log.Printf("recovered panic in plugin %q\n", pname)
			log.Printf("panic: %q", r)
		}
	}()

	for _, inst := range instances {
		ibname := inst.Broker.Name()
		pname = inst.Plugin.Name // recovery handler ^ will pick this up in a panic

		// a plugin instance matches on broker, channel, and regex
		// first, check if the instance is attached to a specific broker or generic
		if ibname != evt.Broker.Name() && ibname != gBroker.Name() {
			continue
		}

		// if this is a generic broker instance and the event is marked as not
		// generic, skip it
		if ibname == gBroker.Name() && !evt.IsGeneric {
			continue
		}

		// check if it's the correct channel
		if evt.ChannelId != inst.ChannelId {
			continue
		}

		// finally, check message text against the regex
		if inst.Regex == "" || inst.regex.MatchString(evt.Body) {
			// this will copy the struct twice. It's intentional to avoid
			// mutating the evt between calls. The plugin func signature
			// forces the second copy.
			evtcpy := *evt

			// pass the plugin instance pointer to the plugin function so
			// it can access its fields for settings, etc.
			evtcpy.instance = inst

			// call the plugin function
			// this may block other plugins from processing the same event but
			// since it's already in a goroutine, other events won't be blocked
			inst.Func(evtcpy)
		}
	}
}
<commit_msg>add comments<commit_after>package hal

import (
	"fmt"
	"log"
	"sync"
)

// RouterCTX holds the router's context, including input/output chans.
type RouterCTX struct {
	brokers map[string]Broker
	in      chan *Evt     // messages from brokers --> plugins
	out     chan *Evt     // messages from plugins --> brokers
	update  chan struct{} // to notify the router that the instance list changed
	mut     sync.Mutex
	init    sync.Once
}

var routerSingleton RouterCTX

// Router returns the singleton router context. The router is initialized
// on the first call to this function.
func Router() *RouterCTX {
	routerSingleton.init.Do(func() {
		routerSingleton.in = make(chan *Evt, 1000)
		routerSingleton.out = make(chan *Evt, 1000)
		routerSingleton.update = make(chan struct{}, 1)
		routerSingleton.brokers = make(map[string]Broker)
	})

	return &routerSingleton
}

// forward from one (go) channel to another
// TODO: figure out if this needs to check for closed channels, etc.
func forward(from, to chan *Evt) {
	for {
		select {
		case evt := <-from:
			to <- evt
		}
	}
}

// AddBroker adds a broker to the router and starts forwarding
// events between it and the router.
func (r *RouterCTX) AddBroker(b Broker) {
	r.mut.Lock()
	defer r.mut.Unlock()

	if _, exists := r.brokers[b.Name()]; exists {
		panic(fmt.Sprintf("BUG: broker '%s' added > 1 times.", b.Name()))
	}

	b2r := make(chan *Evt, 1000) // messages from the broker to the router

	// start the broker's event stream
	go b.Stream(b2r)

	// forward events from the broker to the router's input channel
	go forward(b2r, r.in)

	r.brokers[b.Name()] = b
}

// GetBroker retrieves a broker handle by name.
func (r *RouterCTX) GetBroker(name string) Broker {
	r.mut.Lock()
	defer r.mut.Unlock()

	if broker, exists := r.brokers[name]; exists {
		return broker
	}

	return nil
}

// Brokers returns all brokers that have been added to the router.
func (r *RouterCTX) Brokers() []Broker {
	r.mut.Lock()
	defer r.mut.Unlock()

	out := make([]Broker, 0)
	for _, b := range r.brokers {
		out = append(out, b)
	}

	return out
}

// Route is the main method for the router. It blocks and should be run in a goroutine
// exactly once.
func (r *RouterCTX) Route() {
	for {
		select {
		case evt := <-r.in:
			if evt.Broker == nil {
				panic("BUG: received event with nil Broker. This breaks all the things!")
			}

			go r.processEvent(evt)
		}
	}
}

// processEvent processes one event and is intended to run in a goroutine.
func (r *RouterCTX) processEvent(evt *Evt) {
	var pname string // must be in the recovery handler's scope

	// get a snapshot of the instance list
	// TODO: keep an eye on the cost of copying this list for every message
	pr := PluginRegistry()
	instances := pr.InstanceList()

	// if a plugin panics, catch it & log it
	// TODO: report errors to a channel?
	defer func() {
		if r := recover(); r != nil {
			log.Printf("recovered panic in plugin %q\n", pname)
			log.Printf("panic: %q", r)
		}
	}()

	for _, inst := range instances {
		ibname := inst.Broker.Name()
		pname = inst.Plugin.Name // recovery handler ^ will pick this up in a panic

		// a plugin instance matches on broker, channel, and regex
		// first, check if the instance is attached to a specific broker or generic
		if ibname != evt.Broker.Name() && ibname != gBroker.Name() {
			continue
		}

		// if this is a generic broker instance and the event is marked as not
		// generic, skip it
		if ibname == gBroker.Name() && !evt.IsGeneric {
			continue
		}

		// check if it's the correct channel
		if evt.ChannelId != inst.ChannelId {
			continue
		}

		// finally, check message text against the regex
		if inst.Regex == "" || inst.regex.MatchString(evt.Body) {
			// this will copy the struct twice. It's intentional to avoid
			// mutating the evt between calls. The plugin func signature
			// forces the second copy.
			evtcpy := *evt

			// pass the plugin instance pointer to the plugin function so
			// it can access its fields for settings, etc.
			evtcpy.instance = inst

			// call the plugin function
			// this may block other plugins from processing the same event but
			// since it's already in a goroutine, other events won't be blocked
			inst.Func(evtcpy)
		}
	}
}
<|endoftext|>


<commit_before><commit_msg>Add PUSH_ARTIFACT to harbor_webhook_trigger<commit_after><|endoftext|>


<commit_before>package schedule

import (
	"errors"
	"github.com/0xfoo/punchcard/git"
	"github.com/0xfoo/punchcard/utils"
	"strings"
	"time"
)

const (
	SCHEDULE_WIDTH = 53
)

// TextSchedule creates commits over the past 365/366 days to build the given text.
// These commits will be created in the given git repo using the FileGenerator.
func TextSchedule(text string, repo git.Git, filegen utils.FileGenerator) error {
	messageBase := GetCommitMessageBase()
	days := GetDaysSinceNowMinusOneYear()
	commits, err := getTextCommitSchedule(text, days, messageBase)
	if err != nil {
		return err
	}
	for _, commit := range commits {
		repo.Add(filegen.CreateFile())
		repo.Commit(commit.Message, commit.DateTime.String())
	}
	return err
}

// getTextCommitSchedule returns a []Commit or an error if the given text will
// not fit onto the CommitSchedule.
func getTextCommitSchedule(text string, days []time.Time, messageBase []string) ([]Commit, error) {
	if !textFits(text) {
		return nil, errors.New("Text does not fit.")
	}
	schedule := buildTextCommitSchedule(days, text)
	commits := convertScheduleToCommits(schedule)
	return commits, nil
}

// textFits checks wether or not the text will fit onto a CommitSchedule.
func textFits(text string) bool {
	textWidth := getTextWidth(text)
	textIsNotToWide := textWidth <= SCHEDULE_WIDTH-2 // adjust for margins
	textIsNotEmpty := textWidth > 0
	return textIsNotEmpty && textIsNotToWide
}

// getTextWidth returns the width the text will need if put onto the CommitSchedule.
func getTextWidth(text string) int {
	width := 0
	for _, char := range strings.Split(text, "") {
		letter, _ := utils.TranslateLetter(char)
		width += len(letter[0]) + 1 // adjust for space between letters
	}
	return width - 1 // last letter does not need an extra space
}

func convertScheduleToCommits(schedule CommitSchedule) []Commit {
	var commits []Commit
	return commits
}

func buildTextCommitSchedule(days []time.Time, text string) CommitSchedule {
	schedule := BuildCommitSchedule(days)
	mapTextOntoCommitSchedule(text, &schedule)
	return schedule
}

func mapTextOntoCommitSchedule(text string, schedule *CommitSchedule) {
	letters := buildTextFields(text)
	rightShift := 0
	for _, fields := range letters {
		for rowIndex, row := range fields {
			for columnIndex, field := range row {
				schedule[rowIndex][columnIndex+rightShift].NumCommits = field
			}
		}
		rightShift += len(fields[0])
	}
}

// buildTextFields return [][][]int representation of the given text.
func buildTextFields(text string) [][][]int {
	var letters [][][]int
	if text == "" {
		return letters
	}
	space, _ := utils.TranslateLetter(" ")
	for _, char := range strings.Split(text, "") {
		letter, _ := utils.TranslateLetter(char)
		letters = append(letters, letter, space)
	}
	return letters[0 : len(letters)-1] // remove last extra space
}
<commit_msg>Implement convertScheduleToCommits and add docs.<commit_after>package schedule

import (
	"errors"
	"github.com/0xfoo/punchcard/git"
	"github.com/0xfoo/punchcard/utils"
	// "log"
	"strings"
	"time"
)

const (
	SCHEDULE_WIDTH = 53
)

// TextSchedule creates commits over the past 365/366 days to build the given text.
// These commits will be created in the given git repo using the FileGenerator.
func TextSchedule(text string, repo git.Git, filegen utils.FileGenerator) error {
	messageBase := GetCommitMessageBase()
	days := GetDaysSinceNowMinusOneYear()
	commits, err := getTextCommitSchedule(text, days, messageBase)
	if err != nil {
		return err
	}
	for _, commit := range commits {
		repo.Add(filegen.CreateFile())
		repo.Commit(commit.Message, commit.DateTime.String())
	}
	return err
}

// getTextCommitSchedule returns a []Commit or an error if the given text will
// not fit onto the CommitSchedule.
func getTextCommitSchedule(text string, days []time.Time, messageBase []string) ([]Commit, error) {
	if !textFits(text) {
		return nil, errors.New("Text does not fit.")
	}
	schedule := buildTextCommitSchedule(days, text)
	commits := convertScheduleToCommits(schedule)
	return commits, nil
}

// textFits checks wether or not the text will fit onto a CommitSchedule.
func textFits(text string) bool {
	textWidth := getTextWidth(text)
	textIsNotToWide := textWidth <= SCHEDULE_WIDTH-2 // adjust for margins
	textIsNotEmpty := textWidth > 0
	return textIsNotEmpty && textIsNotToWide
}

// getTextWidth returns the width the text will need if put onto the CommitSchedule.
func getTextWidth(text string) int {
	width := 0
	for _, char := range strings.Split(text, "") {
		letter, _ := utils.TranslateLetter(char)
		width += len(letter[0]) + 1 // adjust for space between letters
	}
	return width - 1 // last letter does not need an extra space
}

// convertScheduleToCommits creates NumCommits commits for every entry.
func convertScheduleToCommits(schedule CommitSchedule) []Commit {
	var commits []Commit
	messageBase := GetCommitMessageBase()
	for _, row := range schedule {
		for _, entry := range row {
			for commit := range GenerateRandomCommits(entry.DateTime, entry.NumCommits, messageBase) {
				commits = append(commits, commit)
			}
		}
	}
	return commits
}

// buildTextCommitSchedule returns a CommitSchedule representing the given text.
func buildTextCommitSchedule(days []time.Time, text string) CommitSchedule {
	schedule := BuildCommitSchedule(days)
	mapTextOntoCommitSchedule(text, &schedule)
	return schedule
}

// mapTextOntoCommitSchedule will put text onto a CommitSchedule.
func mapTextOntoCommitSchedule(text string, schedule *CommitSchedule) {
	letters := buildTextFields(text)
	rightShift := 0
	for _, fields := range letters {
		for rowIndex, row := range fields {
			for columnIndex, field := range row {
				schedule[rowIndex][columnIndex+rightShift].NumCommits = field
			}
		}
		rightShift += len(fields[0])
	}
}

// buildTextFields return [][][]int representation of the given text.
func buildTextFields(text string) [][][]int {
	var letters [][][]int
	if text == "" {
		return letters
	}
	space, _ := utils.TranslateLetter(" ")
	for _, char := range strings.Split(text, "") {
		letter, _ := utils.TranslateLetter(char)
		letters = append(letters, letter, space)
	}
	return letters[0 : len(letters)-1] // remove last extra space
}
<|endoftext|>


<commit_before>package scoring

// An entry represents a path and a score.
type Entry struct {
	Path  string
	Score *Score
}

// Update the score for an entry.
func (e *Entry) UpdateScore() {
	e.Score.Update()
}

// Calculates the score for an entry.
func (e *Entry) CalculateScore() float64 {
	return e.Score.Calculate()
}

func NewEntry(path string) *Entry {
	return &Entry{path, NewScore()}
}
<commit_msg>Document scoring.NewEntry<commit_after>package scoring

// An entry represents a path and a score.
type Entry struct {
	Path  string
	Score *Score
}

// Update the score for an entry.
func (e *Entry) UpdateScore() {
	e.Score.Update()
}

// Calculates the score for an entry.
func (e *Entry) CalculateScore() float64 {
	return e.Score.Calculate()
}

// Create a new entry with the specified path. The score is created with
// NewScore.
func NewEntry(path string) *Entry {
	return &Entry{path, NewScore()}
}
<|endoftext|>


<commit_before>package suss

import (
	"encoding/binary"
	"math"
	"math/rand"
)

type SliceGen struct {
	Avg int
	Min int
	Max int

	g *Generator
}

func (g *Generator) Slice() *SliceGen {
	return &SliceGen{
		Avg: 50,
		Min: 0,
		Max: int(^uint(0) >> 1),
		g:   g,
	}
}

func (s *SliceGen) Gen(f func()) {
	// The intuitive way to turn an infinite bytestream into a
	// slice would be to grab a value at the beginning
	// and then generate that number of elements
	// However, this gets in the way of shrinking
	//
	// Instead, for each element, grab a byte
	// asking us if we want more elements.
	// That way, deleting a span in the byte
	// stream turns into the element not being
	// added.
	l := uint64(0)
	stopvalue := 1 - (1.0 / (1 + float64(s.Avg)))
	if s.Min < 0 {
		panic("invalid min slice length")
	}
	min := uint64(s.Min)
	max := uint64(s.Max)
	for l < max {
		s.g.StartExample()
		more := s.g.biasBool(stopvalue)
		if !more && l >= min {
			s.g.EndExample()
			return
		}
		l++
		f()
		s.g.EndExample()
	}
}

func (g *Generator) Bool() bool {
	b := g.Draw(1, Uniform)
	return b[0]&1 == 1
}

func (g *Generator) Float64() float64 {
	g.StartExample()
	fbits := g.Draw(10, func(r *rand.Rand, n int) []byte {
		if n != 10 {
			panic("bad float size")
		}
		flavor := r.Intn(5)
		var f float64
		switch flavor {
		case 0:
			f = math.NaN()
		case 1:
			f = math.Inf(0)
		case 2:
			f = math.Inf(-1)
		case 3:
			// TODO incorporate evil floats from hypothesis
			f = 0
		default:
			f = r.Float64()
			f *= math.MaxFloat64
			if r.Intn(2) == 1 {
				f = -f
			}
		}
		b := encodefloat64(f)
		return b[:]
	})
	g.EndExample()
	f, invalid := decodefloat64(fbits)
	if invalid {
		g.Invalid()
	}
	return f
}

// encodefloat64 attempts to encode a floating point number
// so that its lexicographical ordering follows human intuition
//
// Design goals were:
// - Integers are simpler than fractionals
// - positive numbers are simpler than negative ones
// - exponents of smaller magnitude are simpler, regardless of sign
// - 0 is the simplest number, 1 is the second most simple number
func encodefloat64(f float64) [10]byte {
	var b [10]byte
	bits := math.Float64bits(f)
	// encode the sign bit as a single byte
	b[0] = byte((bits & (1 << 63)) >> 63)

	// for the mantissa, we want simpler fractions
	// This means we get numbers that require fewer
	// digits to print it
	// Encoding as a little endian number
	// makes shrinking go towards a number with
	// fewer significant digits
	mant := bits & (^uint64(0) >> (64 - 52))
	binary.LittleEndian.PutUint64(b[1:], mant)

	// if the exponent is 0, that means this value
	// is a zero. don't unbias the exponent in this case
	// TODO: handle subnormals so that they're more complex
	sexp := int16((bits >> 52) & 0x7ff)
	var exp uint16
	if sexp != 0 {
		sexp -= 1023
		// if exponent is positive, bias it +1
		// so that an exponent of 1 becomes 0
		// This keeps the invariant that 0 is
		// simpler than 1
		if sexp >= 0 {
			exp = uint16(sexp) + 1
		} else {
			// for negative exponents
			// use signed regular integer
			// This makes -1 simpler than -2
			// when interpreted as a byte stream
			// the sign keeps the invariant that
			// integers are simpler than fractionals
			sexp *= -1
			exp = uint16(sexp)
			exp ^= (1 << 15)
		}
	}
	binary.BigEndian.PutUint16(b[8:], exp)
	return b
}

func decodefloat64(b []byte) (float64, bool) {
	fbits := uint64(0)
	sign := b[0]
	if sign != 0 && sign != 1 {
		return 0, true
	}
	fbits = uint64(sign) << 63
	exp := binary.BigEndian.Uint16(b[8:])
	if exp&(1<<15) != 0 {
		// this is a signed exponent
		// clear the sign bit
		sexp := int16(exp & (^uint16(0) >> 1))
		// make into negative number
		sexp *= -1
		// unbias
		sexp += 1023
		exp = uint16(sexp)
	} else if exp != 0 {
		// positive exponent
		exp -= 1
		exp += 1023
	}
	fbits ^= uint64(exp) << 52
	// mantissa only take 7 bytes in our binary packing
	// but binary only lets us read in chunks of 8
	// copy the mantissa value into an empty array
	// and then decode to make sure that we don't
	var mb [8]byte
	copy(mb[:], b[1:8])
	mant := binary.LittleEndian.Uint64(mb[:])
	fbits ^= mant & (^uint64(0) >> (64 - 52))
	return math.Float64frombits(fbits), false
}

func (g *Generator) Uint64() uint64 {
	g.StartExample()
	f := g.Draw(8, Uniform)
	g.EndExample()
	return binary.BigEndian.Uint64(f)
}

func (g *Generator) Int16() int16 {
	g.StartExample()
	f := g.Draw(8, Uniform)
	g.EndExample()
	return int16(binary.BigEndian.Uint16(f))
}

func (g *Generator) Byte() byte {
	g.StartExample()
	defer g.EndExample()
	return g.Draw(1, Uniform)[0]
}

func (g *Generator) biasBool(f float64) bool {
	bits := g.Draw(1, func(r *rand.Rand, n int) []byte {
		roll := r.Float64()
		b := byte(0)
		if roll < f {
			b = 1
		}
		return []byte{b}
	})
	return bits[0] != 0
}
<commit_msg>define an ordering for inf, nan and subnormals<commit_after>package suss

import (
	"encoding/binary"
	"math"
	"math/rand"
)

type SliceGen struct {
	Avg int
	Min int
	Max int

	g *Generator
}

func (g *Generator) Slice() *SliceGen {
	return &SliceGen{
		Avg: 50,
		Min: 0,
		Max: int(^uint(0) >> 1),
		g:   g,
	}
}

func (s *SliceGen) Gen(f func()) {
	// The intuitive way to turn an infinite bytestream into a
	// slice would be to grab a value at the beginning
	// and then generate that number of elements
	// However, this gets in the way of shrinking
	//
	// Instead, for each element, grab a byte
	// asking us if we want more elements.
	// That way, deleting a span in the byte
	// stream turns into the element not being
	// added.
	l := uint64(0)
	stopvalue := 1 - (1.0 / (1 + float64(s.Avg)))
	if s.Min < 0 {
		panic("invalid min slice length")
	}
	min := uint64(s.Min)
	max := uint64(s.Max)
	for l < max {
		s.g.StartExample()
		more := s.g.biasBool(stopvalue)
		if !more && l >= min {
			s.g.EndExample()
			return
		}
		l++
		f()
		s.g.EndExample()
	}
}

func (g *Generator) Bool() bool {
	b := g.Draw(1, Uniform)
	return b[0]&1 == 1
}

func (g *Generator) Float64() float64 {
	g.StartExample()
	fbits := g.Draw(10, func(r *rand.Rand, n int) []byte {
		if n != 10 {
			panic("bad float size")
		}
		flavor := r.Intn(5)
		var f float64
		switch flavor {
		case 0:
			f = math.NaN()
		case 1:
			f = math.Inf(0)
		case 2:
			f = math.Inf(-1)
		case 3:
			// TODO incorporate evil floats from hypothesis
			f = 0
		default:
			f = r.Float64()
			f *= math.MaxFloat64
			if r.Intn(2) == 1 {
				f = -f
			}
		}
		b := encodefloat64(f)
		return b[:]
	})
	g.EndExample()
	f, invalid := decodefloat64(fbits)
	if invalid {
		g.Invalid()
	}
	return f
}

// encodefloat64 attempts to encode a floating point number
// so that its lexicographical ordering follows human intuition
//
// Design goals were:
// - Integers are simpler than fractionals
// - positive numbers are simpler than negative ones
// - exponents of smaller magnitude are simpler, regardless of sign
// - 0 is the simplest number, 1 is the second most simple number
func encodefloat64(f float64) [10]byte {
	var b [10]byte
	bits := math.Float64bits(f)
	// encode the sign bit as a single byte
	b[0] = byte((bits & (1 << 63)) >> 63)

	// for the mantissa, we want simpler fractions
	// This means we get numbers that require fewer
	// digits to print it
	// Encoding as a little endian number
	// makes shrinking go towards a number with
	// fewer significant digits
	mant := bits & (^uint64(0) >> (64 - 52))
	binary.LittleEndian.PutUint64(b[1:], mant)

	// if the exponent is 0, that means this value
	// is a zero. don't unbias the exponent in this case
	// TODO: handle subnormals so that they're more complex
	sexp := int16((bits >> 52) & 0x7ff)
	var exp uint16
	if sexp == 0 {
		if mant != 0 {
			// subnormal number, use the extra range we get from
			// int16 to signal this
			sexp = 1024
			exp = uint16(sexp)
			exp ^= (1 << 15)
		}
	} else if exp == 0x7ff {
		// infinity and NaN, they're more complex that negative
		// exponent and subnormals
		sexp = 1025
		exp = uint16(sexp)
		exp ^= (1 << 15)

	} else {
		// regular exponent
		// unbias
		sexp -= 1023
		// if exponent is positive, bias it +1
		// so that an exponent of 1 becomes 0
		// This keeps the invariant that 0 is
		// simpler than 1
		if sexp >= 0 {
			exp = uint16(sexp) + 1
		} else {
			// for negative exponents
			// use signed regular integer
			// This makes -1 simpler than -2
			// when interpreted as a byte stream
			// the sign keeps the invariant that
			// integers are simpler than fractionals
			sexp *= -1
			exp = uint16(sexp)
			exp ^= (1 << 15)
		}
	}
	binary.BigEndian.PutUint16(b[8:], exp)
	return b
}

func decodefloat64(b []byte) (float64, bool) {
	fbits := uint64(0)
	sign := b[0]
	if sign != 0 && sign != 1 {
		return 0, true
	}
	fbits = uint64(sign) << 63
	// mantissa only take 7 bytes in our binary packing
	// but binary only lets us read in chunks of 8
	// copy the mantissa value into an empty array
	// and then decode to make sure that we don't
	var mb [8]byte
	copy(mb[:], b[1:8])
	mant := binary.LittleEndian.Uint64(mb[:])

	exp := binary.BigEndian.Uint16(b[8:])
	if exp&(1<<15) != 0 {
		// this is a signed exponent
		// clear the sign bit
		sexp := int16(exp & (^uint16(0) >> 1))
		if sexp == 1024 {
			exp = 0
		} else if sexp == 1025 {
			exp = 0x7ff
		} else {
			// this is a regular negative exponent
			// make into negative number
			sexp *= -1
			// bias
			sexp += 1023
			exp = uint16(sexp)
		}
	} else if exp != 0 {
		// positive exponent
		exp -= 1
		exp += 1023
	} else if mant != 0 {
		mant = 0
	}
	fbits ^= mant & (^uint64(0) >> (64 - 52))
	fbits ^= uint64(exp) << 52
	return math.Float64frombits(fbits), false
}

func (g *Generator) Uint64() uint64 {
	g.StartExample()
	f := g.Draw(8, Uniform)
	g.EndExample()
	return binary.BigEndian.Uint64(f)
}

func (g *Generator) Int16() int16 {
	g.StartExample()
	f := g.Draw(2, Uniform)
	g.EndExample()
	return int16(binary.BigEndian.Uint16(f))
}

func (g *Generator) Byte() byte {
	g.StartExample()
	defer g.EndExample()
	return g.Draw(1, Uniform)[0]
}

func (g *Generator) biasBool(f float64) bool {
	bits := g.Draw(1, func(r *rand.Rand, n int) []byte {
		roll := r.Float64()
		b := byte(0)
		if roll < f {
			b = 1
		}
		return []byte{b}
	})
	return bits[0] != 0
}
<|endoftext|>


<commit_before>package scrape

import (
	"errors"
	"fmt"
	"strings"

	"github.com/PuerkitoBio/goquery"
)

type dummyPaginator struct {
}

func (p dummyPaginator) NextPage(uri string, doc *goquery.Selection) (string, error) {
	return "", nil
}

// DividePageBySelector returns a function that divides a page into blocks by
// CSS selector.  Each element in the page with the given selector is treated
// as a new block.
func DividePageBySelector(sel string) DividePageFunc {
	ret := func(doc *goquery.Selection) []*goquery.Selection {
		sels := []*goquery.Selection{}
		doc.Find(sel).Each(func(i int, s *goquery.Selection) {
			sels = append(sels, s)
		})

		return sels
	}
	return ret
}

var errNoSelectors = errors.New("No selectors found")

func intersectionFL(sel *goquery.Selection) *goquery.Selection {
	first := sel.First()
	last := sel.Last()
	intersection := first.Parents().Intersection(last.Parents())
	return intersection
}

func attrOrDataValue(s *goquery.Selection) (value string) {
	if s.Length() == 0 {
		return "Empty Selection"
	}
	attr, exists := s.Attr("class")
	if exists && attr != "" { //in some cases tag is invalid f.e. <tr class>
		return fmt.Sprintf(".%s", strings.Replace(strings.TrimSpace(attr), " ", ".", -1))
	}
	attr, exists = s.Attr("id")

	if exists && attr != "" {
		return fmt.Sprintf("#%s", attr)
	}
	//if len(s.Nodes)>0 {
	return s.Nodes[0].Data
	//}
}

func findIntersection(doc *goquery.Selection, selectors []string) (*goquery.Selection, error) {
	var intersection *goquery.Selection
	for i, f := range selectors {
		//err := validate.Struct(f)
		//if err != nil {
		//	return nil, err
		//}
		sel := doc.Find(f)
		//logger.Println(f, sel.Length())
		//col.genAttrFieldName(f.Name, sel)
		if sel.Length() > 0 { //don't add selectors to intersection if length is 0. Otherwise the whole intersection returns No selectors error
			if i == 0 {
				intersection = intersectionFL(sel)
			} else {
				intersection = intersection.Intersection(intersectionFL(sel))
			}
		}
	}
	//logger.Println(attrOrDataValue(intersection))
	if intersection == nil || intersection.Length() == 0 {
		return nil, errNoSelectors
	}
	intersectionWithParent := fmt.Sprintf("%s>%s",
		attrOrDataValue(intersection.Parent()),
		attrOrDataValue(intersection))
	//logger.Println(intersectionWithParent)
	items := doc.Find(intersectionWithParent)
	//return intersectionWithParent, nil
	//logger.Println(items.Length())

	var inter1 *goquery.Selection
	if items.Length() == 1 {
		inter1 = items.Children()
		//sel = fmt.Sprintf("%s>%s>%s",
		//	attrOrDataValue(intersection.Parent()),
		//	attrOrDataValue(intersection),
		//	attrOrDataValue(intersection.Children()))

	}
	if items.Length() > 1 {
		inter1 = items
		//sel = intersectionWithParent
	}
	return inter1, nil
}

func DividePageByIntersection(selectors []string) DividePageFunc {
	ret := func(doc *goquery.Selection) []*goquery.Selection {
		sels := []*goquery.Selection{}
		//doc.Find(sel).Each(func(i int, s *goquery.Selection) {
		sel, err := findIntersection(doc, selectors)
		if err != nil {
			logger.Println(err)
		}
		sel.Each(func(i int, s *goquery.Selection) {
			sels = append(sels, s)
		//	logger.Println(attrOrDataValue(s))
		})

		return sels
	}
	return ret
}
<commit_msg>get ancestor func added<commit_after>package scrape

import (
	"errors"
	"fmt"
	"strings"

	"github.com/PuerkitoBio/goquery"
)

type dummyPaginator struct {
}

func (p dummyPaginator) NextPage(uri string, doc *goquery.Selection) (string, error) {
	return "", nil
}

// DividePageBySelector returns a function that divides a page into blocks by
// CSS selector.  Each element in the page with the given selector is treated
// as a new block.
func DividePageBySelector(sel string) DividePageFunc {
	ret := func(doc *goquery.Selection) []*goquery.Selection {
		sels := []*goquery.Selection{}
		doc.Find(sel).Each(func(i int, s *goquery.Selection) {
			sels = append(sels, s)
		})

		return sels
	}
	return ret
}

var errNoSelectors = errors.New("No selectors found")

func intersectionFL(sel *goquery.Selection) *goquery.Selection {
	first := sel.First()
	last := sel.Last()
	intersection := first.Parents().Intersection(last.Parents())
	return intersection
}

func attrOrDataValue(s *goquery.Selection) (value string) {
	if s.Length() == 0 {
		return "Empty Selection"
	}
	attr, exists := s.Attr("class")
	if exists && attr != "" { //in some cases tag is invalid f.e. <tr class>
		return fmt.Sprintf(".%s", strings.Replace(strings.TrimSpace(attr), " ", ".", -1))
	}
	attr, exists = s.Attr("id")

	if exists && attr != "" {
		return fmt.Sprintf("#%s", attr)
	}
	//if len(s.Nodes)>0 {
	return s.Nodes[0].Data
	//}
}

func findIntersection(doc *goquery.Selection, selectors []string) (*goquery.Selection, error) {
	var intersection *goquery.Selection
	for i, f := range selectors {
		//err := validate.Struct(f)
		//if err != nil {
		//	return nil, err
		//}
		sel := doc.Find(f)
		//logger.Println(f, sel.Length())
		//col.genAttrFieldName(f.Name, sel)
		if sel.Length() > 0 { //don't add selectors to intersection if length is 0. Otherwise the whole intersection returns No selectors error
			if i == 0 {
				intersection = intersectionFL(sel)
			} else {
				intersection = intersection.Intersection(intersectionFL(sel))
			}
		}
	}
	//logger.Println(attrOrDataValue(intersection))
	if intersection == nil || intersection.Length() == 0 {
		return nil, errNoSelectors
	}
	intersectionWithParent := fmt.Sprintf("%s>%s",
		attrOrDataValue(intersection.Parent()),
		attrOrDataValue(intersection))
	//logger.Println(intersectionWithParent)
	items := doc.Find(intersectionWithParent)
	//return intersectionWithParent, nil
	//logger.Println(items.Length())

	var inter1 *goquery.Selection
	if items.Length() == 1 {
		inter1 = items.Children()
		//sel = fmt.Sprintf("%s>%s>%s",
		//	attrOrDataValue(intersection.Parent()),
		//	attrOrDataValue(intersection),
		//	attrOrDataValue(intersection.Children()))

	}
	if items.Length() > 1 {
		inter1 = items
		//sel = intersectionWithParent
	}
	return inter1, nil
}

func DividePageByIntersection(selectors []string) DividePageFunc {
	ret := func(doc *goquery.Selection) []*goquery.Selection {
		sels := []*goquery.Selection{}
		//doc.Find(sel).Each(func(i int, s *goquery.Selection) {
		sel, err := getCommonAncestor(doc, selectors)
		//sel, err = findIntersection(doc, selectors)
		if err != nil {
			logger.Println(err)
		}

		sel.Each(func(i int, s *goquery.Selection) {
			sels = append(sels, s)
			//	logger.Println(attrOrDataValue(s))
		})

		return sels
	}
	return ret
}

func getCommonAncestor(doc *goquery.Selection, selectors []string) (*goquery.Selection, error) {
	if len(selectors) > 1 {
		bFound := false
		selectorAncestor := doc.Find(selectors[0]).First().Parent()
		selectorsSlice := selectors[1:]
		for !bFound {
			logger.Println(goquery.NodeName(selectorAncestor))
			for _, f := range selectorsSlice {
				sel := doc.Find(f).First()
				logger.Println(sel.Text())
				sel = sel.ParentsUntilSelection(selectorAncestor).Last()
				logger.Println(goquery.NodeName(sel))
				//check last node.. if it equal html its mean that first selector's parent
				//not found
				if goquery.NodeName(sel) == "html" {
					selectorAncestor = doc.FindSelection(selectorAncestor.Parent().First())
					bFound = false
					break
				}
				bFound = true
			}
		}
		intersectionWithParent := fmt.Sprintf("%s>%s",
			attrOrDataValue(selectorAncestor.Parent()),
			attrOrDataValue(selectorAncestor))

		items := doc.Find(intersectionWithParent)

		var inter1 *goquery.Selection
		if items.Length() == 1 {
			inter1 = items.Children()
		}
		if items.Length() > 1 {
			inter1 = items
		}
		return inter1, nil
	}
	return nil, errors.New("It seems current selectors has no common ancestor")
}
<|endoftext|>


<commit_before>package bench

import (
	"bytes"
	stdjson "encoding/json"
	"io"
	"io/ioutil"
	"os"
	"testing"

	jsoniter "github.com/json-iterator/go"
	"github.com/ugorji/go/codec"
	"github.com/urso/go-structform/cborl"
	"github.com/urso/go-structform/gotype"
	"github.com/urso/go-structform/json"
	"github.com/urso/go-structform/ubjson"
)

type encoderFactory func(io.Writer) func(interface{}) error
type decoderFactory func(io.Reader) func(interface{}) error
type decoderFactoryBuf func([]byte) func(interface{}) error

func BenchmarkDecodeBeatsEventsFile(b *testing.B) {
	// b.Run("go-codec", makeBenchmarkDecodeBeatsEvents(gocodecJSONDecoder))
	b.Run("structform", makeBenchmarkDecodeBeatsEvents(structformJSONDecoder(0)))
	b.Run("structform-keycache", makeBenchmarkDecodeBeatsEvents(structformJSONDecoder(1000)))
	b.Run("std-json", makeBenchmarkDecodeBeatsEvents(stdJSONDecoder))

	// fails with panic
	// b.Run("jsoniter", makeBenchmarkDecodeBeatsEvents(jsoniterDecoder))
}

func BenchmarkDecodeBeatsEventsMem(b *testing.B) {
	b.Run("structform", makeBenchmarkDecodeBeatsEventsBuffered(structformJSONBufDecoder(0)))
	b.Run("structform-keycache", makeBenchmarkDecodeBeatsEventsBuffered(structformJSONBufDecoder(1000)))
	b.Run("std-json", makeBenchmarkDecodeBeatsEventsBuffered(stdJSONBufDecoder))

	// fails with panic
	// b.Run("jsoniter", makeBenchmarkDecodeBeatsEventsBuffered(jsoniterBufDecoder))
}

func BenchmarkEncodeBeatsEvents(b *testing.B) {
	events := loadBeatsEvents()
	b.Run("std-json", makeBenchmarkEncodeEvents(stdJSONEncoder, events))
	b.Run("structform-json", makeBenchmarkEncodeEvents(structformJSONEncoder, events))
	b.Run("structform-ubjson", makeBenchmarkEncodeEvents(structformUBJSONEncoder, events))
	b.Run("structform-cborl", makeBenchmarkEncodeEvents(structformCBORLEncoder, events))
}

func stdJSONEncoder(w io.Writer) func(interface{}) error {
	enc := stdjson.NewEncoder(w)
	return enc.Encode
}

func stdJSONDecoder(r io.Reader) func(interface{}) error {
	dec := stdjson.NewDecoder(r)
	return dec.Decode
}

func stdJSONBufDecoder(b []byte) func(interface{}) error {
	return stdJSONDecoder(bytes.NewReader(b))
}

func gocodecJSONDecoder(r io.Reader) func(interface{}) error {
	h := &codec.JsonHandle{}
	dec := codec.NewDecoder(r, h)
	return dec.Decode
}

func jsoniterDecoder(r io.Reader) func(interface{}) error {
	iter := jsoniter.Parse(r, 4096)
	return func(v interface{}) error {
		iter.ReadVal(v)
		return iter.Error
	}
}

func jsoniterBufDecoder(b []byte) func(interface{}) error {
	iter := jsoniter.ParseBytes(b)
	return func(v interface{}) error {
		iter.ReadVal(v)
		return iter.Error
	}
}

func structformJSONEncoder(w io.Writer) func(interface{}) error {
	vs := json.NewVisitor(w)
	folder := gotype.NewIterator(vs)
	return folder.Fold
}

func structformUBJSONEncoder(w io.Writer) func(interface{}) error {
	vs := ubjson.NewVisitor(w)
	folder := gotype.NewIterator(vs)
	return folder.Fold
}

func structformCBORLEncoder(w io.Writer) func(interface{}) error {
	vs := cborl.NewVisitor(w)
	folder := gotype.NewIterator(vs)
	return folder.Fold
}

func structformJSONDecoder(keyCache int) func(io.Reader) func(interface{}) error {
	return func(r io.Reader) func(interface{}) error {
		u, _ := gotype.NewUnfolder(nil)
		dec := json.NewDecoder(r, 2*4096, u)
		return makeStructformJSONDecoder(u, dec, keyCache)
	}
}

func structformJSONBufDecoder(keyCache int) func([]byte) func(interface{}) error {
	return func(b []byte) func(interface{}) error {
		u, _ := gotype.NewUnfolder(nil)
		dec := json.NewBytesDecoder(b, u)
		return makeStructformJSONDecoder(u, dec, keyCache)
	}
}

func makeStructformJSONDecoder(
	u *gotype.Unfolder,
	d *json.Decoder,
	keyCache int,
) func(interface{}) error {
	if keyCache > 0 {
		u.EnableKeyCache(keyCache)
	}
	return func(v interface{}) error {
		if err := u.SetTarget(v); err != nil {
			return err
		}
		return d.Next()
	}
}

func makeBenchmarkDecodeBeatsEvents(factory decoderFactory) func(*testing.B) {
	return func(b *testing.B) {
		for i := 0; i < b.N; i++ {
			func() {
				file, err := os.Open("files/beats_events.json")
				// file, err := os.Open("test.json")
				if err != nil {
					b.Error(err)
					return
				}

				defer file.Close()

				decode := factory(file)
				for {
					var to map[string]interface{}
					if err := decode(&to); err != nil {
						if err != io.EOF {
							b.Error(err)
						}
						return
					}
				}
			}()
		}
	}
}

func makeBenchmarkDecodeBeatsEventsBuffered(factory decoderFactoryBuf) func(*testing.B) {
	content, err := ioutil.ReadFile("files/beats_events.json")
	if err != nil {
		panic(err)
	}

	return func(b *testing.B) {
		b.SetBytes(int64(len(content)))

		for i := 0; i < b.N; i++ {
			decode := factory(content)
			for {
				var to map[string]interface{}
				if err := decode(&to); err != nil {
					if err != io.EOF {
						b.Error(err)
					}
					break
				}
			}
		}
	}
}

func makeBenchmarkEncodeEvents(factory encoderFactory, events []map[string]interface{}) func(*testing.B) {
	var buf bytes.Buffer
	buf.Grow(16 * 1024)
	encode := factory(&buf)

	return func(b *testing.B) {
		for i := 0; i < b.N; i++ {
			var written int64

			for _, event := range events {
				buf.Reset()
				if err := encode(event); err != nil {
					b.Error(err)
					return
				}
				written += int64(buf.Len())
			}
			b.SetBytes(written)
		}
	}
}

func loadBeatsEvents() []map[string]interface{} {
	content, err := ioutil.ReadFile("files/beats_events.json")
	if err != nil {
		panic(err)
	}

	var events []map[string]interface{}
	dec := stdjson.NewDecoder(bytes.NewReader(content))
	for {
		var e map[string]interface{}
		if err := dec.Decode(&e); err != nil {
			if err == io.EOF {
				break
			}

			panic(err)
		}

		events = append(events, e)
	}

	return events
}
<commit_msg>fix benchmark build<commit_after>package bench

import (
	"bytes"
	stdjson "encoding/json"
	"io"
	"io/ioutil"
	"os"
	"testing"

	jsoniter "github.com/json-iterator/go"
	"github.com/ugorji/go/codec"
	"github.com/urso/go-structform/cborl"
	"github.com/urso/go-structform/gotype"
	"github.com/urso/go-structform/json"
	"github.com/urso/go-structform/ubjson"
)

type encoderFactory func(io.Writer) func(interface{}) error
type decoderFactory func(io.Reader) func(interface{}) error
type decoderFactoryBuf func([]byte) func(interface{}) error

func BenchmarkDecodeBeatsEventsFile(b *testing.B) {
	// b.Run("go-codec", makeBenchmarkDecodeBeatsEvents(gocodecJSONDecoder))
	b.Run("structform", makeBenchmarkDecodeBeatsEvents(structformJSONDecoder(0)))
	b.Run("structform-keycache", makeBenchmarkDecodeBeatsEvents(structformJSONDecoder(1000)))
	b.Run("std-json", makeBenchmarkDecodeBeatsEvents(stdJSONDecoder))

	// fails with panic
	// b.Run("jsoniter", makeBenchmarkDecodeBeatsEvents(jsoniterDecoder))
}

func BenchmarkDecodeBeatsEventsMem(b *testing.B) {
	b.Run("structform", makeBenchmarkDecodeBeatsEventsBuffered(structformJSONBufDecoder(0)))
	b.Run("structform-keycache", makeBenchmarkDecodeBeatsEventsBuffered(structformJSONBufDecoder(1000)))
	b.Run("std-json", makeBenchmarkDecodeBeatsEventsBuffered(stdJSONBufDecoder))

	// fails with panic
	// b.Run("jsoniter", makeBenchmarkDecodeBeatsEventsBuffered(jsoniterBufDecoder))
}

func BenchmarkEncodeBeatsEvents(b *testing.B) {
	events := loadBeatsEvents()
	b.Run("std-json", makeBenchmarkEncodeEvents(stdJSONEncoder, events))
	b.Run("structform-json", makeBenchmarkEncodeEvents(structformJSONEncoder, events))
	b.Run("structform-ubjson", makeBenchmarkEncodeEvents(structformUBJSONEncoder, events))
	b.Run("structform-cborl", makeBenchmarkEncodeEvents(structformCBORLEncoder, events))
}

func stdJSONEncoder(w io.Writer) func(interface{}) error {
	enc := stdjson.NewEncoder(w)
	return enc.Encode
}

func stdJSONDecoder(r io.Reader) func(interface{}) error {
	dec := stdjson.NewDecoder(r)
	return dec.Decode
}

func stdJSONBufDecoder(b []byte) func(interface{}) error {
	return stdJSONDecoder(bytes.NewReader(b))
}

func gocodecJSONDecoder(r io.Reader) func(interface{}) error {
	h := &codec.JsonHandle{}
	dec := codec.NewDecoder(r, h)
	return dec.Decode
}

func jsoniterDecoder(r io.Reader) func(interface{}) error {
	iter := jsoniter.Parse(r, 4096)
	return func(v interface{}) error {
		iter.ReadVal(v)
		return iter.Error
	}
}

func jsoniterBufDecoder(b []byte) func(interface{}) error {
	iter := jsoniter.ParseBytes(b)
	return func(v interface{}) error {
		iter.ReadVal(v)
		return iter.Error
	}
}

func structformJSONEncoder(w io.Writer) func(interface{}) error {
	vs := json.NewVisitor(w)
	folder, _ := gotype.NewIterator(vs)
	return folder.Fold
}

func structformUBJSONEncoder(w io.Writer) func(interface{}) error {
	vs := ubjson.NewVisitor(w)
	folder, _ := gotype.NewIterator(vs)
	return folder.Fold
}

func structformCBORLEncoder(w io.Writer) func(interface{}) error {
	vs := cborl.NewVisitor(w)
	folder, _ := gotype.NewIterator(vs)
	return folder.Fold
}

func structformJSONDecoder(keyCache int) func(io.Reader) func(interface{}) error {
	return func(r io.Reader) func(interface{}) error {
		u, _ := gotype.NewUnfolder(nil)
		dec := json.NewDecoder(r, 2*4096, u)
		return makeStructformJSONDecoder(u, dec, keyCache)
	}
}

func structformJSONBufDecoder(keyCache int) func([]byte) func(interface{}) error {
	return func(b []byte) func(interface{}) error {
		u, _ := gotype.NewUnfolder(nil)
		dec := json.NewBytesDecoder(b, u)
		return makeStructformJSONDecoder(u, dec, keyCache)
	}
}

func makeStructformJSONDecoder(
	u *gotype.Unfolder,
	d *json.Decoder,
	keyCache int,
) func(interface{}) error {
	if keyCache > 0 {
		u.EnableKeyCache(keyCache)
	}
	return func(v interface{}) error {
		if err := u.SetTarget(v); err != nil {
			return err
		}
		return d.Next()
	}
}

func makeBenchmarkDecodeBeatsEvents(factory decoderFactory) func(*testing.B) {
	return func(b *testing.B) {
		for i := 0; i < b.N; i++ {
			func() {
				file, err := os.Open("files/beats_events.json")
				// file, err := os.Open("test.json")
				if err != nil {
					b.Error(err)
					return
				}

				defer file.Close()

				decode := factory(file)
				for {
					var to map[string]interface{}
					if err := decode(&to); err != nil {
						if err != io.EOF {
							b.Error(err)
						}
						return
					}
				}
			}()
		}
	}
}

func makeBenchmarkDecodeBeatsEventsBuffered(factory decoderFactoryBuf) func(*testing.B) {
	content, err := ioutil.ReadFile("files/beats_events.json")
	if err != nil {
		panic(err)
	}

	return func(b *testing.B) {
		b.SetBytes(int64(len(content)))

		for i := 0; i < b.N; i++ {
			decode := factory(content)
			for {
				var to map[string]interface{}
				if err := decode(&to); err != nil {
					if err != io.EOF {
						b.Error(err)
					}
					break
				}
			}
		}
	}
}

func makeBenchmarkEncodeEvents(factory encoderFactory, events []map[string]interface{}) func(*testing.B) {
	var buf bytes.Buffer
	buf.Grow(16 * 1024)
	encode := factory(&buf)

	return func(b *testing.B) {
		for i := 0; i < b.N; i++ {
			var written int64

			for _, event := range events {
				buf.Reset()
				if err := encode(event); err != nil {
					b.Error(err)
					return
				}
				written += int64(buf.Len())
			}
			b.SetBytes(written)
		}
	}
}

func loadBeatsEvents() []map[string]interface{} {
	content, err := ioutil.ReadFile("files/beats_events.json")
	if err != nil {
		panic(err)
	}

	var events []map[string]interface{}
	dec := stdjson.NewDecoder(bytes.NewReader(content))
	for {
		var e map[string]interface{}
		if err := dec.Decode(&e); err != nil {
			if err == io.EOF {
				break
			}

			panic(err)
		}

		events = append(events, e)
	}

	return events
}
<|endoftext|>


<commit_before>// (C) Copyright 2012, Jeramey Crawford <jeramey@antihe.ro>. All
// rights reserved. Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package apr1_crypt implements the standard Unix MD5-Crypt algorithm created
// by Poul-Henning Kamp for FreeBSD, and modified by the Apache project.
//
// The only change from MD5-Crypt is the use of the magic constant "$apr1$"
// instead of "$1$". The algorithms are otherwise identical.
package apr1_crypt

import (
	"bytes"
	"crypto/md5"

	"github.com/kless/crypt/common"
)

var Salt = &common.Salt{
	MagicPrefix: []byte("$apr1$"),
	SaltLenMin:  1, // Real minimum is 0, but that isn't useful.
	SaltLenMax:  8,
}

// Crypt performs the MD5-Crypt hashing algorithm, returning a full hash string
// suitable for storage and later password verification.
//
// If the salt is empty, a randomly-generated salt will be generated of length
// SaltLenMax.
func Crypt(key, salt []byte) (string, error) {
	if len(salt) == 0 {
		salt = Salt.Generate(Salt.SaltLenMax)
	}
	if !bytes.HasPrefix(salt, Salt.MagicPrefix) {
		return "", common.ErrSaltPrefix
	}

	saltToks := bytes.Split(salt, []byte{'$'})

	if len(saltToks) < 3 {
		return "", common.ErrSaltFormat
	} else {
		salt = saltToks[2]
	}

	if len(salt) > 8 {
		salt = salt[0:8]
	}

	B := md5.New()
	B.Write(key)
	B.Write(salt)
	B.Write(key)
	Bsum := B.Sum(nil)

	A := md5.New()
	A.Write(key)
	A.Write(Salt.MagicPrefix)
	A.Write(salt)
	i := len(key)
	for ; i > 16; i -= 16 {
		A.Write(Bsum)
	}
	A.Write(Bsum[0:i])
	for i = len(key); i > 0; i >>= 1 {
		if (i & 1) == 0 {
			A.Write(key[0:1])
		} else {
			A.Write([]byte{0})
		}
	}
	Asum := A.Sum(nil)

	Csum := Asum
	for round := 0; round < 1000; round++ {
		C := md5.New()

		if (round & 1) != 0 {
			C.Write(key)
		} else {
			C.Write(Csum)
		}

		if (round % 3) != 0 {
			C.Write(salt)
		}

		if (round % 7) != 0 {
			C.Write(key)
		}

		if (round & 1) == 0 {
			C.Write(key)
		} else {
			C.Write(Csum)
		}

		Csum = C.Sum(nil)
	}

	out := make([]byte, 0, 23+len(Salt.MagicPrefix)+len(salt))
	out = append(out, Salt.MagicPrefix...)
	out = append(out, salt...)
	out = append(out, '$')
	out = append(out, common.Base64_24Bit([]byte{
		Csum[12], Csum[6], Csum[0],
		Csum[13], Csum[7], Csum[1],
		Csum[14], Csum[8], Csum[2],
		Csum[15], Csum[9], Csum[3],
		Csum[5], Csum[10], Csum[4],
		Csum[11],
	})...)

	return string(out), nil
}

// Verify hashes a key using the same salt parameter as the given in the hash
// string, and if the results match, it returns true.
func Verify(key []byte, hash string) bool {
	c, err := Crypt(key, []byte(hash))
	if err != nil {
		return false
	}
	return c == hash
}
<commit_msg>Simplifies code for apr1<commit_after>// (C) Copyright 2012, Jeramey Crawford <jeramey@antihe.ro>. All
// rights reserved. Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package apr1_crypt implements the standard Unix MD5-Crypt algorithm created
// by Poul-Henning Kamp for FreeBSD, and modified by the Apache project.
//
// The only change from MD5-Crypt is the use of the magic constant "$apr1$"
// instead of "$1$". The algorithms are otherwise identical.
package apr1_crypt

import "github.com/kless/crypt/md5_crypt"

var Salt = md5_crypt.Salt

func init() {
	Salt.MagicPrefix = []byte("$apr1$")
}

// Crypt performs the MD5-Crypt hashing algorithm, returning a full hash string
// suitable for storage and later password verification.
//
// If the salt is empty, a randomly-generated salt will be generated of length
// SaltLenMax.
func Crypt(key, salt []byte) (string, error) { return md5_crypt.Crypt(key, salt) }

// Verify hashes a key using the same salt parameter as the given in the hash
// string, and if the results match, it returns true.
func Verify(key []byte, hash string) bool {
	c, err := Crypt(key, []byte(hash))
	if err != nil {
		return false
	}
	return c == hash
}
<|endoftext|>


<commit_before>package driver

import (
	"encoding/json"
	"net/http"
	"strconv"
	"time"

	"gopkg.in/mgo.v2/bson"

	"github.com/Gujarats/API-Golang/util"
	"github.com/Gujarats/API-Golang/util/logger"

	"github.com/Gujarats/API-Golang/model/city/interface"

	driverModel "github.com/Gujarats/API-Golang/model/driver"
	"github.com/Gujarats/API-Golang/model/driver/interface"

	"github.com/Gujarats/API-Golang/model/global"
)

// find specific driver with their ID or name.
// if the desired data didn't exist then insert new data
func UpdateDriver(driver driverInterface.DriverInterfacce) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		//start time for lenght of the process
		startTimer := time.Now()

		w.Header().Set("Access-Control-Allow-Methods", "POST")

		id := r.FormValue("id")
		name := r.FormValue("name")
		lat := r.FormValue("latitude")
		lon := r.FormValue("longitude")
		status := r.FormValue("status")

		isAllExist := util.CheckValue(id, name, lat, lon, status)
		if !isAllExist {
			logger.PrintLog("Required Params Empty")

			//return Bad response
			w.WriteHeader(http.StatusBadRequest)
			global.SetResponse(w, "Failed", "Required Params Empty")
			return
		}

		// convert string to bool
		statusBool, err := strconv.ParseBool(status)
		if err != nil {
			//return Bad response
			logger.PrintLog("Failed to Parse Boolean")
			w.WriteHeader(http.StatusBadRequest)
			global.SetResponse(w, "Failed", "Parse Boolean Erro")
			return
		}

		// convert string to float64
		convertedFloat, err := util.ConvertToFloat64(lat, lon)
		if err != nil {
			w.WriteHeader(http.StatusBadRequest)
			global.SetResponse(w, "Failed", "Failed to conver float value")
			return
		}

		latFloat := convertedFloat[0]
		lonFloat := convertedFloat[1]

		driverData := driverModel.DriverData{Id: bson.ObjectId(id), Name: name, Status: statusBool, Location: driverModel.GeoJson{Coordinates: []float64{lonFloat, latFloat}}}
		driver.Update(driverData)

		//return succes response
		elpasedTime := time.Since(startTimer).Seconds()
		w.WriteHeader(http.StatusOK)
		global.SetResponseTime(w, "Succes", "Driver Inserted", elpasedTime)
		return
	})

}

func FindDriver(driver driverInterface.DriverInterfacce, cityInterface cityInterface.CityInterfacce) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		//start time for lenght of the process
		startTimer := time.Now()
		w.Header().Set("Access-Control-Allow-Methods", "GET")

		lat := r.FormValue("latitude")
		lon := r.FormValue("longitude")
		city := r.FormValue("city")
		distance := r.FormValue("distance")

		//checking empty value
		checkValue := util.CheckValue(lat, lon, city, distance)
		if !checkValue {
			logger.PrintLog("Required Params Empty")

			//return Bad response
			w.WriteHeader(http.StatusBadRequest)
			global.SetResponse(w, "Failed", "Required Params Empty")
			return
		}

		floatNumbers, err := util.ConvertToFloat64(lat, lon)
		if err != nil {
			w.WriteHeader(http.StatusBadRequest)
			global.SetResponse(w, "Failed", "Failed to convert float value")
			return
		}
		latFloat := floatNumbers[0]
		lonFloat := floatNumbers[1]

		intNumbers, err := util.ConvertToInt64(distance)
		if err != nil {
			w.WriteHeader(http.StatusBadRequest)
			global.SetResponse(w, "Failed", "Failed to convert integer value")
			return
		}
		distanceInt := intNumbers[0]

		// get all district from redis and calculate it
		// calculate nearest location district with given location and city from mongodb
		district, err := cityInterface.GetNearestDistrict(city, latFloat, lonFloat, distanceInt)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			global.SetResponse(w, "Failed", "Failed to get nearest district")
			return
		}

		// checking district result from mongodb
		if district.Name == "" {
			w.WriteHeader(http.StatusInternalServerError)
			global.SetResponse(w, "Failed", "No District found")
			return
		}

		//response variable for getting the drivers
		var driverResponse driverModel.DriverData

		// checks drivers int the district from the redis
		drivers := driver.DriversRedis(district.Name, district.Id.Hex())
		if len(drivers) > 0 {
			// get the first index drvier from redis and save it again to redis
			driverResponse = drivers[0]

			// update the driver's status to unavailable in mongodb
			// Latitude is 1 in the index and Longitude is 0. Rules from mongodb
			drivers[0].Status = false
			driver.Update(drivers[0])

			// update redis data by removing the first index
			drivers = drivers[1:]
			// save the drivers to redis replacing previous data
			driver.SaveDriversRedis(drivers, district.Name, district.Id.Hex())

		} else {
			// we could not find any data in redis and mongo
			w.WriteHeader(http.StatusOK)
			global.SetResponse(w, "Success", "We couldn't find any driver")
			return
		}

		//return succes response
		w.WriteHeader(http.StatusOK)
		elapsedTime := time.Since(startTimer).Seconds()
		response := global.Response{Status: "Success", Message: "Data Found", Latency: elapsedTime, Data: driverResponse}
		json.NewEncoder(w).Encode(response)
		return

	})
}

func InsertDriver(driver driverInterface.DriverInterfacce) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

		//start time for lenght of the process
		startTimer := time.Now()
		w.Header().Set("Access-Control-Allow-Methods", "POST")

		// getting the parameters
		name := r.FormValue("name")
		lat := r.FormValue("latitude")
		lon := r.FormValue("longitude")
		status := r.FormValue("status")

		isAllExist := util.CheckValue(name, lat, lon, status)
		if !isAllExist {
			logger.PrintLog("Required Params Empty")

			//return Bad response
			w.WriteHeader(http.StatusBadRequest)
			global.SetResponse(w, "Failed", "Required Params Empty")
			return
		}

		// convert string to bool
		statusBool, err := strconv.ParseBool(status)
		if err != nil {
			//return Bad response
			logger.PrintLog("Failed to Parse Boolean")
			w.WriteHeader(http.StatusBadRequest)
			global.SetResponse(w, "Failed", "Parse Boolean Erro")
			return
		}

		// convert string to float64
		convertedFloat, err := util.ConvertToFloat64(lat, lon)
		if err != nil {
			w.WriteHeader(http.StatusBadRequest)
			global.SetResponse(w, "Failed", "Failed to convert float value")
			return
		}
		latFloat := convertedFloat[0]
		lonFloat := convertedFloat[1]

		// insert driver
		driver.Insert(name, latFloat, lonFloat, statusBool)

		//return succes response
		w.WriteHeader(http.StatusOK)
		elapsedTime := time.Since(startTimer).Seconds()
		response := global.Response{Status: "Success", Message: "Data Inserted", Latency: elapsedTime}
		json.NewEncoder(w).Encode(response)
		return
	})
}
<commit_msg>fix build for app.go<commit_after>package driver

import (
	"encoding/json"
	"net/http"
	"strconv"
	"time"

	"gopkg.in/mgo.v2/bson"

	"github.com/Gujarats/API-Golang/util"
	"github.com/Gujarats/API-Golang/util/logger"

	"github.com/Gujarats/API-Golang/model/city/interface"

	driverModel "github.com/Gujarats/API-Golang/model/driver"
	"github.com/Gujarats/API-Golang/model/driver/interface"

	"github.com/Gujarats/API-Golang/model/global"
)

// find specific driver with their ID or name.
// if the desired data didn't exist then insert new data
func UpdateDriver(driver driverInterface.DriverInterfacce) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		//start time for lenght of the process
		startTimer := time.Now()

		w.Header().Set("Access-Control-Allow-Methods", "POST")

		id := r.FormValue("id")
		name := r.FormValue("name")
		lat := r.FormValue("latitude")
		lon := r.FormValue("longitude")
		status := r.FormValue("status")

		isAllExist := util.CheckValue(id, name, lat, lon, status)
		if !isAllExist {
			logger.PrintLog("Required Params Empty")

			//return Bad response
			w.WriteHeader(http.StatusBadRequest)
			global.SetResponse(w, "Failed", "Required Params Empty")
			return
		}

		// convert string to bool
		statusBool, err := strconv.ParseBool(status)
		if err != nil {
			//return Bad response
			logger.PrintLog("Failed to Parse Boolean")
			w.WriteHeader(http.StatusBadRequest)
			global.SetResponse(w, "Failed", "Parse Boolean Erro")
			return
		}

		// convert string to float64
		convertedFloat, err := util.ConvertToFloat64(lat, lon)
		if err != nil {
			w.WriteHeader(http.StatusBadRequest)
			global.SetResponse(w, "Failed", "Failed to conver float value")
			return
		}

		latFloat := convertedFloat[0]
		lonFloat := convertedFloat[1]

		driverData := driverModel.DriverData{Id: bson.ObjectId(id), Name: name, Status: statusBool, Location: driverModel.GeoJson{Coordinates: []float64{lonFloat, latFloat}}}
		driver.Update(driverData)

		//return succes response
		elpasedTime := time.Since(startTimer).Seconds()
		w.WriteHeader(http.StatusOK)
		global.SetResponseTime(w, "Succes", "Driver Inserted", elpasedTime)
		return
	})

}

func FindDriver(driver driverInterface.DriverInterfacce, cityInterface cityInterface.CityInterfacce) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		//start time for lenght of the process
		startTimer := time.Now()
		w.Header().Set("Access-Control-Allow-Methods", "GET")

		lat := r.FormValue("latitude")
		lon := r.FormValue("longitude")
		city := r.FormValue("city")
		distance := r.FormValue("distance")

		//checking empty value
		checkValue := util.CheckValue(lat, lon, city, distance)
		if !checkValue {
			logger.PrintLog("Required Params Empty")

			//return Bad response
			w.WriteHeader(http.StatusBadRequest)
			global.SetResponse(w, "Failed", "Required Params Empty")
			return
		}

		floatNumbers, err := util.ConvertToFloat64(lat, lon)
		if err != nil {
			w.WriteHeader(http.StatusBadRequest)
			global.SetResponse(w, "Failed", "Failed to convert float value")
			return
		}
		latFloat := floatNumbers[0]
		lonFloat := floatNumbers[1]

		intNumbers, err := util.ConvertToInt64(distance)
		if err != nil {
			w.WriteHeader(http.StatusBadRequest)
			global.SetResponse(w, "Failed", "Failed to convert integer value")
			return
		}
		distanceInt := intNumbers[0]

		// get all district from redis and calculate it
		// calculate nearest location district with given location and city from mongodb
		district, err := cityInterface.GetNearestDistrict(city, latFloat, lonFloat, distanceInt)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			global.SetResponse(w, "Failed", "Failed to get nearest district")
			return
		}

		// checking district result from mongodb
		if district.Name == "" {
			w.WriteHeader(http.StatusInternalServerError)
			global.SetResponse(w, "Failed", "No District found")
			return
		}

		//response variable for getting the drivers
		var driverResponse driverModel.DriverData

		// checks drivers int the district from the redis
		drivers := driver.DriversRedis(district.Name, district.Id.Hex())
		if len(drivers) > 0 {
			// get the first index drvier from redis and save it again to redis
			driverResponse = drivers[0]

			// update the driver's status to unavailable in mongodb
			// Latitude is 1 in the index and Longitude is 0. Rules from mongodb
			drivers[0].Status = false
			driver.Update(drivers[0])

			// update redis data by removing the first index
			drivers = drivers[1:]
			// save the drivers to redis replacing previous data
			driver.SaveDriversRedis(drivers, district.Name, district.Id.Hex())

		} else {
			// we could not find any data in redis and mongo
			w.WriteHeader(http.StatusOK)
			global.SetResponse(w, "Success", "We couldn't find any driver")
			return
		}

		//return succes response
		w.WriteHeader(http.StatusOK)
		elapsedTime := time.Since(startTimer).Seconds()
		response := global.Response{Status: "Success", Message: "Data Found", Latency: elapsedTime, Data: driverResponse}
		json.NewEncoder(w).Encode(response)
		return

	})
}

func InsertDriver(driver driverInterface.DriverInterfacce) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

		//start time for lenght of the process
		startTimer := time.Now()
		w.Header().Set("Access-Control-Allow-Methods", "POST")

		// getting the parameters
		name := r.FormValue("name")
		lat := r.FormValue("latitude")
		lon := r.FormValue("longitude")
		status := r.FormValue("status")

		isAllExist := util.CheckValue(name, lat, lon, status)
		if !isAllExist {
			logger.PrintLog("Required Params Empty")

			//return Bad response
			w.WriteHeader(http.StatusBadRequest)
			global.SetResponse(w, "Failed", "Required Params Empty")
			return
		}

		// convert string to bool
		statusBool, err := strconv.ParseBool(status)
		if err != nil {
			//return Bad response
			logger.PrintLog("Failed to Parse Boolean")
			w.WriteHeader(http.StatusBadRequest)
			global.SetResponse(w, "Failed", "Parse Boolean Erro")
			return
		}

		// convert string to float64
		convertedFloat, err := util.ConvertToFloat64(lat, lon)
		if err != nil {
			w.WriteHeader(http.StatusBadRequest)
			global.SetResponse(w, "Failed", "Failed to convert float value")
			return
		}
		latFloat := convertedFloat[0]
		lonFloat := convertedFloat[1]

		// insert driver
		driver.Insert(name, name, latFloat, lonFloat, statusBool)

		//return succes response
		w.WriteHeader(http.StatusOK)
		elapsedTime := time.Since(startTimer).Seconds()
		response := global.Response{Status: "Success", Message: "Data Inserted", Latency: elapsedTime}
		json.NewEncoder(w).Encode(response)
		return
	})
}
<|endoftext|>


<commit_before>package speed

// Instance wraps a PCP compatible Instance
type Instance struct {
	name   string
	id     uint32
	indom  InstanceDomain
	offset int
}

// newInstance generates a new Instance type based on the passed parameters
// the id is passed explicitly as it is assumed that this will be constructed
// after initializing the InstanceDomain
// this is not a part of the public API as this is not supposed to be used directly,
// but instead added using the AddInstance method of InstanceDomain
func newInstance(id uint32, name string, indom InstanceDomain) *Instance {
	return &Instance{
		name, id, indom, 0,
	}
}

func (i *Instance) String() string {
	return "Instance: " + i.name
}

func (i *Instance) Offset() int { return i.offset }

func (i *Instance) SetOffset(offset int) { i.offset = offset }
<commit_msg>instance: make it private<commit_after>package speed

// pcpInstance wraps a PCP compatible Instance
type pcpInstance struct {
	name   string
	id     uint32
	indom  *PCPInstanceDomain
	offset int
}

// newpcpInstance generates a new Instance type based on the passed parameters
// the id is passed explicitly as it is assumed that this will be constructed
// after initializing the InstanceDomain
// this is not a part of the public API as this is not supposed to be used directly,
// but instead added using the AddInstance method of InstanceDomain
func newpcpInstance(id uint32, name string, indom *PCPInstanceDomain) *pcpInstance {
	return &pcpInstance{
		name, id, indom, 0,
	}
}

func (i *pcpInstance) String() string {
	return "Instance: " + i.name
}

func (i *pcpInstance) Offset() int { return i.offset }

func (i *pcpInstance) SetOffset(offset int) { i.offset = offset }
<|endoftext|>


<commit_before>package street

import (
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"
	"testing"

	"github.com/smartystreets/assertions/should"
	"github.com/smartystreets/gunit"
)

func TestClientFixture(t *testing.T) {
	gunit.Run(new(ClientFixture), t)
}

type ClientFixture struct {
	*gunit.Fixture

	sender *FakeSender
	client *Client
	batch  *Batch
}

func (f *ClientFixture) Setup() {
	f.sender = &FakeSender{}
	f.client = NewClient(f.sender)
	f.batch = NewBatch()
}

func (f *ClientFixture) TestSingleAddressBatch_SentInQueryStringAsGET() {
	f.sender.response = `[{"input_index": 0, "input_id": "42"}]`
	input := &Lookup{InputID: "42"}
	f.batch.Append(input)

	err := f.client.SendBatch(f.batch)

	f.So(err, should.BeNil)
	f.So(f.sender.request, should.NotBeNil)
	f.So(f.sender.request.Method, should.Equal, "GET")
	f.So(f.sender.request.URL.Path, should.Equal, "/street-address")
	f.So(f.sender.requestBody, should.BeNil)
	f.So(f.sender.request.ContentLength, should.Equal, 0)
	f.So(f.sender.request.URL.String(), should.StartWith, verifyURL)
	f.So(f.sender.request.URL.Query(), should.Resemble, url.Values{"input_id": {"42"}})
}

func (f *ClientFixture) TestAddressBatchSerializedAndSent__ResponseCandidatesIncorporatedIntoBatch() {
	f.sender.response = `[
		{"input_index": 0, "input_id": "42"},
		{"input_index": 2, "input_id": "44"},
		{"input_index": 2, "input_id": "44", "candidate_index": 1}
	]`
	input0 := &Lookup{InputID: "42"}
	input1 := &Lookup{InputID: "43"}
	input2 := &Lookup{InputID: "44"}
	f.batch.Append(input0)
	f.batch.Append(input1)
	f.batch.Append(input2)

	err := f.client.SendBatch(f.batch)

	f.So(err, should.BeNil)
	f.So(f.sender.request, should.NotBeNil)
	f.So(f.sender.request.Method, should.Equal, "POST")
	f.So(f.sender.request.URL.Path, should.Equal, "/street-address")
	f.So(f.sender.request.ContentLength, should.Equal, len(f.sender.requestBody))
	f.So(string(f.sender.requestBody), should.Equal, `[{"input_id":"42"},{"input_id":"43"},{"input_id":"44"}]`)
	f.So(f.sender.request.URL.String(), should.Equal, verifyURL)

	f.So(input0.Results, should.Resemble, []*Candidate{{InputID: "42"}})
	f.So(input1.Results, should.BeEmpty)
	f.So(input2.Results, should.Resemble, []*Candidate{{InputID: "44", InputIndex: 2}, {InputID: "44", InputIndex: 2, CandidateIndex: 1}})
}

func (f *ClientFixture) TestNilBatchNOP() {
	err := f.client.SendBatch(nil)
	f.So(err, should.BeNil)
	f.So(f.sender.request, should.BeNil)
}

func (f *ClientFixture) TestEmptyBatch_NOP() {
	err := f.client.SendBatch(new(Batch))
	f.So(err, should.BeNil)
	f.So(f.sender.request, should.BeNil)
}

func (f *ClientFixture) TestSenderErrorPreventsDeserialization() {
	f.sender.err = errors.New("GOPHERS!")
	f.sender.response = `[
		{"input_index": 0, "input_id": "42"},
		{"input_index": 2, "input_id": "44"},
		{"input_index": 2, "input_id": "44", "candidate_index": 1}
	]` // would be deserialized if not for the err (above)

	input := new(Lookup)
	f.batch.Append(input)

	err := f.client.SendBatch(f.batch)

	f.So(err, should.NotBeNil)
	f.So(input.Results, should.BeEmpty)
}

func (f *ClientFixture) TestDeserializationErrorPreventsDeserialization() {
	f.sender.response = `I can't haz JSON`
	input := new(Lookup)
	f.batch.Append(input)

	err := f.client.SendBatch(f.batch)

	f.So(err, should.NotBeNil)
	f.So(input.Results, should.BeEmpty)
}

func (f *ClientFixture) TestNullCandidatesWithinResponseArrayAreIgnoredAfterDeserialization() {
	f.sender.response = `[null]`
	lookup := new(Lookup)
	f.batch.Append(lookup)
	f.So(func() { f.client.SendBatch(f.batch) }, should.NotPanic)
	f.So(lookup.Results, should.BeEmpty)
}

func (f *ClientFixture) TestOutOfRangeCandidatesWithinResponseArrayAreIgnoredAfterDeserialization() {
	f.sender.response = `[{"input_index": 9999999}]`
	lookup := new(Lookup)
	f.batch.Append(lookup)
	f.So(func() { f.client.SendBatch(f.batch) }, should.NotPanic)
	f.So(lookup.Results, should.BeEmpty)
}

func (f *ClientFixture) FocusTestFullJSONResponseDeserialization() {
	f.sender.response = `[
  {
	"input_id": "blah",
    "input_index": 0,
    "candidate_index": 4242,
	"addressee": "John Smith",
    "delivery_line_1": "3214 N University Ave # 409",
    "delivery_line_2": "blah blah",
    "last_line": "Provo UT 84604-4405",
    "delivery_point_barcode": "846044405140",
    "components": {
      "primary_number": "3214",
      "street_predirection": "N",
      "street_postdirection": "Q",
      "street_name": "University",
      "street_suffix": "Ave",
      "secondary_number": "409",
      "secondary_designator": "#",
      "extra_secondary_number": "410",
      "extra_secondary_designator": "Apt",
      "pmb_number": "411",
      "pmb_designator": "Box",
      "city_name": "Provo",
      "default_city_name": "Provo",
      "state_abbreviation": "UT",
      "zipcode": "84604",
      "plus4_code": "4405",
      "delivery_point": "14",
      "delivery_point_check_digit": "0",
      "urbanization": "urbanization",
	  "ews_match": true
    },
    "metadata": {
      "record_type": "S",
      "zip_type": "Standard",
      "county_fips": "49049",
      "county_name": "Utah",
      "carrier_route": "C016",
      "congressional_district": "03",
	  "building_default_indicator": "hi",
      "rdi": "Commercial",
      "elot_sequence": "0016",
      "elot_sort": "A",
      "latitude": 40.27658,
      "longitude": -111.65759,
      "precision": "Zip9",
      "time_zone": "Mountain",
      "utc_offset": -7,
      "dst": true
    },
    "analysis": {
      "dpv_match_code": "S",
      "dpv_footnotes": "AACCRR",
      "dpv_cmra": "Y",
      "dpv_vacant": "N",
      "active": "Y",
      "footnotes": "footnotes",
      "lacslink_code": "lacslink_code",
      "lacslink_indicator": "lacslink_indicator",
      "suitelink_match": true
    }
  }
]`
	lookup := new(Lookup)
	f.batch.Append(lookup)
	err := f.client.SendBatch(f.batch)
	f.So(err, should.BeNil)
	f.So(lookup.Results, should.Resemble, []*Candidate{
		{
			InputID:              "blah",
			InputIndex:           0,
			CandidateIndex:       4242,
			Addressee:            "John Smith",
			DeliveryLine1:        "3214 N University Ave # 409",
			DeliveryLine2:        "blah blah",
			LastLine:             "Provo UT 84604-4405",
			DeliveryPointBarcode: "846044405140",
			Components: Components{
				PrimaryNumber:            "3214",
				StreetPredirection:       "N",
				StreetName:               "University",
				StreetPostdirection:      "Q",
				StreetSuffix:             "Ave",
				SecondaryNumber:          "409",
				SecondaryDesignator:      "#",
				ExtraSecondaryNumber:     "410",
				ExtraSecondaryDesignator: "Apt",
				PMBNumber:                "411",
				PMBDesignator:            "Box",
				CityName:                 "Provo",
				DefaultCityName:          "Provo",
				StateAbbreviation:        "UT",
				ZIPCode:                  "84604",
				Plus4Code:                "4405",
				DeliveryPoint:            "14",
				DeliveryPointCheckDigit:  "0",
				Urbanization:             "urbanization",
			},
			Metadata: Metadata{
				RecordType:               "S",
				ZIPType:                  "Standard",
				CountyFIPS:               "49049",
				CountyName:               "Utah",
				CarrierRoute:             "C016",
				CongressionalDistrict:    "03",
				BuildingDefaultIndicator: "hi",
				RDI:                      "Commercial",
				ELOTSequence:             "0016",
				ELOTSort:                 "A",
				Latitude:                 40.27658,
				Longitude:                -111.65759,
				Precision:                "Zip9",
				TimeZone:                 "Mountain",
				UTCOffset:                -7,
				DST:                      true,
			},
			Analysis: Analysis{
				DPVMatchCode:      "S",
				DPVFootnotes:      "AACCRR",
				DPVCMRACode:       "Y",
				DPVVacantCode:     "N",
				Active:            "Y",
				Footnotes:         "footnotes",
				LACSLinkCode:      "lacslink_code",
				LACSLinkIndicator: "lacslink_indicator",
				SuiteLinkMatch:    true,
				EWSMatch:          false,
			},
		},
	})
}

/*////////////////////////////////////////////////////////////////////////*/

type FakeSender struct {
	callCount int

	request     *http.Request
	requestBody []byte

	response string
	err      error
}

func (f *FakeSender) Send(request *http.Request) ([]byte, error) {
	f.callCount++
	f.request = request
	if request != nil && request.Body != nil {
		f.requestBody, _ = ioutil.ReadAll(request.Body)
	}
	return []byte(f.response), f.err
}
<commit_msg>No need for focus.<commit_after>package street

import (
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"
	"testing"

	"github.com/smartystreets/assertions/should"
	"github.com/smartystreets/gunit"
)

func TestClientFixture(t *testing.T) {
	gunit.Run(new(ClientFixture), t)
}

type ClientFixture struct {
	*gunit.Fixture

	sender *FakeSender
	client *Client
	batch  *Batch
}

func (f *ClientFixture) Setup() {
	f.sender = &FakeSender{}
	f.client = NewClient(f.sender)
	f.batch = NewBatch()
}

func (f *ClientFixture) TestSingleAddressBatch_SentInQueryStringAsGET() {
	f.sender.response = `[{"input_index": 0, "input_id": "42"}]`
	input := &Lookup{InputID: "42"}
	f.batch.Append(input)

	err := f.client.SendBatch(f.batch)

	f.So(err, should.BeNil)
	f.So(f.sender.request, should.NotBeNil)
	f.So(f.sender.request.Method, should.Equal, "GET")
	f.So(f.sender.request.URL.Path, should.Equal, "/street-address")
	f.So(f.sender.requestBody, should.BeNil)
	f.So(f.sender.request.ContentLength, should.Equal, 0)
	f.So(f.sender.request.URL.String(), should.StartWith, verifyURL)
	f.So(f.sender.request.URL.Query(), should.Resemble, url.Values{"input_id": {"42"}})
}

func (f *ClientFixture) TestAddressBatchSerializedAndSent__ResponseCandidatesIncorporatedIntoBatch() {
	f.sender.response = `[
		{"input_index": 0, "input_id": "42"},
		{"input_index": 2, "input_id": "44"},
		{"input_index": 2, "input_id": "44", "candidate_index": 1}
	]`
	input0 := &Lookup{InputID: "42"}
	input1 := &Lookup{InputID: "43"}
	input2 := &Lookup{InputID: "44"}
	f.batch.Append(input0)
	f.batch.Append(input1)
	f.batch.Append(input2)

	err := f.client.SendBatch(f.batch)

	f.So(err, should.BeNil)
	f.So(f.sender.request, should.NotBeNil)
	f.So(f.sender.request.Method, should.Equal, "POST")
	f.So(f.sender.request.URL.Path, should.Equal, "/street-address")
	f.So(f.sender.request.ContentLength, should.Equal, len(f.sender.requestBody))
	f.So(string(f.sender.requestBody), should.Equal, `[{"input_id":"42"},{"input_id":"43"},{"input_id":"44"}]`)
	f.So(f.sender.request.URL.String(), should.Equal, verifyURL)

	f.So(input0.Results, should.Resemble, []*Candidate{{InputID: "42"}})
	f.So(input1.Results, should.BeEmpty)
	f.So(input2.Results, should.Resemble, []*Candidate{{InputID: "44", InputIndex: 2}, {InputID: "44", InputIndex: 2, CandidateIndex: 1}})
}

func (f *ClientFixture) TestNilBatchNOP() {
	err := f.client.SendBatch(nil)
	f.So(err, should.BeNil)
	f.So(f.sender.request, should.BeNil)
}

func (f *ClientFixture) TestEmptyBatch_NOP() {
	err := f.client.SendBatch(new(Batch))
	f.So(err, should.BeNil)
	f.So(f.sender.request, should.BeNil)
}

func (f *ClientFixture) TestSenderErrorPreventsDeserialization() {
	f.sender.err = errors.New("GOPHERS!")
	f.sender.response = `[
		{"input_index": 0, "input_id": "42"},
		{"input_index": 2, "input_id": "44"},
		{"input_index": 2, "input_id": "44", "candidate_index": 1}
	]` // would be deserialized if not for the err (above)

	input := new(Lookup)
	f.batch.Append(input)

	err := f.client.SendBatch(f.batch)

	f.So(err, should.NotBeNil)
	f.So(input.Results, should.BeEmpty)
}

func (f *ClientFixture) TestDeserializationErrorPreventsDeserialization() {
	f.sender.response = `I can't haz JSON`
	input := new(Lookup)
	f.batch.Append(input)

	err := f.client.SendBatch(f.batch)

	f.So(err, should.NotBeNil)
	f.So(input.Results, should.BeEmpty)
}

func (f *ClientFixture) TestNullCandidatesWithinResponseArrayAreIgnoredAfterDeserialization() {
	f.sender.response = `[null]`
	lookup := new(Lookup)
	f.batch.Append(lookup)
	f.So(func() { f.client.SendBatch(f.batch) }, should.NotPanic)
	f.So(lookup.Results, should.BeEmpty)
}

func (f *ClientFixture) TestOutOfRangeCandidatesWithinResponseArrayAreIgnoredAfterDeserialization() {
	f.sender.response = `[{"input_index": 9999999}]`
	lookup := new(Lookup)
	f.batch.Append(lookup)
	f.So(func() { f.client.SendBatch(f.batch) }, should.NotPanic)
	f.So(lookup.Results, should.BeEmpty)
}

func (f *ClientFixture) TestFullJSONResponseDeserialization() {
	f.sender.response = `[
  {
	"input_id": "blah",
    "input_index": 0,
    "candidate_index": 4242,
	"addressee": "John Smith",
    "delivery_line_1": "3214 N University Ave # 409",
    "delivery_line_2": "blah blah",
    "last_line": "Provo UT 84604-4405",
    "delivery_point_barcode": "846044405140",
    "components": {
      "primary_number": "3214",
      "street_predirection": "N",
      "street_postdirection": "Q",
      "street_name": "University",
      "street_suffix": "Ave",
      "secondary_number": "409",
      "secondary_designator": "#",
      "extra_secondary_number": "410",
      "extra_secondary_designator": "Apt",
      "pmb_number": "411",
      "pmb_designator": "Box",
      "city_name": "Provo",
      "default_city_name": "Provo",
      "state_abbreviation": "UT",
      "zipcode": "84604",
      "plus4_code": "4405",
      "delivery_point": "14",
      "delivery_point_check_digit": "0",
      "urbanization": "urbanization",
	  "ews_match": true
    },
    "metadata": {
      "record_type": "S",
      "zip_type": "Standard",
      "county_fips": "49049",
      "county_name": "Utah",
      "carrier_route": "C016",
      "congressional_district": "03",
	  "building_default_indicator": "hi",
      "rdi": "Commercial",
      "elot_sequence": "0016",
      "elot_sort": "A",
      "latitude": 40.27658,
      "longitude": -111.65759,
      "precision": "Zip9",
      "time_zone": "Mountain",
      "utc_offset": -7,
      "dst": true
    },
    "analysis": {
      "dpv_match_code": "S",
      "dpv_footnotes": "AACCRR",
      "dpv_cmra": "Y",
      "dpv_vacant": "N",
      "active": "Y",
      "footnotes": "footnotes",
      "lacslink_code": "lacslink_code",
      "lacslink_indicator": "lacslink_indicator",
      "suitelink_match": true
    }
  }
]`
	lookup := new(Lookup)
	f.batch.Append(lookup)
	err := f.client.SendBatch(f.batch)
	f.So(err, should.BeNil)
	f.So(lookup.Results, should.Resemble, []*Candidate{
		{
			InputID:              "blah",
			InputIndex:           0,
			CandidateIndex:       4242,
			Addressee:            "John Smith",
			DeliveryLine1:        "3214 N University Ave # 409",
			DeliveryLine2:        "blah blah",
			LastLine:             "Provo UT 84604-4405",
			DeliveryPointBarcode: "846044405140",
			Components: Components{
				PrimaryNumber:            "3214",
				StreetPredirection:       "N",
				StreetName:               "University",
				StreetPostdirection:      "Q",
				StreetSuffix:             "Ave",
				SecondaryNumber:          "409",
				SecondaryDesignator:      "#",
				ExtraSecondaryNumber:     "410",
				ExtraSecondaryDesignator: "Apt",
				PMBNumber:                "411",
				PMBDesignator:            "Box",
				CityName:                 "Provo",
				DefaultCityName:          "Provo",
				StateAbbreviation:        "UT",
				ZIPCode:                  "84604",
				Plus4Code:                "4405",
				DeliveryPoint:            "14",
				DeliveryPointCheckDigit:  "0",
				Urbanization:             "urbanization",
			},
			Metadata: Metadata{
				RecordType:               "S",
				ZIPType:                  "Standard",
				CountyFIPS:               "49049",
				CountyName:               "Utah",
				CarrierRoute:             "C016",
				CongressionalDistrict:    "03",
				BuildingDefaultIndicator: "hi",
				RDI:                      "Commercial",
				ELOTSequence:             "0016",
				ELOTSort:                 "A",
				Latitude:                 40.27658,
				Longitude:                -111.65759,
				Precision:                "Zip9",
				TimeZone:                 "Mountain",
				UTCOffset:                -7,
				DST:                      true,
			},
			Analysis: Analysis{
				DPVMatchCode:      "S",
				DPVFootnotes:      "AACCRR",
				DPVCMRACode:       "Y",
				DPVVacantCode:     "N",
				Active:            "Y",
				Footnotes:         "footnotes",
				LACSLinkCode:      "lacslink_code",
				LACSLinkIndicator: "lacslink_indicator",
				SuiteLinkMatch:    true,
				EWSMatch:          false,
			},
		},
	})
}

/*////////////////////////////////////////////////////////////////////////*/

type FakeSender struct {
	callCount int

	request     *http.Request
	requestBody []byte

	response string
	err      error
}

func (f *FakeSender) Send(request *http.Request) ([]byte, error) {
	f.callCount++
	f.request = request
	if request != nil && request.Body != nil {
		f.requestBody, _ = ioutil.ReadAll(request.Body)
	}
	return []byte(f.response), f.err
}
<|endoftext|>


<commit_before>package k8s_test

import (
	"github.com/onsi/gomega/gbytes"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Container Limits", func() {
	const (
		TaskCPULimit    = "--set=concourse.web.defaultTaskCpuLimit=512"
		TaskMemoryLimit = "--set=concourse.web.defaultTaskMemoryLimit=1GB"
		COS             = "--set=worker.nodeSelector.nodeImage=cos"
		UBUNTU          = "--set=worker.nodeSelector.nodeImage=ubuntu"
	)

	BeforeEach(func() {
		setReleaseNameAndNamespace("cl")
	})

	onPks(func() {
		containerLimitsWork(TaskCPULimit, TaskMemoryLimit)
	})

	onGke(func() {
		containerLimitsWork(COS, TaskCPULimit, TaskMemoryLimit)
		containerLimitsFail(UBUNTU, TaskCPULimit, TaskMemoryLimit)
	})

	AfterEach(func() {
		cleanupReleases()
	})

})

func deployWithSelectors(selectorFlags ...string) {
	helmDeployTestFlags := []string{
		"--set=concourse.web.kubernetes.enabled=false",
		"--set=worker.replicas=1",
	}

	deployConcourseChart(releaseName, append(helmDeployTestFlags, selectorFlags...)...)
}

func containerLimitsWork(selectorFlags ...string) {
	Context("container limits work", func() {
		It("returns the configure default container limit", func() {
			deployWithSelectors(selectorFlags...)

			atc := waitAndLogin(namespace, releaseName+"-web")
			defer atc.Close()

			buildSession := fly.Start("execute", "-c", "tasks/tiny.yml")
			<-buildSession.Exited

			Expect(buildSession.ExitCode()).To(Equal(0))

			hijackSession := fly.Start(
				"hijack",
				"-b", "1",
				"-s", "one-off",
				"--", "sh", "-c",
				"cat /sys/fs/cgroup/memory/memory.memsw.limit_in_bytes; cat /sys/fs/cgroup/cpu/cpu.shares",
			)
			<-hijackSession.Exited

			Expect(hijackSession.ExitCode()).To(Equal(0))
			Expect(hijackSession).To(gbytes.Say("1073741824\n512"))
		})
	})
}

func containerLimitsFail(selectorFlags ...string) {
	// Disabling until https://github.com/concourse/concourse/issues/7086 is fixed.
	XContext("container limits fail", func() {
		It("fails to set the memory limit", func() {
			deployWithSelectors(selectorFlags...)

			atc := waitAndLogin(namespace, releaseName+"-web")
			defer atc.Close()

			buildSession := fly.Start("execute", "-c", "tasks/tiny.yml")
			<-buildSession.Exited
			Expect(buildSession.ExitCode()).To(Equal(2))
			Expect(buildSession).To(gbytes.Say(
				"memory.memsw.limit_in_bytes: permission denied",
			))
		})
	})
}
<commit_msg>k8s/topgun: use regex to match error message<commit_after>package k8s_test

import (
	"github.com/onsi/gomega/gbytes"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Container Limits", func() {
	const (
		TaskCPULimit    = "--set=concourse.web.defaultTaskCpuLimit=512"
		TaskMemoryLimit = "--set=concourse.web.defaultTaskMemoryLimit=1GB"
		COS             = "--set=worker.nodeSelector.nodeImage=cos"
		UBUNTU          = "--set=worker.nodeSelector.nodeImage=ubuntu"
	)

	BeforeEach(func() {
		setReleaseNameAndNamespace("cl")
	})

	onPks(func() {
		containerLimitsWork(TaskCPULimit, TaskMemoryLimit)
	})

	onGke(func() {
		containerLimitsWork(COS, TaskCPULimit, TaskMemoryLimit)
		containerLimitsFail(UBUNTU, TaskCPULimit, TaskMemoryLimit)
	})

	AfterEach(func() {
		cleanupReleases()
	})

})

func deployWithSelectors(selectorFlags ...string) {
	helmDeployTestFlags := []string{
		"--set=concourse.web.kubernetes.enabled=false",
		"--set=worker.replicas=1",
	}

	deployConcourseChart(releaseName, append(helmDeployTestFlags, selectorFlags...)...)
}

func containerLimitsWork(selectorFlags ...string) {
	Context("container limits work", func() {
		It("returns the configure default container limit", func() {
			deployWithSelectors(selectorFlags...)

			atc := waitAndLogin(namespace, releaseName+"-web")
			defer atc.Close()

			buildSession := fly.Start("execute", "-c", "tasks/tiny.yml")
			<-buildSession.Exited

			Expect(buildSession.ExitCode()).To(Equal(0))

			hijackSession := fly.Start(
				"hijack",
				"-b", "1",
				"-s", "one-off",
				"--", "sh", "-c",
				"cat /sys/fs/cgroup/memory/memory.memsw.limit_in_bytes; cat /sys/fs/cgroup/cpu/cpu.shares",
			)
			<-hijackSession.Exited

			Expect(hijackSession.ExitCode()).To(Equal(0))
			Expect(hijackSession).To(gbytes.Say("1073741824\n512"))
		})
	})
}

func containerLimitsFail(selectorFlags ...string) {
	Context("container limits fail", func() {
		It("fails to set the memory limit", func() {
			deployWithSelectors(selectorFlags...)

			atc := waitAndLogin(namespace, releaseName+"-web")
			defer atc.Close()

			buildSession := fly.Start("execute", "-c", "tasks/tiny.yml")
			<-buildSession.Exited
			Expect(buildSession.ExitCode()).To(Equal(2))
			Expect(buildSession).To(gbytes.Say(
				"memory.memsw.limit_in_bytes: (permission denied|no such file or directory)",
			))
		})
	})
}
<|endoftext|>


<commit_before>// Copyright 2013 The go-github AUTHORS. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package github provides a client for using the GitHub API.

Construct a new GitHub client, then use the various services on the client to
access different parts of the GitHub API. For example:

	client := github.NewClient(nil)

	// list all organizations for user "willnorris"
	orgs, _, err := client.Organizations.List("willnorris", nil)

Set optional parameters for an API method by passing an Options object.

	// list recently updated repositories for org "github"
	opt := &github.RepositoryListByOrgOptions{Sort: "updated"}
	repos, _, err := client.Repositories.ListByOrg("github", opt)

The services of a client divide the API into logical chunks and correspond to
the structure of the GitHub API documentation at
http://developer.github.com/v3/.

Authentication

The go-github library does not directly handle authentication. Instead, when
creating a new client, pass an http.Client that can handle authentication for
you. The easiest and recommended way to do this is using the goauth2 library,
but you can always use any other library that provides an http.Client. If you
have an OAuth2 access token (for example, a personal API token), you can use it
with the goauth2 using:

	import "code.google.com/p/goauth2/oauth"

	// simple OAuth transport if you already have an access token;
	// see goauth2 library for full usage
	t := &oauth.Transport{
		Token: &oauth.Token{AccessToken: "..."},
	}

	client := github.NewClient(t.Client())

	// list all repositories for the authenticated user
	repos, _, err := client.Repositories.List("", nil)

Note that when using an authenticated Client, all calls made by the client will
include the specified OAuth token. Therefore, authenticated clients should
almost never be shared between different users.

Rate Limiting

GitHub imposes a rate limit on all API clients.  Unauthenticated clients are
limited to 60 requests per hour, while authenticated clients can make up to
5,000 requests per hour.  To receive the higher rate limit when making calls
that are not issued on behalf of a user, use the
UnauthenticatedRateLimitedTransport.

The Rate field on a client tracks the rate limit information based on the most
recent API call.  This is updated on every call, but may be out of date if it's
been some time since the last API call and other clients have made subsequent
requests since then.  You can always call RateLimit() directly to get the most
up-to-date rate limit data for the client.

Learn more about GitHub rate limiting at
http://developer.github.com/v3/#rate-limiting.

Creating and Updating Resources

All structs for GitHub resources use pointer values for all non-repeated fields.
This allows distinguishing between unset fields and those set to a zero-value.
Helper functions have been provided to easily create these pointers for string,
bool, and int values.  For example:

	// create a new private repository named "foo"
	repo := &github.Repository{
		Name:    github.String("foo"),
		Private: github.Bool(true),
	}
	client.Repositories.Create("", repo)

Users who have worked with protocol buffers should find this pattern familiar.
*/
package github
<commit_msg>add doc section on conditional requests<commit_after>// Copyright 2013 The go-github AUTHORS. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package github provides a client for using the GitHub API.

Construct a new GitHub client, then use the various services on the client to
access different parts of the GitHub API. For example:

	client := github.NewClient(nil)

	// list all organizations for user "willnorris"
	orgs, _, err := client.Organizations.List("willnorris", nil)

Set optional parameters for an API method by passing an Options object.

	// list recently updated repositories for org "github"
	opt := &github.RepositoryListByOrgOptions{Sort: "updated"}
	repos, _, err := client.Repositories.ListByOrg("github", opt)

The services of a client divide the API into logical chunks and correspond to
the structure of the GitHub API documentation at
http://developer.github.com/v3/.

Authentication

The go-github library does not directly handle authentication. Instead, when
creating a new client, pass an http.Client that can handle authentication for
you. The easiest and recommended way to do this is using the goauth2 library,
but you can always use any other library that provides an http.Client. If you
have an OAuth2 access token (for example, a personal API token), you can use it
with the goauth2 using:

	import "code.google.com/p/goauth2/oauth"

	// simple OAuth transport if you already have an access token;
	// see goauth2 library for full usage
	t := &oauth.Transport{
		Token: &oauth.Token{AccessToken: "..."},
	}

	client := github.NewClient(t.Client())

	// list all repositories for the authenticated user
	repos, _, err := client.Repositories.List("", nil)

Note that when using an authenticated Client, all calls made by the client will
include the specified OAuth token. Therefore, authenticated clients should
almost never be shared between different users.

Rate Limiting

GitHub imposes a rate limit on all API clients.  Unauthenticated clients are
limited to 60 requests per hour, while authenticated clients can make up to
5,000 requests per hour.  To receive the higher rate limit when making calls
that are not issued on behalf of a user, use the
UnauthenticatedRateLimitedTransport.

The Rate field on a client tracks the rate limit information based on the most
recent API call.  This is updated on every call, but may be out of date if it's
been some time since the last API call and other clients have made subsequent
requests since then.  You can always call RateLimit() directly to get the most
up-to-date rate limit data for the client.

Learn more about GitHub rate limiting at
http://developer.github.com/v3/#rate-limiting.

Conditional Requests

The GitHub API has good support for conditional requests which will help
prevent you from burning through your rate limit, as well as help speed up your
application.  go-github does not handle conditional requests directly, but is
instead designed to work with a caching http.Transport.  We recommend using
https://github.com/gregjones/httpcache, which can be used in conjuction with
https://github.com/sourcegraph/apiproxy to provide additional flexibility and
control of caching rules.

Creating and Updating Resources

All structs for GitHub resources use pointer values for all non-repeated fields.
This allows distinguishing between unset fields and those set to a zero-value.
Helper functions have been provided to easily create these pointers for string,
bool, and int values.  For example:

	// create a new private repository named "foo"
	repo := &github.Repository{
		Name:    github.String("foo"),
		Private: github.Bool(true),
	}
	client.Repositories.Create("", repo)

Users who have worked with protocol buffers should find this pattern familiar.
*/
package github
<|endoftext|>