<commit_before>/*
Copyright (c) 2014 VMware, Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package event

import (
	"reflect"
	"sync"

	"github.com/vmware/govmomi/property"
	"github.com/vmware/govmomi/vim25"
	"github.com/vmware/govmomi/vim25/methods"
	"github.com/vmware/govmomi/vim25/mo"
	"github.com/vmware/govmomi/vim25/types"
	"golang.org/x/net/context"
)

type Manager struct {
	reference types.ManagedObjectReference

	c *vim25.Client

	eventCategory   map[string]string
	eventCategoryMu *sync.Mutex
}

func NewManager(c *vim25.Client) *Manager {
	m := Manager{
		reference: *c.ServiceContent.EventManager,

		c: c,

		eventCategoryMu: new(sync.Mutex),
	}

	return &m
}

func (m Manager) CreateCollectorForEvents(ctx context.Context, filter types.EventFilterSpec) (*HistoryCollector, error) {
	req := types.CreateCollectorForEvents{
		This:   m.reference,
		Filter: filter,
	}

	res, err := methods.CreateCollectorForEvents(ctx, m.c, &req)
	if err != nil {
		return nil, err
	}

	return NewHistoryCollector(m.c, res.Returnval), nil
}

func (m Manager) LogUserEvent(ctx context.Context, entity types.ManagedObjectReference, msg string) error {
	req := types.LogUserEvent{
		This:   m.reference,
		Entity: entity,
		Msg:    msg,
	}

	_, err := methods.LogUserEvent(ctx, m.c, &req)
	if err != nil {
		return err
	}

	return nil
}

func (m Manager) PostEvent(ctx context.Context, eventToPost types.BaseEvent, taskInfo types.TaskInfo) error {
	req := types.PostEvent{
		This:        m.reference,
		EventToPost: eventToPost,
		TaskInfo:    &taskInfo,
	}

	_, err := methods.PostEvent(ctx, m.c, &req)
	if err != nil {
		return err
	}

	return nil
}

func (m Manager) QueryEvents(ctx context.Context, filter types.EventFilterSpec) ([]types.BaseEvent, error) {
	req := types.QueryEvents{
		This:   m.reference,
		Filter: filter,
	}

	res, err := methods.QueryEvents(ctx, m.c, &req)
	if err != nil {
		return nil, err
	}

	return res.Returnval, nil
}

func (m Manager) RetrieveArgumentDescription(ctx context.Context, eventTypeID string) ([]types.EventArgDesc, error) {
	req := types.RetrieveArgumentDescription{
		This:        m.reference,
		EventTypeId: eventTypeID,
	}

	res, err := methods.RetrieveArgumentDescription(ctx, m.c, &req)
	if err != nil {
		return nil, err
	}

	return res.Returnval, nil
}

func (m Manager) eventCategoryMap(ctx context.Context) (map[string]string, error) {
	m.eventCategoryMu.Lock()
	defer m.eventCategoryMu.Unlock()

	if m.eventCategory != nil {
		return m.eventCategory, nil
	}

	var o mo.EventManager

	ps := []string{"description.eventInfo"}
	err := property.DefaultCollector(m.c).RetrieveOne(ctx, m.reference, ps, &o)
	if err != nil {
		return nil, err
	}

	m.eventCategory = make(map[string]string, len(o.Description.EventInfo))

	for _, info := range o.Description.EventInfo {
		m.eventCategory[info.Key] = info.Category
	}

	return m.eventCategory, nil
}

// EventCategory returns the category for an event, such as "info" or "error" for example.
func (m Manager) EventCategory(ctx context.Context, event types.BaseEvent) (string, error) {
	// Most of the event details are included in the Event.FullFormattedMessage, but the category
	// is only available via the EventManager description.eventInfo property.  The value of this
	// property is static, so we fetch and once and cache.
	eventCategory, err := m.eventCategoryMap(ctx)
	if err != nil {
		return "", err
	}

	class := reflect.TypeOf(event).Elem().Name()

	return eventCategory[class], nil
}
<commit_msg>Fix event.Manager category cache<commit_after>/*
Copyright (c) 2014 VMware, Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package event

import (
	"reflect"
	"sync"

	"github.com/vmware/govmomi/property"
	"github.com/vmware/govmomi/vim25"
	"github.com/vmware/govmomi/vim25/methods"
	"github.com/vmware/govmomi/vim25/mo"
	"github.com/vmware/govmomi/vim25/types"
	"golang.org/x/net/context"
)

type Manager struct {
	reference types.ManagedObjectReference

	c *vim25.Client

	eventCategory   map[string]string
	eventCategoryMu *sync.Mutex
}

func NewManager(c *vim25.Client) *Manager {
	m := Manager{
		reference: *c.ServiceContent.EventManager,

		c: c,

		eventCategory:   make(map[string]string),
		eventCategoryMu: new(sync.Mutex),
	}

	return &m
}

func (m Manager) CreateCollectorForEvents(ctx context.Context, filter types.EventFilterSpec) (*HistoryCollector, error) {
	req := types.CreateCollectorForEvents{
		This:   m.reference,
		Filter: filter,
	}

	res, err := methods.CreateCollectorForEvents(ctx, m.c, &req)
	if err != nil {
		return nil, err
	}

	return NewHistoryCollector(m.c, res.Returnval), nil
}

func (m Manager) LogUserEvent(ctx context.Context, entity types.ManagedObjectReference, msg string) error {
	req := types.LogUserEvent{
		This:   m.reference,
		Entity: entity,
		Msg:    msg,
	}

	_, err := methods.LogUserEvent(ctx, m.c, &req)
	if err != nil {
		return err
	}

	return nil
}

func (m Manager) PostEvent(ctx context.Context, eventToPost types.BaseEvent, taskInfo types.TaskInfo) error {
	req := types.PostEvent{
		This:        m.reference,
		EventToPost: eventToPost,
		TaskInfo:    &taskInfo,
	}

	_, err := methods.PostEvent(ctx, m.c, &req)
	if err != nil {
		return err
	}

	return nil
}

func (m Manager) QueryEvents(ctx context.Context, filter types.EventFilterSpec) ([]types.BaseEvent, error) {
	req := types.QueryEvents{
		This:   m.reference,
		Filter: filter,
	}

	res, err := methods.QueryEvents(ctx, m.c, &req)
	if err != nil {
		return nil, err
	}

	return res.Returnval, nil
}

func (m Manager) RetrieveArgumentDescription(ctx context.Context, eventTypeID string) ([]types.EventArgDesc, error) {
	req := types.RetrieveArgumentDescription{
		This:        m.reference,
		EventTypeId: eventTypeID,
	}

	res, err := methods.RetrieveArgumentDescription(ctx, m.c, &req)
	if err != nil {
		return nil, err
	}

	return res.Returnval, nil
}

func (m Manager) eventCategoryMap(ctx context.Context) (map[string]string, error) {
	m.eventCategoryMu.Lock()
	defer m.eventCategoryMu.Unlock()

	if len(m.eventCategory) != 0 {
		return m.eventCategory, nil
	}

	var o mo.EventManager

	ps := []string{"description.eventInfo"}
	err := property.DefaultCollector(m.c).RetrieveOne(ctx, m.reference, ps, &o)
	if err != nil {
		return nil, err
	}

	for _, info := range o.Description.EventInfo {
		m.eventCategory[info.Key] = info.Category
	}

	return m.eventCategory, nil
}

// EventCategory returns the category for an event, such as "info" or "error" for example.
func (m Manager) EventCategory(ctx context.Context, event types.BaseEvent) (string, error) {
	// Most of the event details are included in the Event.FullFormattedMessage, but the category
	// is only available via the EventManager description.eventInfo property.  The value of this
	// property is static, so we fetch and once and cache.
	eventCategory, err := m.eventCategoryMap(ctx)
	if err != nil {
		return "", err
	}

	class := reflect.TypeOf(event).Elem().Name()

	return eventCategory[class], nil
}
<|endoftext|>


<commit_before>package sqlread

type SummaryColumn struct {
	Name string
	Type string
}

type SummaryDataLoc struct {
	Start LexItem
	End   LexItem
}

type SummaryTree map[string]SummaryTable

type SummaryTable struct {
	Create   LexItem
	Cols     []SummaryColumn
	DataLocs []SummaryDataLoc

	SummaryDataLoc
}

type SummaryParser struct {
	Tree SummaryTree
}

func NewSummaryParser() *SummaryParser {
	return &SummaryParser{
		Tree: make(SummaryTree),
	}
}

func (t *SummaryParser) ParseStart(p *Parser) parseState {
	for {
		c, ok := p.scan()
		if !ok {
			break
		}

		if isOfAny(c, TCreateTable) {
			return t.parseCreateBuilder(c)
		}

		if isOfAny(c, TInsertInto) {
			return t.parseInsertIntoBuilder(c)
		}

		skips := []lexItemType{TComment, TDelim, TSemi, TDropTableFullStmt, TLockTableFullStmt, TUnlockTablesFullStmt, TSetFullStmt}
		if isOfAny(c, skips...) {
			continue
		}

		expects := append(skips, TCreateTable, TInsertInto)
		p.errorUnexpectedLex(c, expects...)
		break
	}
	return nil
}

func (t *SummaryParser) parseCreateBuilder(start LexItem) parseState {
	return func(p *Parser) parseState {
		c, ok := p.scan()
		if !ok {
			p.errorUnexpectedEOF()
			return nil
		}
		if c.Type != TIdentifier {
			p.errorUnexpectedLex(c, TIdentifier)
			return nil
		}

		if _, ok := t.Tree[c.Val]; !ok {
			t.Tree[c.Val] = SummaryTable{
				Create: c,
				Cols:   make([]SummaryColumn, 0),
				SummaryDataLoc: SummaryDataLoc{
					Start: start,
				},
			}
		}

		for {
			x, ok := p.scanUntil(TIdentifier, TSemi)
			if !ok {
				p.errorUnexpectedEOF()
				return nil
			}

			if x.Type == TSemi {
				et, _ := t.Tree[c.Val]
				et.End = x
				t.Tree[c.Val] = et

				return t.ParseStart
			}

			y, ok := p.scan()
			if !ok {
				p.errorUnexpectedEOF()
				return nil
			}
			if y.Type != TColumnType {
				p.errorUnexpectedLex(y, TColumnType)
				return nil
			}

			v := t.Tree[c.Val]

			v.Cols = append(v.Cols, SummaryColumn{
				Name: x.Val,
				Type: y.Val,
			})

			t.Tree[c.Val] = v
		}
	}
}

func (t *SummaryParser) parseInsertIntoBuilder(il LexItem) parseState {
	return func(p *Parser) parseState {
		c, ok := p.scan()
		if !ok {
			p.errorUnexpectedEOF()
			return nil
		}
		if c.Type != TIdentifier {
			p.errorUnexpectedLex(c, TIdentifier)
			return nil
		}

		s, ok := p.scanUntil(TSemi)
		if !ok {
			p.errorUnexpectedEOF()
			return nil
		}

		v := t.Tree[c.Val]

		v.DataLocs = append(v.DataLocs, SummaryDataLoc{
			Start: il,
			End:   s,
		})

		t.Tree[c.Val] = v

		return t.ParseStart
	}
}
<commit_msg>Cleans up map lookup<commit_after>package sqlread

type SummaryColumn struct {
	Name string
	Type string
}

type SummaryDataLoc struct {
	Start LexItem
	End   LexItem
}

type SummaryTree map[string]SummaryTable

type SummaryTable struct {
	Create   LexItem
	Cols     []SummaryColumn
	DataLocs []SummaryDataLoc

	SummaryDataLoc
}

type SummaryParser struct {
	Tree SummaryTree
}

func NewSummaryParser() *SummaryParser {
	return &SummaryParser{
		Tree: make(SummaryTree),
	}
}

func (t *SummaryParser) ParseStart(p *Parser) parseState {
	for {
		c, ok := p.scan()
		if !ok {
			break
		}

		if isOfAny(c, TCreateTable) {
			return t.parseCreateBuilder(c)
		}

		if isOfAny(c, TInsertInto) {
			return t.parseInsertIntoBuilder(c)
		}

		skips := []lexItemType{TComment, TDelim, TSemi, TDropTableFullStmt, TLockTableFullStmt, TUnlockTablesFullStmt, TSetFullStmt}
		if isOfAny(c, skips...) {
			continue
		}

		expects := append(skips, TCreateTable, TInsertInto)
		p.errorUnexpectedLex(c, expects...)
		break
	}
	return nil
}

func (t *SummaryParser) parseCreateBuilder(start LexItem) parseState {
	return func(p *Parser) parseState {
		c, ok := p.scan()
		if !ok {
			p.errorUnexpectedEOF()
			return nil
		}
		if c.Type != TIdentifier {
			p.errorUnexpectedLex(c, TIdentifier)
			return nil
		}

		if _, ok := t.Tree[c.Val]; !ok {
			t.Tree[c.Val] = SummaryTable{
				Create: c,
				Cols:   make([]SummaryColumn, 0),
				SummaryDataLoc: SummaryDataLoc{
					Start: start,
				},
			}
		}

		for {
			x, ok := p.scanUntil(TIdentifier, TSemi)
			if !ok {
				p.errorUnexpectedEOF()
				return nil
			}

			if x.Type == TSemi {
				et := t.Tree[c.Val]
				et.End = x
				t.Tree[c.Val] = et

				return t.ParseStart
			}

			y, ok := p.scan()
			if !ok {
				p.errorUnexpectedEOF()
				return nil
			}
			if y.Type != TColumnType {
				p.errorUnexpectedLex(y, TColumnType)
				return nil
			}

			v := t.Tree[c.Val]

			v.Cols = append(v.Cols, SummaryColumn{
				Name: x.Val,
				Type: y.Val,
			})

			t.Tree[c.Val] = v
		}
	}
}

func (t *SummaryParser) parseInsertIntoBuilder(il LexItem) parseState {
	return func(p *Parser) parseState {
		c, ok := p.scan()
		if !ok {
			p.errorUnexpectedEOF()
			return nil
		}
		if c.Type != TIdentifier {
			p.errorUnexpectedLex(c, TIdentifier)
			return nil
		}

		s, ok := p.scanUntil(TSemi)
		if !ok {
			p.errorUnexpectedEOF()
			return nil
		}

		v := t.Tree[c.Val]

		v.DataLocs = append(v.DataLocs, SummaryDataLoc{
			Start: il,
			End:   s,
		})

		t.Tree[c.Val] = v

		return t.ParseStart
	}
}
<|endoftext|>


<commit_before>package tagfast

import (
	"reflect"
	"strconv"
	"sync"
)

//[struct_name][field_name]
var CachedStructTags map[string]map[string]TagFast = make(map[string]map[string]TagFast)
var lock *sync.RWMutex = new(sync.RWMutex)

func CacheTag(struct_name string, field_name string, value TagFast) {
	lock.Lock()
	defer lock.Unlock()
	if _, ok := CachedStructTags[struct_name]; !ok {
		CachedStructTags[struct_name] = make(map[string]TagFast)
	}
	CachedStructTags[struct_name][field_name] = value
}

func GetTag(struct_name string, field_name string) (r TagFast, ok bool) {
	lock.RLock()
	defer lock.RUnlock()
	var v map[string]TagFast
	v, ok = CachedStructTags[struct_name]
	if !ok {
		return
	}
	r, ok = v[field_name]
	return
}

//usage: Tag(typ.Name(),typ.Field(i).Name,typ.Field(i).Tag,"form")
func Tag(i_struct interface{}, field_no int, key string) (tag string) {
	t := reflect.TypeOf(i_struct)
	if t.Field(field_no).Tag == "" {
		return ""
	}
	if v, ok := GetTag(t.String(), t.Field(field_no).Name); ok {
		tag = v.Get(key)
	} else {
		v := TagFast{Tag: t.Field(field_no).Tag}
		tag = v.Get(key)
		CacheTag(t.String(), t.Field(field_no).Name, v)
	}
	return
}

func ClearTag() {
	CachedStructTags = make(map[string]map[string]TagFast)
}

func ParseStructTag(tag string) map[string]string {
	lock.Lock()
	defer lock.Unlock()
	var tagsArray map[string]string = make(map[string]string)
	for tag != "" {
		// skip leading space
		i := 0
		for i < len(tag) && tag[i] == ' ' {
			i++
		}
		tag = tag[i:]
		if tag == "" {
			break
		}

		// scan to colon.
		// a space or a quote is a syntax error
		i = 0
		for i < len(tag) && tag[i] != ' ' && tag[i] != ':' && tag[i] != '"' {
			i++
		}
		if i+1 >= len(tag) || tag[i] != ':' || tag[i+1] != '"' {
			break
		}
		name := string(tag[:i])
		tag = tag[i+1:]

		// scan quoted string to find value
		i = 1
		for i < len(tag) && tag[i] != '"' {
			if tag[i] == '\\' {
				i++
			}
			i++
		}
		if i >= len(tag) {
			break
		}
		qvalue := string(tag[:i+1])
		tag = tag[i+1:]

		value, _ := strconv.Unquote(qvalue)
		tagsArray[name] = value
	}
	return tagsArray
}

type TagFast struct {
	Tag    reflect.StructTag
	Cached map[string]string
}

func (a *TagFast) Get(key string) string {
	if a.Cached == nil {
		a.Cached = ParseStructTag(string(a.Tag))
	}
	lock.RLock()
	defer lock.RUnlock()
	if v, ok := a.Cached[key]; ok {
		return v
	}
	return ""
}
<commit_msg>update<commit_after>package tagfast

import (
	"reflect"
	"strconv"
	"sync"
)

//[struct_name][field_name]
var CachedStructTags map[string]map[string]TagFast = make(map[string]map[string]TagFast)
var lock *sync.RWMutex = new(sync.RWMutex)

func CacheTag(struct_name string, field_name string, value TagFast) {
	lock.Lock()
	defer lock.Unlock()
	if _, ok := CachedStructTags[struct_name]; !ok {
		CachedStructTags[struct_name] = make(map[string]TagFast)
	}
	CachedStructTags[struct_name][field_name] = value
}

func GetTag(struct_name string, field_name string) (r TagFast, ok bool) {
	lock.RLock()
	defer lock.RUnlock()
	var v map[string]TagFast
	v, ok = CachedStructTags[struct_name]
	if !ok {
		return
	}
	r, ok = v[field_name]
	return
}

//usage: Tag(m,i,"form")
func Tag(i_struct interface{}, field_no int, key string) (tag string) {
	t := reflect.TypeOf(i_struct)
	if t.Field(field_no).Tag == "" {
		return ""
	}
	if v, ok := GetTag(t.String(), t.Field(field_no).Name); ok {
		tag = v.Get(key)
	} else {
		v := TagFast{Tag: t.Field(field_no).Tag}
		tag = v.Get(key)
		CacheTag(t.String(), t.Field(field_no).Name, v)
	}
	return
}

func ClearTag() {
	CachedStructTags = make(map[string]map[string]TagFast)
}

func ParseStructTag(tag string) map[string]string {
	lock.Lock()
	defer lock.Unlock()
	var tagsArray map[string]string = make(map[string]string)
	for tag != "" {
		// skip leading space
		i := 0
		for i < len(tag) && tag[i] == ' ' {
			i++
		}
		tag = tag[i:]
		if tag == "" {
			break
		}

		// scan to colon.
		// a space or a quote is a syntax error
		i = 0
		for i < len(tag) && tag[i] != ' ' && tag[i] != ':' && tag[i] != '"' {
			i++
		}
		if i+1 >= len(tag) || tag[i] != ':' || tag[i+1] != '"' {
			break
		}
		name := string(tag[:i])
		tag = tag[i+1:]

		// scan quoted string to find value
		i = 1
		for i < len(tag) && tag[i] != '"' {
			if tag[i] == '\\' {
				i++
			}
			i++
		}
		if i >= len(tag) {
			break
		}
		qvalue := string(tag[:i+1])
		tag = tag[i+1:]

		value, _ := strconv.Unquote(qvalue)
		tagsArray[name] = value
	}
	return tagsArray
}

type TagFast struct {
	Tag    reflect.StructTag
	Cached map[string]string
}

func (a *TagFast) Get(key string) string {
	if a.Cached == nil {
		a.Cached = ParseStructTag(string(a.Tag))
	}
	lock.RLock()
	defer lock.RUnlock()
	if v, ok := a.Cached[key]; ok {
		return v
	}
	return ""
}
<|endoftext|>


<commit_before>/*
 Copyright (c) 2014 Kouhei Maeda <mkouhei@palmtb.net>

 This software is release under the Expat License.
*/
package modules

import (
	"github.com/ajstarks/svgo"
	"math/rand"
)

func (canv Canvas) Network() *svg.SVG {
	canvas := svg.New(canv.W)
	canvas.Start(canv.Width, canv.Height)

	//x := canv.Width / 20
	//y := canv.Height / 20
	d := 5 // diameter
	//span := 50
	linestyle := "fill:none;stroke:black"
	circlestyle := "fill:gray"

	nd := 100
	xpoly := make([]int, nd)
	ypoly := make([]int, nd)
	for i := 0; i < nd; i++ {
		xpoly[i] = rand.Intn(canv.Width)
		ypoly[i] = rand.Intn(canv.Height)
		canvas.Circle(xpoly[i], ypoly[i], d, circlestyle)
	}
	/*
		xpoly[0] = x
		ypoly[0] = y
		xpoly[1] = x + span*2
		ypoly[1] = y + span*3
		xpoly[2] = x + span*4
		ypoly[2] = y + span*5
		canvas.Circle(xpoly[0], ypoly[0], d, circlestyle)
		canvas.Circle(xpoly[1], ypoly[1], d, circlestyle)
		canvas.Circle(xpoly[2], ypoly[2], d, circlestyle)
	*/
	canvas.Polyline(xpoly, ypoly, linestyle)

	canvas.End()
	return canvas
}
<commit_msg>Removed comment out code.<commit_after>/*
 Copyright (c) 2014 Kouhei Maeda <mkouhei@palmtb.net>

 This software is release under the Expat License.
*/
package modules

import (
	"github.com/ajstarks/svgo"
	"math/rand"
)

func (canv Canvas) Network() *svg.SVG {
	canvas := svg.New(canv.W)
	canvas.Start(canv.Width, canv.Height)

	d := 5 // diameter
	linestyle := "fill:none;stroke:black"
	circlestyle := "fill:gray"

	nd := 100
	xpoly := make([]int, nd)
	ypoly := make([]int, nd)
	for i := 0; i < nd; i++ {
		xpoly[i] = rand.Intn(canv.Width)
		ypoly[i] = rand.Intn(canv.Height)
		canvas.Circle(xpoly[i], ypoly[i], d, circlestyle)
	}
	canvas.Polyline(xpoly, ypoly, linestyle)

	canvas.End()
	return canvas
}
<|endoftext|>


<commit_before>package tar

import (
	"os"
	"os/exec"
)

// TODO: find solution that doesn't depend on os environemnt i.e. entirely golang based solution for tar unpack & pack
func Unpack(tarball, dest string) {
	cmd := exec.Command("tar", "xf", tarball, "-C", dest, ".")
	err := cmd.Run()
	if err != nil {
		panic(err)
	}
}

func Pack(src, tarball string) {
	fi, err := os.Stat(src)
	if err != nil {
		panic(err)
	}
	var cmd *exec.Cmd
	// determine whether packing a file or a directory
	switch mode := fi.Mode(); {
	case mode.IsDir():
		cmd = exec.Command("tar", "czf", tarball, "-C", src, ".")
	case mode.IsRegular():
		cmd = exec.Command("tar", "czf", tarball, src)
	}
	err = cmd.Run()
	if err != nil {
		panic(err)
	}
}

// func main() {
// 	Pack(os.Args[2], os.Args[1])
// }
<commit_msg>fix bug in tar.go's Unpack, preventing unpacking of single-file archives<commit_after>package tar

import (
	"os"
	"os/exec"
)

// TODO: find solution that doesn't depend on os environemnt i.e. entirely golang based solution for tar unpack & pack
func Unpack(tarball, dest string) {
	cmd := exec.Command("tar", "xf", tarball, "-C", dest)
	err := cmd.Run()
	if err != nil {
		panic(err)
	}
}

func Pack(src, tarball string) {
	fi, err := os.Stat(src)
	if err != nil {
		panic(err)
	}
	var cmd *exec.Cmd
	// determine whether packing a file or a directory
	switch mode := fi.Mode(); {
	case mode.IsDir():
		cmd = exec.Command("tar", "czf", tarball, "-C", src, ".")
	case mode.IsRegular():
		cmd = exec.Command("tar", "czf", tarball, src)
	}
	err = cmd.Run()
	if err != nil {
		panic(err)
	}
}

// func main() {
// 	Unpack(os.Args[1], os.Args[2])
// }
<|endoftext|>


<commit_before>package main

import (
	"⚛sdl"
	"⚛sdl/ttf"
	"fmt"
	"flag"
	"os"
	"clingon"
)

var (
	config                   configuration
	console                  *clingon.Console
	sdlrenderer              *clingon.SDLRenderer
	running, toggleAnimation bool
	r                        *renderer
	slideDown, slideUp       *clingon.Animation
)

type configuration struct {
	consoleX, consoleY  int16
	consoleW, consoleH  uint16
	fullscreen, verbose bool
	fps                 float
	bgImage             string
	animationDuration   int64
}

type renderer struct {
	config                                 *configuration
	appSurface, bgImageSurface, cliSurface *sdl.Surface
}

const ANIMATION_TIME = 500 * 1e6

func (r *renderer) render(updatedRects []sdl.Rect, y int16) {
	if updatedRects == nil { // Initially and during animations we blit the entire surface
		if r.bgImageSurface != nil {
			r.appSurface.Blit(nil, r.bgImageSurface, nil)
		}
		r.appSurface.Blit(&sdl.Rect{config.consoleX, y, 0, 0}, sdlrenderer.GetSurface(), nil)
		r.appSurface.Flip()
	} else { // When idle we can keep updating modified regions only
		for _, rect := range updatedRects {
			if r.bgImageSurface != nil {
				r.appSurface.Blit(
					&sdl.Rect{rect.X + r.config.consoleX, rect.Y + y, 0, 0},
					r.bgImageSurface,
					&sdl.Rect{rect.X + r.config.consoleX, rect.Y + y, rect.W, rect.H})
			}
			r.appSurface.Blit(
				&sdl.Rect{rect.X + r.config.consoleX, rect.Y + y, 0, 0},
				sdlrenderer.GetSurface(), &rect)
			r.appSurface.UpdateRect(int32(rect.X+r.config.consoleX), int32(rect.Y+y), uint32(rect.W), uint32(rect.H))
		}
	}
}

// Initialization boilerplate
func initialize(config *configuration) {
	var bgImage, appSurface *sdl.Surface

	if sdl.Init(sdl.INIT_VIDEO) != 0 {
		panic(sdl.GetError())
	}

	if ttf.Init() != 0 {
		panic(sdl.GetError())
	}

	font := ttf.OpenFont(flag.Arg(0), 12)

	if font == nil {
		panic(sdl.GetError())
	}

	sdl.EnableUNICODE(1)

	if config.fullscreen {
		appSurface = sdl.SetVideoMode(640, 480, 32, sdl.FULLSCREEN)
		sdl.ShowCursor(sdl.DISABLE)
	} else {
		appSurface = sdl.SetVideoMode(640, 480, 32, 0)
	}
	if config.bgImage != "" {
		bgImage = sdl.Load(config.bgImage)
	}

	sdlrenderer = clingon.NewSDLRenderer(sdl.CreateRGBSurface(sdl.SRCALPHA, int(config.consoleW), int(config.consoleH), 32, 0, 0, 0, 0), font)
	sdlrenderer.GetSurface().SetAlpha(sdl.SRCALPHA, 0xaa)

	if config.fps > 0 {
		sdlrenderer.FPSCh() <- config.fps
	}

	console = clingon.NewConsole(sdlrenderer, &ShellEvaluator{})
	console.SetPrompt("shell:$ ")
	console.GreetingText = "Welcome to the CLIngon shell!\n=============================\nPress F10 to toggle/untoggle\n\n"

	r = &renderer{
		config:         config,
		appSurface:     appSurface,
		cliSurface:     sdlrenderer.GetSurface(),
		bgImageSurface: bgImage,
	}

	slidingDistance := int16(appSurface.H) - config.consoleY
	slideDown = clingon.NewSlideDownAnimation(config.animationDuration, float64(slidingDistance))
	slideUp = clingon.NewSlideUpAnimation(config.animationDuration, float64(slidingDistance))
}

func main() {
	// Handle options
	help := flag.Bool("help", false, "Show usage")
	verbose := flag.Bool("verbose", false, "Verbose output")
	fullscreen := flag.Bool("fullscreen", false, "Go fullscreen!")
	fps := flag.Float("fps", clingon.DEFAULT_CONSOLE_RENDERER_FPS, "Frames per second")
	bgImage := flag.String("bg-image", "", "Background image file")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "shell - A system shell based on CLIngon (Command Line INterface for Go Nerds\n\n")
		fmt.Fprintf(os.Stderr, "Usage:\n\n")
		fmt.Fprintf(os.Stderr, "\tshell [options] <fontfile> \n\n")
		fmt.Fprintf(os.Stderr, "Options are:\n\n")
		flag.PrintDefaults()
	}

	flag.Parse()

	if *help == true {
		flag.Usage()
		return
	}

	if len(flag.Args()) < 1 {
		flag.Usage()
		return
	}

	config = configuration{
		verbose:           *verbose,
		fps:               *fps,
		fullscreen:        *fullscreen,
		bgImage:           *bgImage,
		consoleX:          40,
		consoleY:          40,
		consoleW:          560,
		consoleH:          400,
		animationDuration: 1e9,
	}

	initialize(&config)
	r.render(nil, config.consoleY)

	running = true

	go func() {
		for running {

			select {
			case event := <-sdl.Events:
				switch e := event.(type) {
				case sdl.QuitEvent:
					running = false
				case sdl.KeyboardEvent:
					keyName := sdl.GetKeyName(sdl.Key(e.Keysym.Sym))

					if config.verbose {
						fmt.Printf("\n")
						fmt.Printf("%v: %v", e.Keysym.Sym, ": ", keyName)

						fmt.Printf("%04x ", e.Type)

						for i := 0; i < len(e.Pad0); i++ {
							fmt.Printf("%02x ", e.Pad0[i])
						}
						fmt.Printf("\n")

						fmt.Printf("Type: %02x Which: %02x State: %02x Pad: %02x\n", e.Type, e.Which, e.State, e.Pad0[0])
						fmt.Printf("Scancode: %02x Sym: %08x Mod: %04x Unicode: %04x\n", e.Keysym.Scancode, e.Keysym.Sym, e.Keysym.Mod, e.Keysym.Unicode)
					}
					if (keyName == "escape") && (e.Type == sdl.KEYDOWN) {
						running = false
					} else if (keyName == "f10") && (e.Type == sdl.KEYDOWN) {
						toggleAnimation = !toggleAnimation
						console.Paused = toggleAnimation
						if console.Paused {
							t := slideUp.Pause()
							slideDown.Resume(t)
						} else {
							t := slideDown.Pause()
							if t == 0 {
								slideUp.Start()
							} else {
								slideUp.Resume(1e9 - t)
							}
						}
					} else if (keyName == "up") && (e.Type == sdl.KEYDOWN) {
						console.ReadlineCh() <- clingon.HISTORY_PREV
					} else if (keyName == "down") && (e.Type == sdl.KEYDOWN) {
						console.ReadlineCh() <- clingon.HISTORY_NEXT
					} else if (keyName == "left") && (e.Type == sdl.KEYDOWN) {
						console.ReadlineCh() <- clingon.CURSOR_LEFT
					} else if (keyName == "right") && (e.Type == sdl.KEYDOWN) {
						console.ReadlineCh() <- clingon.CURSOR_RIGHT
					} else {
						unicode := e.Keysym.Unicode
						if unicode > 0 {
							console.CharCh() <- unicode
						}
					}

				}
			}

		}
	}()

	var (
		y int16 = config.consoleY
//		animating bool
	)

	for running {
		select {
		case value := <-slideDown.ValueCh():
			y = 40 + int16(value)
			r.render(nil, y)
		case value := <-slideUp.ValueCh():
			y = 40 + int16(value)
			r.render(nil, y)
		case <-slideDown.FinishedCh():
		case <-slideUp.FinishedCh():
		case rects := <-sdlrenderer.UpdatedRectsCh():
			r.render(rects, y)
		}
	}

	sdl.Quit()
}
<commit_msg>Use config.animationDuration in slideUp.Resume<commit_after>package main

import (
	"⚛sdl"
	"⚛sdl/ttf"
	"fmt"
	"flag"
	"os"
	"clingon"
)

var (
	config                   configuration
	console                  *clingon.Console
	sdlrenderer              *clingon.SDLRenderer
	running, toggleAnimation bool
	r                        *renderer
	slideDown, slideUp       *clingon.Animation
)

type configuration struct {
	consoleX, consoleY  int16
	consoleW, consoleH  uint16
	fullscreen, verbose bool
	fps                 float
	bgImage             string
	animationDuration   int64
}

type renderer struct {
	config                                 *configuration
	appSurface, bgImageSurface, cliSurface *sdl.Surface
}

const ANIMATION_TIME = 500 * 1e6

func (r *renderer) render(updatedRects []sdl.Rect, y int16) {
	if updatedRects == nil { // Initially and during animations we blit the entire surface
		if r.bgImageSurface != nil {
			r.appSurface.Blit(nil, r.bgImageSurface, nil)
		}
		r.appSurface.Blit(&sdl.Rect{config.consoleX, y, 0, 0}, sdlrenderer.GetSurface(), nil)
		r.appSurface.Flip()
	} else { // When idle we can keep updating modified regions only
		for _, rect := range updatedRects {
			if r.bgImageSurface != nil {
				r.appSurface.Blit(
					&sdl.Rect{rect.X + r.config.consoleX, rect.Y + y, 0, 0},
					r.bgImageSurface,
					&sdl.Rect{rect.X + r.config.consoleX, rect.Y + y, rect.W, rect.H})
			}
			r.appSurface.Blit(
				&sdl.Rect{rect.X + r.config.consoleX, rect.Y + y, 0, 0},
				sdlrenderer.GetSurface(), &rect)
			r.appSurface.UpdateRect(int32(rect.X+r.config.consoleX), int32(rect.Y+y), uint32(rect.W), uint32(rect.H))
		}
	}
}

// Initialization boilerplate
func initialize(config *configuration) {
	var bgImage, appSurface *sdl.Surface

	if sdl.Init(sdl.INIT_VIDEO) != 0 {
		panic(sdl.GetError())
	}

	if ttf.Init() != 0 {
		panic(sdl.GetError())
	}

	font := ttf.OpenFont(flag.Arg(0), 12)

	if font == nil {
		panic(sdl.GetError())
	}

	sdl.EnableUNICODE(1)

	if config.fullscreen {
		appSurface = sdl.SetVideoMode(640, 480, 32, sdl.FULLSCREEN)
		sdl.ShowCursor(sdl.DISABLE)
	} else {
		appSurface = sdl.SetVideoMode(640, 480, 32, 0)
	}
	if config.bgImage != "" {
		bgImage = sdl.Load(config.bgImage)
	}

	sdlrenderer = clingon.NewSDLRenderer(sdl.CreateRGBSurface(sdl.SRCALPHA, int(config.consoleW), int(config.consoleH), 32, 0, 0, 0, 0), font)
	sdlrenderer.GetSurface().SetAlpha(sdl.SRCALPHA, 0xaa)

	if config.fps > 0 {
		sdlrenderer.FPSCh() <- config.fps
	}

	console = clingon.NewConsole(sdlrenderer, &ShellEvaluator{})
	console.SetPrompt("shell:$ ")
	console.GreetingText = "Welcome to the CLIngon shell!\n=============================\nPress F10 to toggle/untoggle\n\n"

	r = &renderer{
		config:         config,
		appSurface:     appSurface,
		cliSurface:     sdlrenderer.GetSurface(),
		bgImageSurface: bgImage,
	}

	slidingDistance := int16(appSurface.H) - config.consoleY
	slideDown = clingon.NewSlideDownAnimation(config.animationDuration, float64(slidingDistance))
	slideUp = clingon.NewSlideUpAnimation(config.animationDuration, float64(slidingDistance))
}

func main() {
	// Handle options
	help := flag.Bool("help", false, "Show usage")
	verbose := flag.Bool("verbose", false, "Verbose output")
	fullscreen := flag.Bool("fullscreen", false, "Go fullscreen!")
	fps := flag.Float("fps", clingon.DEFAULT_CONSOLE_RENDERER_FPS, "Frames per second")
	bgImage := flag.String("bg-image", "", "Background image file")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "shell - A system shell based on CLIngon (Command Line INterface for Go Nerds\n\n")
		fmt.Fprintf(os.Stderr, "Usage:\n\n")
		fmt.Fprintf(os.Stderr, "\tshell [options] <fontfile> \n\n")
		fmt.Fprintf(os.Stderr, "Options are:\n\n")
		flag.PrintDefaults()
	}

	flag.Parse()

	if *help == true {
		flag.Usage()
		return
	}

	if len(flag.Args()) < 1 {
		flag.Usage()
		return
	}

	config = configuration{
		verbose:           *verbose,
		fps:               *fps,
		fullscreen:        *fullscreen,
		bgImage:           *bgImage,
		consoleX:          40,
		consoleY:          40,
		consoleW:          560,
		consoleH:          400,
		animationDuration: 500*1e6,
	}

	initialize(&config)
	r.render(nil, config.consoleY)

	running = true

	go func() {
		for running {

			select {
			case event := <-sdl.Events:
				switch e := event.(type) {
				case sdl.QuitEvent:
					running = false
				case sdl.KeyboardEvent:
					keyName := sdl.GetKeyName(sdl.Key(e.Keysym.Sym))

					if config.verbose {
						fmt.Printf("\n")
						fmt.Printf("%v: %v", e.Keysym.Sym, ": ", keyName)

						fmt.Printf("%04x ", e.Type)

						for i := 0; i < len(e.Pad0); i++ {
							fmt.Printf("%02x ", e.Pad0[i])
						}
						fmt.Printf("\n")

						fmt.Printf("Type: %02x Which: %02x State: %02x Pad: %02x\n", e.Type, e.Which, e.State, e.Pad0[0])
						fmt.Printf("Scancode: %02x Sym: %08x Mod: %04x Unicode: %04x\n", e.Keysym.Scancode, e.Keysym.Sym, e.Keysym.Mod, e.Keysym.Unicode)
					}
					if (keyName == "escape") && (e.Type == sdl.KEYDOWN) {
						running = false
					} else if (keyName == "f10") && (e.Type == sdl.KEYDOWN) {
						toggleAnimation = !toggleAnimation
						console.Paused = toggleAnimation
						if console.Paused {
							t := slideUp.Pause()
							slideDown.Resume(t)
						} else {
							t := slideDown.Pause()
							if t == 0 {
								slideUp.Start()
							} else {
								slideUp.Resume(config.animationDuration - t)
							}
						}
					} else if (keyName == "up") && (e.Type == sdl.KEYDOWN) {
						console.ReadlineCh() <- clingon.HISTORY_PREV
					} else if (keyName == "down") && (e.Type == sdl.KEYDOWN) {
						console.ReadlineCh() <- clingon.HISTORY_NEXT
					} else if (keyName == "left") && (e.Type == sdl.KEYDOWN) {
						console.ReadlineCh() <- clingon.CURSOR_LEFT
					} else if (keyName == "right") && (e.Type == sdl.KEYDOWN) {
						console.ReadlineCh() <- clingon.CURSOR_RIGHT
					} else {
						unicode := e.Keysym.Unicode
						if unicode > 0 {
							console.CharCh() <- unicode
						}
					}

				}
			}

		}
	}()

	var (
		y int16 = config.consoleY
//		animating bool
	)

	for running {
		select {
		case value := <-slideDown.ValueCh():
			y = 40 + int16(value)
			r.render(nil, y)
		case value := <-slideUp.ValueCh():
			y = 40 + int16(value)
			r.render(nil, y)
		case <-slideDown.FinishedCh():
		case <-slideUp.FinishedCh():
		case rects := <-sdlrenderer.UpdatedRectsCh():
			r.render(rects, y)
		}
	}

	sdl.Quit()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math/rand"
	"os"
	"time"

	"github.com/jessevdk/go-flags"

	"github.com/zimmski/tavor"
	"github.com/zimmski/tavor/fuzz/strategy"
	"github.com/zimmski/tavor/parser"
)

const (
	returnOk = iota
	returnHelp
)

var opts struct {
	Config      func(s string) error `long:"config" description:"INI config file" no-ini:"true"`
	ConfigWrite string               `long:"config-write" description:"Write all arguments to an INI config file or to STDOUT with \"-\" as argument." no-ini:"true"`

	ListStrategies bool `long:"list-strategies" description:"List all available strategies." no-ini:"true"`

	InputFile string `long:"input-file" description:"Input tavor file" required:"true" no-ini:"true"`
	Seed      int64  `long:"seed" description:"Seed for all the randomness"`
	Strategy  string `long:"strategy" description:"The fuzzing strategy" default:"random"`
	Verbose   bool   `long:"verbose" description:"Do verbose output."`
	Version   bool   `long:"version" description:"Print the version of this program." no-ini:"true"`

	Debug bool `long:"debug" description:"Temporary debugging argument"`

	configFile string
}

func V(msg string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "[VERBOSE] "+msg+"\n", args...)
}

func checkArguments() {
	p := flags.NewNamedParser("tavor", flags.HelpFlag)
	p.ShortDescription = "A fuzzing and delta-debugging platform."

	opts.Config = func(s string) error {
		ini := flags.NewIniParser(p)

		opts.configFile = s

		return ini.ParseFile(s)
	}

	p.AddGroup("Tavor", "Tavor arguments", &opts)

	if _, err := p.ParseArgs(os.Args); err != nil {
		doListArguments()

		if e, ok := err.(*flags.Error); !ok || e.Type != flags.ErrHelp {
			panic(err)
		} else {
			p.WriteHelp(os.Stdout)

			os.Exit(returnHelp)
		}
	}

	doListArguments()

	if opts.ConfigWrite != "" {
		ini := flags.NewIniParser(p)

		var iniOptions flags.IniOptions = flags.IniIncludeComments | flags.IniIncludeDefaults | flags.IniCommentDefaults

		if opts.ConfigWrite == "-" {
			(ini.Write(os.Stdout, iniOptions))
		} else {
			ini.WriteFile(opts.ConfigWrite, iniOptions)
		}

		os.Exit(returnOk)
	}

	if opts.Seed == 0 {
		opts.Seed = time.Now().UTC().UnixNano()
	}
}

func doListArguments() {
	if opts.Version {
		fmt.Printf("Tavor v%s\n", tavor.Version)

		os.Exit(returnOk)
	} else if opts.ListStrategies {
		for _, name := range strategy.List() {
			fmt.Println(name)
		}

		os.Exit(returnOk)
	}
}

func main() {
	checkArguments()

	if opts.Verbose {
		V("Open file %s", opts.InputFile)
	}

	file, err := os.Open(opts.InputFile)
	if err != nil {
		panic(fmt.Errorf("cannot open tavor file %s: %v", opts.InputFile, err))
	}
	defer file.Close()

	if opts.Debug {
		tavor.DEBUG = true
	}

	doc, err := parser.ParseTavor(file)
	if err != nil {
		panic(fmt.Errorf("cannot parse tavor file: %v", err))
	}

	if opts.Verbose {
		V("Using seed %d", opts.Seed)
	}

	if opts.Verbose {
		V("Counted %d overall permutations", doc.PermutationsAll())
	}

	r := rand.New(rand.NewSource(opts.Seed))

	strat, err := strategy.New(opts.Strategy, doc)
	if err != nil {
		panic(err)
	}

	if opts.Verbose {
		V("Using %s strategy", opts.Strategy)
	}

	ch := strat.Fuzz(r)
	another := false
	for i := range ch {
		if !tavor.DEBUG {
			if another {
				fmt.Println()
			} else {
				another = true
			}
		}

		if tavor.DEBUG {
			fmt.Printf("Result:\n%s\n", doc.String())
		} else {
			fmt.Print(doc.String())
		}

		ch <- i
	}
}
<commit_msg>add Validate argument<commit_after>package main

import (
	"fmt"
	"math/rand"
	"os"
	"time"

	"github.com/jessevdk/go-flags"

	"github.com/zimmski/tavor"
	"github.com/zimmski/tavor/fuzz/strategy"
	"github.com/zimmski/tavor/parser"
)

const (
	returnOk = iota
	returnHelp
)

var opts struct {
	Config      func(s string) error `long:"config" description:"INI config file" no-ini:"true"`
	ConfigWrite string               `long:"config-write" description:"Write all arguments to an INI config file or to STDOUT with \"-\" as argument." no-ini:"true"`

	ListStrategies bool `long:"list-strategies" description:"List all available strategies." no-ini:"true"`

	InputFile string `long:"input-file" description:"Input tavor file" required:"true" no-ini:"true"`
	Seed      int64  `long:"seed" description:"Seed for all the randomness"`
	Strategy  string `long:"strategy" description:"The fuzzing strategy" default:"random"`
	Validate  bool   `long:"validate" description:"Just validates the input file"`
	Verbose   bool   `long:"verbose" description:"Do verbose output."`
	Version   bool   `long:"version" description:"Print the version of this program." no-ini:"true"`

	Debug bool `long:"debug" description:"Temporary debugging argument"`

	configFile string
}

func V(msg string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "[VERBOSE] "+msg+"\n", args...)
}

func checkArguments() {
	p := flags.NewNamedParser("tavor", flags.HelpFlag)
	p.ShortDescription = "A fuzzing and delta-debugging platform."

	opts.Config = func(s string) error {
		ini := flags.NewIniParser(p)

		opts.configFile = s

		return ini.ParseFile(s)
	}

	p.AddGroup("Tavor", "Tavor arguments", &opts)

	if _, err := p.ParseArgs(os.Args); err != nil {
		doListArguments()

		if e, ok := err.(*flags.Error); !ok || e.Type != flags.ErrHelp {
			panic(err)
		} else {
			p.WriteHelp(os.Stdout)

			os.Exit(returnHelp)
		}
	}

	doListArguments()

	if opts.ConfigWrite != "" {
		ini := flags.NewIniParser(p)

		var iniOptions flags.IniOptions = flags.IniIncludeComments | flags.IniIncludeDefaults | flags.IniCommentDefaults

		if opts.ConfigWrite == "-" {
			(ini.Write(os.Stdout, iniOptions))
		} else {
			ini.WriteFile(opts.ConfigWrite, iniOptions)
		}

		os.Exit(returnOk)
	}

	if opts.Seed == 0 {
		opts.Seed = time.Now().UTC().UnixNano()
	}
}

func doListArguments() {
	if opts.Version {
		fmt.Printf("Tavor v%s\n", tavor.Version)

		os.Exit(returnOk)
	} else if opts.ListStrategies {
		for _, name := range strategy.List() {
			fmt.Println(name)
		}

		os.Exit(returnOk)
	}
}

func main() {
	checkArguments()

	if opts.Verbose {
		V("Open file %s", opts.InputFile)
	}

	file, err := os.Open(opts.InputFile)
	if err != nil {
		panic(fmt.Errorf("cannot open tavor file %s: %v", opts.InputFile, err))
	}
	defer file.Close()

	if opts.Debug {
		tavor.DEBUG = true
	}

	doc, err := parser.ParseTavor(file)
	if err != nil {
		panic(fmt.Errorf("cannot parse tavor file: %v", err))
	}

	if opts.Verbose {
		V("File is ok")
	}

	if opts.Validate {
		os.Exit(returnOk)
	}

	if opts.Verbose {
		V("Using seed %d", opts.Seed)
	}

	if opts.Verbose {
		V("Counted %d overall permutations", doc.PermutationsAll())
	}

	r := rand.New(rand.NewSource(opts.Seed))

	strat, err := strategy.New(opts.Strategy, doc)
	if err != nil {
		panic(err)
	}

	if opts.Verbose {
		V("Using %s strategy", opts.Strategy)
	}

	ch := strat.Fuzz(r)
	another := false
	for i := range ch {
		if !tavor.DEBUG {
			if another {
				fmt.Println()
			} else {
				another = true
			}
		}

		if tavor.DEBUG {
			fmt.Printf("Result:\n%s\n", doc.String())
		} else {
			fmt.Print(doc.String())
		}

		ch <- i
	}
}
<|endoftext|>


<commit_before>package main

import (
  "fmt"
  "net"
  "bufio"
  "strings"
  "net/http"
  "time"
)

type IPSet struct {
  root BitNode
}

type BitNode struct {
  parent,zero,one *BitNode
  depth uint32
  full bool
  value uint32
}

func main () {

  ipsetCurrent := createIPSet()
  ipsetNew := createIPSet()

  blocklists := []string{ "https://www.confusticate.com/all.txt","https://www.confusticate.com/drop.txt","https://www.confusticate.com/edrop.txt","https://www.confusticate.com/test.txt"}

  for 1==1 {
    fmt.Println("# start")

    for _, url := range blocklists {
      fmt.Printf("# %s\n",url)

      data := downloadBlocklist(url)

      for _, ipnet := range data {
        ipsetNew.add(&ipnet)
      }
    }

    // If IPs in the new set aren't in the current set, announce them
    for _, newip := range ipsetNew.getAll() {
      current := ipsetCurrent.contains(&newip)
      if current == nil {
        fmt.Printf("announce %s\n", newip.String())
      } else {
        // However, if the thing that matched wasn't identical to the existing one
        // I.e. more specific IP, withdraw the existing one *and* annouce the new one
        if current.String() != newip.String() {
          fmt.Printf("withdraw %s\n", current.String())
          fmt.Printf("announce %s\n", newip.String())
        }
      }
    }

    // If IPs from the current set aren't in the new set
    for _, existing := range ipsetCurrent.getAll() {
      newip := ipsetNew.contains(&existing)

      if newip == nil {
        fmt.Printf("withdraw %s\n", existing.String())
      } else {
        // However, if the thing that matched wasn't identical to the existing one
        // I.e. more specific IP, withdraw the existing one *and* annouce the new one
        if newip != nil && newip.String() != existing.String() {
          fmt.Printf("withdraw %s\n", existing.String())
          fmt.Printf("announce %s\n", newip.String())
        }
      }
    }

    ipsetCurrent = ipsetNew
    ipsetNew = createIPSet()
    fmt.Println("# end")
    time.Sleep(10 * time.Second)
  }
}

func downloadBlocklist(url string) []net.IPNet {
  var nets = make([]net.IPNet,0)
  resp, err := http.Get(url)

  if err != nil {
    fmt.Println(err)
    return nil
  }

  scanner := bufio.NewScanner(resp.Body)
	for scanner.Scan() {
    line := scanner.Text()
    items := strings.Split(line,";")
    ipnet := StringToIPNet(items[0])

    if ipnet != nil {
        nets = append(nets,*ipnet)
    }
	}

  return nets
}

// ====================================================================================================================

func createIPSet() IPSet {
  return IPSet { root:  BitNode{parent: nil, zero :nil, one :nil, depth: 32, full: false, value: 0} }
}

func (ipset *IPSet) add(ipnet *net.IPNet) {
  if ipnet != nil {
      var bits,_ = ipnet.Mask.Size()
      var addr = IPtoInt(ipnet.IP)

      addIP(&ipset.root,addr,uint32(bits))
  }
}

func (ipset *IPSet) getAll() []net.IPNet {
  return collectIPs(&ipset.root)
}

func (ipset *IPSet) contains(ipnet *net.IPNet) *net.IPNet {
  if ipnet != nil {
      var bits,_ = ipnet.Mask.Size()
      var addr = IPtoInt(ipnet.IP)

      return containsIP(&ipset.root,addr,uint32(bits))
  } else {
    return nil
  }
}

func StringToIPNet(text string) *net.IPNet {
  var ip,ipnet,error = net.ParseCIDR(strings.TrimSpace(text))

  if error != nil {
    ip = net.ParseIP(text)
    if ip != nil {
      ipnet = &net.IPNet{IP: ip, Mask: net.CIDRMask(32,32)}
    }
  }

  return ipnet
}

func IPtoInt(addr net.IP) uint32 {
  return uint32(addr.To4()[0]) << 24 |
         uint32(addr.To4()[1]) << 16 |
         uint32(addr.To4()[2]) << 8 |
         uint32(addr.To4()[3])
}

func IntToIP(ip uint32) net.IP {
  result := make(net.IP, 4)
           result[3] = byte(ip)
           result[2] = byte(ip >>8)
           result[1] = byte(ip >>16)
           result[0] = byte(ip >> 24)
  return result
}

func CheckBit(num uint32, bit uint32) bool {
  return (num & (1 << bit)) != 0
}

func outputTree(node *BitNode) {
  if node.full {
    var ipnet = IPNetFromNode(node)
    fmt.Println(ipnet.String())
  } else {
    if node.zero != nil {
      outputTree(node.zero)
    }

    if node.one != nil {
      outputTree(node.one)
    }
  }
}

func collectIPs(node *BitNode) []net.IPNet {
  var nets = make([]net.IPNet,0)

  if node.full {
    var ipnet = IPNetFromNode(node)
    return []net.IPNet{ipnet}
  } else {
    if node.zero != nil {
      nets = append(nets,collectIPs(node.zero)...)
    }

    if node.one != nil {
      nets = append(nets,collectIPs(node.one)...)
    }
  }
  return nets
}

func IPFromNode(node *BitNode) net.IP {
  var cur = node
  var accumulate uint32 = 0

  for cur.parent != nil {
    accumulate |= cur.value << cur.depth
    cur = cur.parent
  }

  return IntToIP(accumulate)
}

func IPNetFromNode(node *BitNode) net.IPNet {
  var cur = node
  var accumulate uint32 = 0
  var mask = int(32 - node.depth)

  for cur.parent != nil {
    accumulate |= cur.value << cur.depth
    cur = cur.parent
  }

  return net.IPNet{IP: IntToIP(accumulate), Mask: net.CIDRMask(mask,32)}
}

func containsIP(node *BitNode, addr uint32, mask uint32) *net.IPNet {
  if node.full {
    ipnet := IPNetFromNode(node)

    return &ipnet
  }

  if 32-node.depth > mask {
    return nil
  }

  if CheckBit(addr,node.depth - 1) {
    if node.one == nil {
      return nil
    } else {
      return containsIP(node.one,addr,mask)
    }
  } else {
    if node.zero == nil {
      return nil
    } else {
      return containsIP(node.zero,addr,mask)
    }
  }
}

func addIP(node *BitNode, addr uint32, mask uint32) bool {

  if node.depth == 0 || 32 - node.depth == mask || node.full {
    node.full = true
    return node.full
  }

  var child *BitNode

  if CheckBit(addr,node.depth - 1) {
    if node.one == nil {
      child = &BitNode{parent: node, zero: nil, one: nil, depth: node.depth -1, full: false, value: 1}
      node.one = child
    } else {
      child = node.one
    }
  } else {
    if node.zero == nil {
      child = &BitNode{parent: node, zero: nil, one: nil, depth: node.depth -1, full: false, value: 0 }
      node.zero = child
    } else {
      child = node.zero
    }
  }

  addIP(child, addr, mask)

  node.full = (node.one != nil && node.one.full) && (node.zero != nil && node.zero.full)
  return node.full
}
<commit_msg>Force GC after each run Correct announce/withdraw syntax<commit_after>package main

import (
  "fmt"
  "net"
  "bufio"
  "strings"
  "net/http"
  "time"
  "runtime"
)

type IPSet struct {
  root BitNode
}

type BitNode struct {
  parent,zero,one *BitNode
  depth uint32
  full bool
  value uint32
}

func main () {

  ipsetCurrent := createIPSet()
  ipsetNew := createIPSet()

  blocklists := []string{ "https://www.confusticate.com/all.txt","https://www.confusticate.com/drop.txt","https://www.confusticate.com/edrop.txt"}

  for 1==1 {
    var announcements = 0
    var withdrawls = 0

    fmt.Println("# start blocklist refresh")

    for _, url := range blocklists {

      data := downloadBlocklist(url)

      fmt.Printf("# %d entries downloaded from %s\n",len(data), url)

      for _, ipnet := range data {
        ipsetNew.add(&ipnet)
      }
    }

    // If IPs in the new set aren't in the current set, announce them
    for _, newip := range ipsetNew.getAll() {
      current := ipsetCurrent.contains(&newip)
      if current == nil {
        fmt.Printf("announce route %s next-hop 192.0.2.1 community [65332:666]\n", newip.String())
        announcements++
      } else {
        // However, if the thing that matched wasn't identical to the existing one
        // I.e. more specific IP, withdraw the existing one *and* annouce the new one
        if current.String() != newip.String() {
          fmt.Printf("withdraw route %s next-hop 192.0.2.1 community [65332:666]\n", current.String())
          fmt.Printf("announce route %s next-hop 192.0.2.1 community [65332:666]\n", newip.String())
          announcements++
          withdrawls++
        }
      }
    }

    // If IPs from the current set aren't in the new set
    for _, existing := range ipsetCurrent.getAll() {
      newip := ipsetNew.contains(&existing)

      if newip == nil {
        fmt.Printf("withdraw route %s next-hop 192.0.2.1 community [65332:666]\n", existing.String())
        withdrawls++
      } else {
        // However, if the thing that matched wasn't identical to the existing one
        // I.e. more specific IP, withdraw the existing one *and* annouce the new one
        if newip != nil && newip.String() != existing.String() {
          fmt.Printf("withdraw route %s next-hop 192.0.2.1 community [65332:666]\n", existing.String())
          fmt.Printf("announce route %s next-hop 192.0.2.1 community [65332:666]\n", newip.String())
          announcements++
          withdrawls++
        }
      }
    }

    ipsetCurrent = ipsetNew
    ipsetNew = createIPSet()
    fmt.Printf("# completed with %d routes announced and %d routes withdrawn\n",announcements, withdrawls)
    runtime.GC()
    time.Sleep(30 * time.Minute)
  }
}

func downloadBlocklist(url string) []net.IPNet {
  var nets = make([]net.IPNet,0)
  resp, err := http.Get(url)

  if err != nil {
    fmt.Println(err)
    return nil
  }

  scanner := bufio.NewScanner(resp.Body)
	for scanner.Scan() {
    line := scanner.Text()
    items := strings.Split(line,";")
    ipnet := StringToIPNet(items[0])

    if ipnet != nil {
        nets = append(nets,*ipnet)
    }
	}

  return nets
}

// ====================================================================================================================

func createIPSet() IPSet {
  return IPSet { root:  BitNode{parent: nil, zero :nil, one :nil, depth: 32, full: false, value: 0} }
}

func (ipset *IPSet) add(ipnet *net.IPNet) {
  if ipnet != nil {
      var bits,_ = ipnet.Mask.Size()
      var addr = IPtoInt(ipnet.IP)

      addIP(&ipset.root,addr,uint32(bits))
  }
}

func (ipset *IPSet) getAll() []net.IPNet {
  return collectIPs(&ipset.root)
}

func (ipset *IPSet) contains(ipnet *net.IPNet) *net.IPNet {
  if ipnet != nil {
      var bits,_ = ipnet.Mask.Size()
      var addr = IPtoInt(ipnet.IP)

      return containsIP(&ipset.root,addr,uint32(bits))
  } else {
    return nil
  }
}

func StringToIPNet(text string) *net.IPNet {
  var ip,ipnet,error = net.ParseCIDR(strings.TrimSpace(text))

  if error != nil {
    ip = net.ParseIP(text)
    if ip != nil {
      ipnet = &net.IPNet{IP: ip, Mask: net.CIDRMask(32,32)}
    }
  }

  return ipnet
}

func IPtoInt(addr net.IP) uint32 {
  return uint32(addr.To4()[0]) << 24 |
         uint32(addr.To4()[1]) << 16 |
         uint32(addr.To4()[2]) << 8 |
         uint32(addr.To4()[3])
}

func IntToIP(ip uint32) net.IP {
  result := make(net.IP, 4)
           result[3] = byte(ip)
           result[2] = byte(ip >>8)
           result[1] = byte(ip >>16)
           result[0] = byte(ip >> 24)
  return result
}

func CheckBit(num uint32, bit uint32) bool {
  return (num & (1 << bit)) != 0
}

func outputTree(node *BitNode) {
  if node.full {
    var ipnet = IPNetFromNode(node)
    fmt.Println(ipnet.String())
  } else {
    if node.zero != nil {
      outputTree(node.zero)
    }

    if node.one != nil {
      outputTree(node.one)
    }
  }
}

func collectIPs(node *BitNode) []net.IPNet {
  var nets = make([]net.IPNet,0)

  if node.full {
    var ipnet = IPNetFromNode(node)
    return []net.IPNet{ipnet}
  } else {
    if node.zero != nil {
      nets = append(nets,collectIPs(node.zero)...)
    }

    if node.one != nil {
      nets = append(nets,collectIPs(node.one)...)
    }
  }
  return nets
}

func IPFromNode(node *BitNode) net.IP {
  var cur = node
  var accumulate uint32 = 0

  for cur.parent != nil {
    accumulate |= cur.value << cur.depth
    cur = cur.parent
  }

  return IntToIP(accumulate)
}

func IPNetFromNode(node *BitNode) net.IPNet {
  var cur = node
  var accumulate uint32 = 0
  var mask = int(32 - node.depth)

  for cur.parent != nil {
    accumulate |= cur.value << cur.depth
    cur = cur.parent
  }

  return net.IPNet{IP: IntToIP(accumulate), Mask: net.CIDRMask(mask,32)}
}

func containsIP(node *BitNode, addr uint32, mask uint32) *net.IPNet {
  if node.full {
    ipnet := IPNetFromNode(node)

    return &ipnet
  }

  if 32-node.depth > mask {
    return nil
  }

  if CheckBit(addr,node.depth - 1) {
    if node.one == nil {
      return nil
    } else {
      return containsIP(node.one,addr,mask)
    }
  } else {
    if node.zero == nil {
      return nil
    } else {
      return containsIP(node.zero,addr,mask)
    }
  }
}

func addIP(node *BitNode, addr uint32, mask uint32) bool {

  if node.depth == 0 || 32 - node.depth == mask || node.full {
    node.full = true
    return node.full
  }

  var child *BitNode

  if CheckBit(addr,node.depth - 1) {
    if node.one == nil {
      child = &BitNode{parent: node, zero: nil, one: nil, depth: node.depth -1, full: false, value: 1}
      node.one = child
    } else {
      child = node.one
    }
  } else {
    if node.zero == nil {
      child = &BitNode{parent: node, zero: nil, one: nil, depth: node.depth -1, full: false, value: 0 }
      node.zero = child
    } else {
      child = node.zero
    }
  }

  addIP(child, addr, mask)

  node.full = (node.one != nil && node.one.full) && (node.zero != nil && node.zero.full)
  return node.full
}
<|endoftext|>


<commit_before>// Copyright 2013 Landon Wainwright. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Contains the base structures
package blog

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"gopkg.in/fsnotify.v1"
	"html/template"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"
)

// The struct for event information
type Event struct {
	Op Op // File operation that triggered the event.
}

// Op describes a set of file operations.
type Op uint32

// These are the generalized file operations that can trigger a notification.
const (
	Update Op = 1 << iota
)

// The base configuration for a new blog.
// The Configuration contains information such as file directories etc
type Configuration struct {
	DevelopmentMode bool
	Postsdir        string
	Templatesdir    string
	Assetsdir       string
	Title           string
}

// Contains the templates that are to be handled by this applicaton
type Templates struct {
	templates []string
}

// Blog is the root data store for this blog
type Blog struct {
	configuration *Configuration
	posts         []*Post
	postMap       map[string]*Post
	templates     *template.Template
}

// Post is a representation of a single post within the blog
type Post struct {
	FileName string
	Created  time.Time
	Updated  time.Time
	Title    string
	Summary  string
	Body     string
}

// This will make the title safe for use within the URL
func (this *Post) SafeTitle() string {

	// Replace all spaces of the title with '-'
	return strings.ToLower(strings.Replace(this.Title, " ", "-", -1))
}

// This will make the title safe for use within the URL
func (this *Post) SafeURL() string {

	// Now make URL safe
	return url.QueryEscape(this.SafeTitle())
}

// Will return the body as HTML (as the html template will automatically escape it by default)
func (this *Post) BodySafe() template.HTML {

	// Return an HTML element
	return template.HTML(this.Body)
}

// Sort functionality to sort the posts in order they were created
type ByCreated []*Post

func (this ByCreated) Len() int           { return len(this) }
func (this ByCreated) Swap(i, j int)      { this[i], this[j] = this[j], this[i] }
func (this ByCreated) Less(i, j int) bool { return this[i].Created.After(this[j].Created) }

// Will create a new Blog serving content from the provided directory
func New(configuration *Configuration) *Blog {

	// New() allocates a new blog
	blog := &Blog{}
	log.Printf("Creating '%s' blog", configuration.Title)
	log.Printf("Loading posts from directory: %s", configuration.Postsdir)
	log.Printf("Loading templates from directory: %s", configuration.Templatesdir)
	log.Printf("Serving assets from directory: %s", configuration.Assetsdir)

	// Initialise the blog values
	return blog.init(configuration)
}

// Init resets the blog data
func (this *Blog) init(configuration *Configuration) *Blog {
	this.configuration = configuration
	this.posts = nil
	this.postMap = make(map[string]*Post)

	// Add the watcher for the post directory
	mutex := &sync.Mutex{}
	updates := this.WatchPosts(configuration.Postsdir)
	go func() {
		for {
			select {
			case event := <-updates:
				if event.Op == Update {
					mutex.Lock()
					log.Print("Reloading Posts")
					this.loadPosts()
					mutex.Unlock()
				}
			}
		}
	}()
	return this
}

// Will return the path for the specific template name
func (this *Blog) getTemplatePath(templateName string) string {

	// Return the path to the template
	return path.Join(this.configuration.Templatesdir, templateName)
}

// Will read all the available posts from the file system
func (this *Blog) loadPosts() error {

	// Open the root application directory where the posts are stored
	// Read in each file and generate the post and tag objects
	fileInfos, err := ioutil.ReadDir(this.configuration.Postsdir)
	if err != nil {
		log.Printf("Cannot read the files from %s", this.configuration.Postsdir)
		return err
	}

	postsno := 0
	this.postMap = make(map[string]*Post)
	log.Printf("Loading posts")
	for _, fi := range fileInfos {

		// Load the file (only .json files should be read)
		if filepath.Ext(fi.Name()) == ".json" {
			filePath := path.Join(this.configuration.Postsdir, fi.Name())
			fi, err := os.Open(filePath)
			defer fi.Close()
			if err == nil {

				// Copy the file contents into the buffer
				var b bytes.Buffer
				_, err := b.ReadFrom(fi)
				if err == nil {

					// Create an empty post to copy the values into
					var post Post
					err := json.Unmarshal(b.Bytes(), &post)
					if err == nil {

						// Is there a post already with the same title?
						for this.postMap[post.SafeTitle()] != nil {

							// Then we need to ensure that this post has a unique name
							post.Title = fmt.Sprintf("%s-", post.Title)
						}

						// Then the data was un-marshalled successfully and the post can be used
						postsno += 1
						post.FileName = fi.Name()
						this.postMap[post.SafeTitle()] = &post
					}
				}
			}
		}
	}
	log.Printf("Finished loading %d posts", postsno)

	// Now sort the posts into the array
	this.posts = make([]*Post, postsno)
	i := 0
	for _, v := range this.postMap {
		this.posts[i] = v
		i += 1
	}

	// Sort the array
	sort.Sort(ByCreated(this.posts))
	return nil
}

// This will write the post to disk
func (this *Blog) SavePost(post Post) error {

	// Add a created time stamp
	if &post.Created == nil {
		log.Println("Created a new time stamp for post")
		post.Created = time.Now()
	}

	// Update the updated time stamp
	post.Updated = time.Now()

	// Marshall this to disk
	b, err := json.Marshal(post)
	if err != nil {
		log.Println("Unable to marshall Post")
		return err
	}

	// Return if the bytes array is empty
	if len(b) == 0 {
		log.Println("The Post contains no content to write to disk")
		return errors.New("There is no content to write to disk")
	}

	// Write the file out to disk

	filePath := path.Join(this.configuration.Postsdir, fmt.Sprintf("%d.json", post.Created.Unix()))
	fo, err := os.Create(filePath)
	defer fo.Close()
	if err != nil {
		log.Println("Unable to create Post: %s", filePath)
		return err
	}

	// Write the bytes to disk
	var buffer bytes.Buffer
	_, err = buffer.Write(b)
	if err != nil {
		log.Println("Unable to write Post bytes to buffer")
		return err
	}
	_, err = buffer.WriteTo(fo)
	return err
}

// This will create a watcher of the directory
func (this *Blog) WatchPosts(directory string) chan Event {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	// Create the channel where events are pushed
	updates := make(chan Event)
	go func() {
		for {
			select {
			case event := <-watcher.Events:
				if event.Op&fsnotify.Write == fsnotify.Write {
					log.Println("modified file:", event.Name)

					// Push the event onto the queue to get the system to update the posts
					updates <- Event{Op: Update}
				}
			}
		}
	}()

	// Attempt to watch the directory
	err = watcher.Add(directory)
	if err != nil {
		log.Fatal(err)
	}
	return updates
}
<commit_msg>The file watcher is now more sensitive and stacks the changes. it only calls the reload posts after the 10 seconds timer. If anything changes during this period, the new timer will be started and the old ones will be cancelled<commit_after>// Copyright 2013 Landon Wainwright. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Contains the base structures
package blog

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"gopkg.in/fsnotify.v1"
	"html/template"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"
)

// The mutex for reading the posts in
var mutex = &sync.Mutex{}

// The struct for event information
type Event struct {
	Op Op // File operation that triggered the event.
}

// Op describes a set of file operations.
type Op uint32

// These are the generalized file operations that can trigger a notification.
const (
	Update Op = 1 << iota
)

// The base configuration for a new blog.
// The Configuration contains information such as file directories etc
type Configuration struct {
	DevelopmentMode bool
	Postsdir        string
	Templatesdir    string
	Assetsdir       string
	Title           string
}

// Contains the templates that are to be handled by this applicaton
type Templates struct {
	templates []string
}

// Blog is the root data store for this blog
type Blog struct {
	configuration *Configuration
	posts         []*Post
	postMap       map[string]*Post
	templates     *template.Template
}

// Post is a representation of a single post within the blog
type Post struct {
	FileName string
	Created  time.Time
	Updated  time.Time
	Title    string
	Summary  string
	Body     string
}

// This will make the title safe for use within the URL
func (this *Post) SafeTitle() string {

	// Replace all spaces of the title with '-'
	return strings.ToLower(strings.Replace(this.Title, " ", "-", -1))
}

// This will make the title safe for use within the URL
func (this *Post) SafeURL() string {

	// Now make URL safe
	return url.QueryEscape(this.SafeTitle())
}

// Will return the body as HTML (as the html template will automatically escape it by default)
func (this *Post) BodySafe() template.HTML {

	// Return an HTML element
	return template.HTML(this.Body)
}

// Sort functionality to sort the posts in order they were created
type ByCreated []*Post

func (this ByCreated) Len() int           { return len(this) }
func (this ByCreated) Swap(i, j int)      { this[i], this[j] = this[j], this[i] }
func (this ByCreated) Less(i, j int) bool { return this[i].Created.After(this[j].Created) }

// Will create a new Blog serving content from the provided directory
func New(configuration *Configuration) *Blog {

	// New() allocates a new blog
	blog := &Blog{}
	log.Printf("Creating '%s' blog", configuration.Title)
	log.Printf("Loading posts from directory: %s", configuration.Postsdir)
	log.Printf("Loading templates from directory: %s", configuration.Templatesdir)
	log.Printf("Serving assets from directory: %s", configuration.Assetsdir)

	// Initialise the blog values
	return blog.init(configuration)
}

// Init resets the blog data
func (this *Blog) init(configuration *Configuration) *Blog {
	this.configuration = configuration
	this.posts = nil
	this.postMap = make(map[string]*Post)

	// Add the watcher for the post directory
	updates := this.WatchPosts(configuration.Postsdir)

	// This is used to exit out of the current timer handlers
	timerExit := make(chan bool)

	// Start listening for the update events
	go func() {
		for {
			select {
			case event := <-updates:
				if event.Op == Update {

					// Cancel any existing timers
					select {
					case timerExit <- false:
					default:
					}

					// This is the function that wil be called when the timer is started
					go handlePostTimer(this, time.NewTimer(time.Second*10), timerExit)
				}
			}
		}
	}()
	return this
}

// This will call the reload posts when the timer has ended or exit when the exit channel is called
func handlePostTimer(blog *Blog, timer *time.Timer, exit chan bool) {
	select {
	case <-timer.C:

		// Now reload the posts
		log.Println("Post directory has changed")
		blog.loadPosts()
	case <-exit:

		// This will drop out of the block
		timer.Stop()
	}
}

// Will return the path for the specific template name
func (this *Blog) getTemplatePath(templateName string) string {

	// Return the path to the template
	return path.Join(this.configuration.Templatesdir, templateName)
}

// Will read all the available posts from the file system
func (this *Blog) loadPosts() error {
	mutex.Lock()
	defer mutex.Unlock()

	// Open the root application directory where the posts are stored
	// Read in each file and generate the post and tag objects
	fileInfos, err := ioutil.ReadDir(this.configuration.Postsdir)
	if err != nil {
		log.Printf("Cannot read the files from %s", this.configuration.Postsdir)
		return err
	}

	postsno := 0
	this.postMap = make(map[string]*Post)
	log.Printf("Loading posts")
	for _, fi := range fileInfos {

		// Load the file (only .json files should be read)
		if filepath.Ext(fi.Name()) == ".json" {
			filePath := path.Join(this.configuration.Postsdir, fi.Name())
			fi, err := os.Open(filePath)
			defer fi.Close()
			if err == nil {

				// Copy the file contents into the buffer
				var b bytes.Buffer
				_, err := b.ReadFrom(fi)
				if err == nil {

					// Create an empty post to copy the values into
					var post Post
					err := json.Unmarshal(b.Bytes(), &post)
					if err == nil {

						// Is there a post already with the same title?
						for this.postMap[post.SafeTitle()] != nil {

							// Then we need to ensure that this post has a unique name
							post.Title = fmt.Sprintf("%s-", post.Title)
						}

						// Then the data was un-marshalled successfully and the post can be used
						postsno += 1
						post.FileName = fi.Name()
						this.postMap[post.SafeTitle()] = &post
					}
				}
			}
		}
	}
	log.Printf("Finished loading %d posts", postsno)

	// Now sort the posts into the array
	this.posts = make([]*Post, postsno)
	i := 0
	for _, v := range this.postMap {
		this.posts[i] = v
		i += 1
	}

	// Sort the array
	sort.Sort(ByCreated(this.posts))
	return nil
}

// This will write the post to disk
func (this *Blog) SavePost(post Post) error {

	// Add a created time stamp
	if &post.Created == nil {
		log.Println("Created a new time stamp for post")
		post.Created = time.Now()
	}

	// Update the updated time stamp
	post.Updated = time.Now()

	// Marshall this to disk
	b, err := json.Marshal(post)
	if err != nil {
		log.Println("Unable to marshall Post")
		return err
	}

	// Return if the bytes array is empty
	if len(b) == 0 {
		log.Println("The Post contains no content to write to disk")
		return errors.New("There is no content to write to disk")
	}

	// Write the file out to disk

	filePath := path.Join(this.configuration.Postsdir, fmt.Sprintf("%d.json", post.Created.Unix()))
	fo, err := os.Create(filePath)
	defer fo.Close()
	if err != nil {
		log.Println("Unable to create Post: %s", filePath)
		return err
	}

	// Write the bytes to disk
	var buffer bytes.Buffer
	_, err = buffer.Write(b)
	if err != nil {
		log.Println("Unable to write Post bytes to buffer")
		return err
	}
	_, err = buffer.WriteTo(fo)
	return err
}

// This will create a watcher of the directory
func (this *Blog) WatchPosts(directory string) chan Event {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	// Create the channel where events are pushed
	updates := make(chan Event)
	go func() {
		for {
			select {
			case event := <-watcher.Events:
				if event.Op&fsnotify.Write == fsnotify.Write {

					// Push the event onto the queue to get the system to update the posts
					updates <- Event{Op: Update}
				}
			}
		}
	}()

	// Attempt to watch the directory
	err = watcher.Add(directory)
	if err != nil {
		log.Fatal(err)
	}
	return updates
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package yarpc_test

import (
	"bytes"
	"context"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"testing"
	"time"

	"go.uber.org/yarpc"
	"go.uber.org/yarpc/encoding/raw"
	yhttp "go.uber.org/yarpc/transport/http"
	ytchannel "go.uber.org/yarpc/transport/tchannel"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"github.com/uber/tchannel-go"
	traw "github.com/uber/tchannel-go/raw"
	ncontext "golang.org/x/net/context"
)

var _reqBody = []byte("hello")

func yarpcEcho(ctx context.Context, body []byte) ([]byte, error) {
	call := yarpc.CallFromContext(ctx)
	for _, k := range call.HeaderNames() {
		if err := call.WriteResponseHeader(k, call.Header(k)); err != nil {
			return nil, err
		}
	}
	return body, nil
}

func httpEcho(t testing.TB) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		defer r.Body.Close()
		hs := w.Header()
		for k, vs := range r.Header {
			hs[k] = vs
		}

		_, err := io.Copy(w, r.Body)
		assert.NoError(t, err, "failed to write HTTP response body")
	}
}

type tchannelEcho struct{ t testing.TB }

func (tchannelEcho) Handle(ctx ncontext.Context, args *traw.Args) (*traw.Res, error) {
	return &traw.Res{Arg2: args.Arg2, Arg3: args.Arg3}, nil
}

func (t tchannelEcho) OnError(ctx ncontext.Context, err error) {
	t.t.Fatalf("request failed: %v", err)
}

func withDispatcher(t testing.TB, cfg yarpc.Config, f func(*yarpc.Dispatcher)) {
	d := yarpc.NewDispatcher(cfg)
	require.NoError(t, d.Start(), "failed to start server")
	defer d.Stop()

	f(d)
}

func withHTTPServer(t testing.TB, listenOn string, h http.Handler, f func()) {
	l, err := net.Listen("tcp", listenOn)
	require.NoError(t, err, "could not listen on %q", listenOn)

	ch := make(chan struct{})
	go func() {
		http.Serve(l, h)
		close(ch)
	}()
	f()
	assert.NoError(t, l.Close(), "failed to stop listener on %q", listenOn)
	<-ch // wait until server has stopped
}

func runYARPCClient(b *testing.B, c raw.Client) {
	for i := 0; i < b.N; i++ {
		ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
		defer cancel()
		_, err := c.Call(ctx, "echo", _reqBody)
		require.NoError(b, err, "request %d failed", i+1)
	}
}

func runHTTPClient(b *testing.B, c *http.Client, url string) {
	for i := 0; i < b.N; i++ {
		ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
		defer cancel()
		req, err := http.NewRequest("POST", url, bytes.NewReader(_reqBody))
		require.NoError(b, err, "failed to build request %d", i+1)
		req = req.WithContext(ctx)

		req.Header = http.Header{
			"Context-TTL-MS": {"100"},
			"Rpc-Caller":     {"http-client"},
			"Rpc-Encoding":   {"raw"},
			"Rpc-Procedure":  {"echo"},
			"Rpc-Service":    {"server"},
		}
		res, err := c.Do(req)
		require.NoError(b, err, "request %d failed", i+1)

		_, err = ioutil.ReadAll(res.Body)
		require.NoError(b, err, "failed to read response %d", i+1)
		require.NoError(b, res.Body.Close(), "failed to close response body %d", i+1)
	}
}

func runTChannelClient(b *testing.B, c *tchannel.Channel, hostPort string) {
	headers := []byte{0x00, 0x00} // TODO: YARPC TChannel should support empty arg2
	for i := 0; i < b.N; i++ {
		ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
		defer cancel()
		call, err := c.BeginCall(ctx, hostPort, "server", "echo",
			&tchannel.CallOptions{Format: tchannel.Raw})
		require.NoError(b, err, "BeginCall %v failed", i+1)

		_, _, _, err = traw.WriteArgs(call, headers, _reqBody)
		require.NoError(b, err, "request %v failed", i+1)
	}
}

func Benchmark_HTTP_YARPCToYARPC(b *testing.B) {
	httpTransport := yhttp.NewTransport()
	serverCfg := yarpc.Config{
		Name:     "server",
		Inbounds: yarpc.Inbounds{httpTransport.NewInbound(":8999")},
	}

	clientCfg := yarpc.Config{
		Name: "client",
		Outbounds: yarpc.Outbounds{
			"server": {
				Unary: httpTransport.NewSingleOutbound("http://localhost:8999"),
			},
		},
	}

	withDispatcher(b, serverCfg, func(server *yarpc.Dispatcher) {
		server.Register(raw.Procedure("echo", yarpcEcho))
		withDispatcher(b, clientCfg, func(client *yarpc.Dispatcher) {
			b.ResetTimer()
			runYARPCClient(b, raw.New(client.ClientConfig("server")))
		})
	})
}

func Benchmark_HTTP_YARPCToNetHTTP(b *testing.B) {
	httpTransport := yhttp.NewTransport()
	clientCfg := yarpc.Config{
		Name: "client",
		Outbounds: yarpc.Outbounds{
			"server": {
				Unary: httpTransport.NewSingleOutbound("http://localhost:8998"),
			},
		},
	}

	withHTTPServer(b, ":8998", httpEcho(b), func() {
		withDispatcher(b, clientCfg, func(client *yarpc.Dispatcher) {
			b.ResetTimer()
			runYARPCClient(b, raw.New(client.ClientConfig("server")))
		})
	})
}

func Benchmark_HTTP_NetHTTPToYARPC(b *testing.B) {
	httpTransport := yhttp.NewTransport()
	serverCfg := yarpc.Config{
		Name:     "server",
		Inbounds: yarpc.Inbounds{httpTransport.NewInbound(":8996")},
	}

	withDispatcher(b, serverCfg, func(server *yarpc.Dispatcher) {
		server.Register(raw.Procedure("echo", yarpcEcho))

		b.ResetTimer()
		runHTTPClient(b, http.DefaultClient, "http://localhost:8996")
	})
}

func Benchmark_HTTP_NetHTTPToNetHTTP(b *testing.B) {
	withHTTPServer(b, ":8997", httpEcho(b), func() {
		b.ResetTimer()
		runHTTPClient(b, http.DefaultClient, "http://localhost:8997")
	})
}

func Benchmark_TChannel_YARPCToYARPC(b *testing.B) {
	serverTChannel, err := ytchannel.NewChannelTransport(ytchannel.ServiceName("server"))
	require.NoError(b, err)

	serverCfg := yarpc.Config{
		Name:     "server",
		Inbounds: yarpc.Inbounds{serverTChannel.NewInbound()},
	}

	clientTChannel, err := ytchannel.NewChannelTransport(ytchannel.ServiceName("client"))
	require.NoError(b, err)

	// no defer close on channels because YARPC will take care of that

	withDispatcher(b, serverCfg, func(server *yarpc.Dispatcher) {
		server.Register(raw.Procedure("echo", yarpcEcho))

		// Need server already started to build client config
		clientCfg := yarpc.Config{
			Name: "client",
			Outbounds: yarpc.Outbounds{
				"server": {
					Unary: clientTChannel.NewSingleOutbound(serverTChannel.ListenAddr()),
				},
			},
		}
		withDispatcher(b, clientCfg, func(client *yarpc.Dispatcher) {
			b.ResetTimer()
			runYARPCClient(b, raw.New(client.ClientConfig("server")))
		})
	})
}

func Benchmark_TChannel_YARPCToTChannel(b *testing.B) {
	serverCh, err := tchannel.NewChannel("server", nil)
	require.NoError(b, err, "failed to build server TChannel")
	defer serverCh.Close()

	serverCh.Register(traw.Wrap(tchannelEcho{t: b}), "echo")
	require.NoError(b, serverCh.ListenAndServe(":0"), "failed to start up TChannel")

	clientTChannel, err := ytchannel.NewChannelTransport(ytchannel.ServiceName("client"))
	require.NoError(b, err)

	clientCfg := yarpc.Config{
		Name: "client",
		Outbounds: yarpc.Outbounds{
			"server": {
				Unary: clientTChannel.NewSingleOutbound(serverCh.PeerInfo().HostPort),
			},
		},
	}

	withDispatcher(b, clientCfg, func(client *yarpc.Dispatcher) {
		b.ResetTimer()
		runYARPCClient(b, raw.New(client.ClientConfig("server")))
	})
}

func Benchmark_TChannel_TChannelToYARPC(b *testing.B) {
	tchannelTransport, err := ytchannel.NewChannelTransport(ytchannel.ServiceName("server"))
	require.NoError(b, err)

	serverCfg := yarpc.Config{
		Name:     "server",
		Inbounds: yarpc.Inbounds{tchannelTransport.NewInbound()},
	}

	withDispatcher(b, serverCfg, func(dispatcher *yarpc.Dispatcher) {
		dispatcher.Register(raw.Procedure("echo", yarpcEcho))

		clientCh, err := tchannel.NewChannel("client", nil)
		require.NoError(b, err, "failed to build client TChannel")
		defer clientCh.Close()

		b.ResetTimer()
		runTChannelClient(b, clientCh, tchannelTransport.ListenAddr())
	})
}

func Benchmark_TChannel_TChannelToTChannel(b *testing.B) {
	serverCh, err := tchannel.NewChannel("server", nil)
	require.NoError(b, err, "failed to build server TChannel")
	defer serverCh.Close()

	serverCh.Register(traw.Wrap(tchannelEcho{t: b}), "echo")
	require.NoError(b, serverCh.ListenAndServe(":0"), "failed to start up TChannel")

	clientCh, err := tchannel.NewChannel("client", nil)
	require.NoError(b, err, "failed to build client TChannel")
	defer clientCh.Close()

	b.ResetTimer()
	runTChannelClient(b, clientCh, serverCh.PeerInfo().HostPort)
}
<commit_msg>dispatcher: Repair benchmarks (#755)<commit_after>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package yarpc_test

import (
	"bytes"
	"context"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"testing"
	"time"

	"go.uber.org/yarpc"
	"go.uber.org/yarpc/api/transport"
	"go.uber.org/yarpc/encoding/raw"
	yhttp "go.uber.org/yarpc/transport/http"
	ytchannel "go.uber.org/yarpc/transport/tchannel"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"github.com/uber/tchannel-go"
	traw "github.com/uber/tchannel-go/raw"
	ncontext "golang.org/x/net/context"
)

var _reqBody = []byte("hello")

func yarpcEcho(ctx context.Context, body []byte) ([]byte, error) {
	call := yarpc.CallFromContext(ctx)
	for _, k := range call.HeaderNames() {
		if err := call.WriteResponseHeader(k, call.Header(k)); err != nil {
			return nil, err
		}
	}
	return body, nil
}

func httpEcho(t testing.TB) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		defer r.Body.Close()
		hs := w.Header()
		for k, vs := range r.Header {
			hs[k] = vs
		}

		_, err := io.Copy(w, r.Body)
		if err != nil {
			t.Errorf("failed to write HTTP response body: %v", err)
		}
	}
}

type tchannelEcho struct{ t testing.TB }

func (tchannelEcho) Handle(ctx ncontext.Context, args *traw.Args) (*traw.Res, error) {
	return &traw.Res{Arg2: args.Arg2, Arg3: args.Arg3}, nil
}

func (t tchannelEcho) OnError(ctx ncontext.Context, err error) {
	t.t.Fatalf("request failed: %v", err)
}

func withDispatcher(t testing.TB, cfg yarpc.Config, f func(*yarpc.Dispatcher), ps ...[]transport.Procedure) {
	d := yarpc.NewDispatcher(cfg)
	for _, p := range ps {
		d.Register(p)
	}
	require.NoError(t, d.Start(), "failed to start server")
	defer d.Stop()

	f(d)
}

func withHTTPServer(t testing.TB, listenOn string, h http.Handler, f func()) {
	l, err := net.Listen("tcp", listenOn)
	require.NoError(t, err, "could not listen on %q", listenOn)

	ch := make(chan struct{})
	go func() {
		http.Serve(l, h)
		close(ch)
	}()
	f()
	assert.NoError(t, l.Close(), "failed to stop listener on %q", listenOn)
	<-ch // wait until server has stopped
}

func runYARPCClient(b *testing.B, c raw.Client) {
	for i := 0; i < b.N; i++ {
		ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
		defer cancel()
		_, err := c.Call(ctx, "echo", _reqBody)
		if err != nil {
			b.Errorf("request %d failed: %v", i+1, err)
		}
	}
}

func runHTTPClient(b *testing.B, c *http.Client, url string) {
	for i := 0; i < b.N; i++ {
		ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
		defer cancel()
		req, err := http.NewRequest("POST", url, bytes.NewReader(_reqBody))
		if err != nil {
			b.Errorf("failed to build request %d: %v", i+1, err)
		}
		req = req.WithContext(ctx)

		req.Header = http.Header{
			"Context-TTL-MS": {"100"},
			"Rpc-Caller":     {"http-client"},
			"Rpc-Encoding":   {"raw"},
			"Rpc-Procedure":  {"echo"},
			"Rpc-Service":    {"server"},
		}
		res, err := c.Do(req)
		if err != nil {
			b.Errorf("request %d failed: %v", i+1, err)
		}

		if _, err := ioutil.ReadAll(res.Body); err != nil {
			b.Errorf("failed to read response %d: %v", i+1, err)
		}
		if err := res.Body.Close(); err != nil {
			b.Errorf("failed to close response body %d: %v", i+1, err)
		}
	}
}

func runTChannelClient(b *testing.B, c *tchannel.Channel, hostPort string) {
	headers := []byte{0x00, 0x00} // TODO: YARPC TChannel should support empty arg2
	for i := 0; i < b.N; i++ {
		ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
		defer cancel()
		call, err := c.BeginCall(ctx, hostPort, "server", "echo",
			&tchannel.CallOptions{Format: tchannel.Raw})

		if err != nil {
			b.Errorf("BeginCall %v failed: %v", i+1, err)
		}

		_, _, _, err = traw.WriteArgs(call, headers, _reqBody)
		if err != nil {
			b.Errorf("request %v failed: %v", i+1, err)
		}
	}
}

func Benchmark_HTTP_YARPCToYARPC(b *testing.B) {
	httpTransport := yhttp.NewTransport()
	serverCfg := yarpc.Config{
		Name:     "server",
		Inbounds: yarpc.Inbounds{httpTransport.NewInbound(":8999")},
	}

	clientCfg := yarpc.Config{
		Name: "client",
		Outbounds: yarpc.Outbounds{
			"server": {
				Unary: httpTransport.NewSingleOutbound("http://localhost:8999"),
			},
		},
	}

	withDispatcher(
		b, serverCfg,
		func(server *yarpc.Dispatcher) {
			withDispatcher(b, clientCfg, func(client *yarpc.Dispatcher) {
				b.ResetTimer()
				runYARPCClient(b, raw.New(client.ClientConfig("server")))
			})
		},
		raw.Procedure("echo", yarpcEcho),
	)
}

func Benchmark_HTTP_YARPCToNetHTTP(b *testing.B) {
	httpTransport := yhttp.NewTransport()
	clientCfg := yarpc.Config{
		Name: "client",
		Outbounds: yarpc.Outbounds{
			"server": {
				Unary: httpTransport.NewSingleOutbound("http://localhost:8998"),
			},
		},
	}

	withHTTPServer(b, ":8998", httpEcho(b), func() {
		withDispatcher(b, clientCfg, func(client *yarpc.Dispatcher) {
			b.ResetTimer()
			runYARPCClient(b, raw.New(client.ClientConfig("server")))
		})
	})
}

func Benchmark_HTTP_NetHTTPToYARPC(b *testing.B) {
	httpTransport := yhttp.NewTransport()
	serverCfg := yarpc.Config{
		Name:     "server",
		Inbounds: yarpc.Inbounds{httpTransport.NewInbound(":8996")},
	}

	withDispatcher(
		b, serverCfg, func(server *yarpc.Dispatcher) {
			b.ResetTimer()
			runHTTPClient(b, http.DefaultClient, "http://localhost:8996")
		},
		raw.Procedure("echo", yarpcEcho),
	)
}

func Benchmark_HTTP_NetHTTPToNetHTTP(b *testing.B) {
	withHTTPServer(b, ":8997", httpEcho(b), func() {
		b.ResetTimer()
		runHTTPClient(b, http.DefaultClient, "http://localhost:8997")
	})
}

func Benchmark_TChannel_YARPCToYARPC(b *testing.B) {
	serverTChannel, err := ytchannel.NewChannelTransport(ytchannel.ServiceName("server"))
	require.NoError(b, err)

	serverCfg := yarpc.Config{
		Name:     "server",
		Inbounds: yarpc.Inbounds{serverTChannel.NewInbound()},
	}

	clientTChannel, err := ytchannel.NewChannelTransport(ytchannel.ServiceName("client"))
	require.NoError(b, err)

	// no defer close on channels because YARPC will take care of that

	withDispatcher(
		b, serverCfg, func(server *yarpc.Dispatcher) {
			// Need server already started to build client config
			clientCfg := yarpc.Config{
				Name: "client",
				Outbounds: yarpc.Outbounds{
					"server": {
						Unary: clientTChannel.NewSingleOutbound(serverTChannel.ListenAddr()),
					},
				},
			}
			withDispatcher(b, clientCfg, func(client *yarpc.Dispatcher) {
				b.ResetTimer()
				runYARPCClient(b, raw.New(client.ClientConfig("server")))
			})
		},
		raw.Procedure("echo", yarpcEcho),
	)
}

func Benchmark_TChannel_YARPCToTChannel(b *testing.B) {
	serverCh, err := tchannel.NewChannel("server", nil)
	require.NoError(b, err, "failed to build server TChannel")
	defer serverCh.Close()

	serverCh.Register(traw.Wrap(tchannelEcho{t: b}), "echo")
	require.NoError(b, serverCh.ListenAndServe(":0"), "failed to start up TChannel")

	clientTChannel, err := ytchannel.NewChannelTransport(ytchannel.ServiceName("client"))
	require.NoError(b, err)

	clientCfg := yarpc.Config{
		Name: "client",
		Outbounds: yarpc.Outbounds{
			"server": {
				Unary: clientTChannel.NewSingleOutbound(serverCh.PeerInfo().HostPort),
			},
		},
	}

	withDispatcher(b, clientCfg, func(client *yarpc.Dispatcher) {
		b.ResetTimer()
		runYARPCClient(b, raw.New(client.ClientConfig("server")))
	})
}

func Benchmark_TChannel_TChannelToYARPC(b *testing.B) {
	tchannelTransport, err := ytchannel.NewChannelTransport(ytchannel.ServiceName("server"))
	require.NoError(b, err)

	serverCfg := yarpc.Config{
		Name:     "server",
		Inbounds: yarpc.Inbounds{tchannelTransport.NewInbound()},
	}

	withDispatcher(
		b, serverCfg, func(dispatcher *yarpc.Dispatcher) {
			dispatcher.Register(raw.Procedure("echo", yarpcEcho))

			clientCh, err := tchannel.NewChannel("client", nil)
			require.NoError(b, err, "failed to build client TChannel")
			defer clientCh.Close()

			b.ResetTimer()
			runTChannelClient(b, clientCh, tchannelTransport.ListenAddr())
		},
		raw.Procedure("echo", yarpcEcho),
	)
}

func Benchmark_TChannel_TChannelToTChannel(b *testing.B) {
	serverCh, err := tchannel.NewChannel("server", nil)
	require.NoError(b, err, "failed to build server TChannel")
	defer serverCh.Close()

	serverCh.Register(traw.Wrap(tchannelEcho{t: b}), "echo")
	require.NoError(b, serverCh.ListenAndServe(":0"), "failed to start up TChannel")

	clientCh, err := tchannel.NewChannel("client", nil)
	require.NoError(b, err, "failed to build client TChannel")
	defer clientCh.Close()

	b.ResetTimer()
	runTChannelClient(b, clientCh, serverCh.PeerInfo().HostPort)
}
<|endoftext|>


<commit_before>//	Package premailer is for inline styling
//
//	import (
//		"fmt"
//		"github.com/vanng822/go-premailer/premailer"
//		"log"
//	)
//
//	func main() {
//		prem := premailer.NewPremailerFromFile(inputFile)
//		html, err := prem.Transform()
//		if err != nil {
//			log.Fatal(err)
//		}
//
//		fmt.Println(html)
//	}

package premailer

<commit_msg>Godoc seems not detect this formatting<commit_after>// Package premailer is for inline styling.
//
// 	import (
// 		"fmt"
// 		"github.com/vanng822/go-premailer/premailer"
// 		"log"
//	)
//
//	func main() {
//		prem := premailer.NewPremailerFromFile(inputFile)
//		html, err := prem.Transform()
//		if err != nil {
//			log.Fatal(err)
//		}
//
//		fmt.Println(html)
//	}

package premailer

<|endoftext|>


<commit_before>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"net"
	"syscall"
	"time"

	"github.com/cilium/cilium/api/v1/models"
	"github.com/cilium/cilium/monitor/payload"
	"github.com/cilium/cilium/pkg/bpf"
	"github.com/cilium/cilium/pkg/lock"
)

const (
	pollTimeout = 5000

	// queueSize is the size of the message queue
	queueSize = 65536
)

var (
	mutex         lock.Mutex
	listeners     = make(map[*monitorListener]struct{})
	monitorEvents *bpf.PerCpuEvents
)

type monitorListener struct {
	conn  net.Conn
	queue chan []byte
}

func newMonitorListener(c net.Conn) *monitorListener {
	ml := &monitorListener{
		conn:  c,
		queue: make(chan []byte, queueSize),
	}

	go ml.drainQueue()

	return ml
}

// Monitor structure for centralizing the responsibilities of the main events reader.
type Monitor struct {
}

// agentPipeReader reads agent events from the agentPipe and distributes to all listeners
func (m *Monitor) agentPipeReader(agentPipe io.Reader, stop chan struct{}) {
	meta, p := payload.Meta{}, payload.Payload{}

	for {
		select {
		default:
			err := payload.ReadMetaPayload(agentPipe, &meta, &p)
			if err == io.EOF || err == io.ErrUnexpectedEOF {
				log.Panic("Agent pipe closed, shutting down")
			} else if err != nil {
				log.WithError(err).Panic("Unable to read from agent pipe")
			}

			m.send(p)

		case <-stop:
			return
		}
	}
}

// Run starts monitoring.
func (m *Monitor) Run(npages int, agentPipe io.Reader) {
	stopAgentPipeReader := make(chan struct{})
	go m.agentPipeReader(agentPipe, stopAgentPipeReader)
	defer close(stopAgentPipeReader)

	c := bpf.DefaultPerfEventConfig()
	c.NumPages = npages

	me, err := bpf.NewPerCpuEvents(c)
	if err != nil {
		log.WithError(err).Error("Error while starting monitor")
		return
	}
	monitorEvents = me

	last := time.Now()
	// Main event loop
	for {
		todo, err := monitorEvents.Poll(pollTimeout)
		if err != nil {
			log.WithError(err).Error("Error in Poll")
			if err == syscall.EBADF {
				break
			}
		}
		if todo > 0 {
			if err := monitorEvents.ReadAll(m.receiveEvent, m.lostEvent); err != nil {
				log.WithError(err).Warn("Error received while reading from perf buffer")
			}
		}

		if time.Since(last) > 5*time.Second {
			last = time.Now()
			m.dumpStat()
		}
	}
}

// dumpStat prints out the monitor status in JSON.
func (m *Monitor) dumpStat() {
	c := int64(monitorEvents.Cpus)
	n := int64(monitorEvents.Npages)
	p := int64(monitorEvents.Pagesize)
	l, u := monitorEvents.Stats()
	ms := models.MonitorStatus{Cpus: c, Npages: n, Pagesize: p, Lost: int64(l), Unknown: int64(u)}

	mp, err := json.Marshal(ms)
	if err != nil {
		log.WithError(err).Error("error marshalling JSON")
		return
	}
	fmt.Println(string(mp))
}

// handleConnection handles all the incoming connections.
func (m *Monitor) handleConnection(server net.Listener) {
	for {
		conn, err := server.Accept()
		if err != nil {
			log.WithError(err).Warn("error accepting connection")
			continue
		}

		mutex.Lock()
		listeners[newMonitorListener(conn)] = struct{}{}
		log.WithField("count.listener", len(listeners)).Info("New monitor connected.")
		mutex.Unlock()
	}
}

// send writes the payload.Meta and the actual payload to the active
// connections.
func (m *Monitor) send(pl payload.Payload) {
	mutex.Lock()
	defer mutex.Unlock()
	if len(listeners) == 0 {
		return
	}

	buf, err := pl.BuildMessage()
	if err != nil {
		log.WithError(err).Error("Unable to send notification to listeners")
	}

	for ml := range listeners {
		ml.enqueue(buf)
	}
}

func (ml *monitorListener) remove() {
	mutex.Lock()
	delete(listeners, ml)
	mutex.Unlock()
}

func (ml *monitorListener) enqueue(msg []byte) {
	select {
	case ml.queue <- msg:
	default:
		log.Debugf("Per listener queue is full, dropping message")
	}
}

func (ml *monitorListener) drainQueue() {
	for {
		msgBuf := <-ml.queue
		if _, err := ml.conn.Write(msgBuf); err != nil {
			ml.conn.Close()
			ml.remove()
			log.WithError(err).Warn("Monitor removed due to write failure")
			return
		}
	}
}

func (m *Monitor) receiveEvent(es *bpf.PerfEventSample, c int) {
	pl := payload.Payload{Data: es.DataCopy(), CPU: c, Lost: 0, Type: payload.EventSample}
	m.send(pl)
}

func (m *Monitor) lostEvent(el *bpf.PerfEventLost, c int) {
	pl := payload.Payload{Data: []byte{}, CPU: c, Lost: el.Lost, Type: payload.RecordLost}
	m.send(pl)
}
<commit_msg>Log monitor client disconnect nicely<commit_after>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"net"
	"os"
	"syscall"
	"time"

	"github.com/cilium/cilium/api/v1/models"
	"github.com/cilium/cilium/monitor/payload"
	"github.com/cilium/cilium/pkg/bpf"
	"github.com/cilium/cilium/pkg/lock"
)

const (
	pollTimeout = 5000

	// queueSize is the size of the message queue
	queueSize = 65536
)

var (
	mutex         lock.Mutex
	listeners     = make(map[*monitorListener]struct{})
	monitorEvents *bpf.PerCpuEvents
)

type monitorListener struct {
	conn  net.Conn
	queue chan []byte
}

func newMonitorListener(c net.Conn) *monitorListener {
	ml := &monitorListener{
		conn:  c,
		queue: make(chan []byte, queueSize),
	}

	go ml.drainQueue()

	return ml
}

// Monitor structure for centralizing the responsibilities of the main events reader.
type Monitor struct {
}

// agentPipeReader reads agent events from the agentPipe and distributes to all listeners
func (m *Monitor) agentPipeReader(agentPipe io.Reader, stop chan struct{}) {
	meta, p := payload.Meta{}, payload.Payload{}

	for {
		select {
		default:
			err := payload.ReadMetaPayload(agentPipe, &meta, &p)
			if err == io.EOF || err == io.ErrUnexpectedEOF {
				log.Panic("Agent pipe closed, shutting down")
			} else if err != nil {
				log.WithError(err).Panic("Unable to read from agent pipe")
			}

			m.send(p)

		case <-stop:
			return
		}
	}
}

// Run starts monitoring.
func (m *Monitor) Run(npages int, agentPipe io.Reader) {
	stopAgentPipeReader := make(chan struct{})
	go m.agentPipeReader(agentPipe, stopAgentPipeReader)
	defer close(stopAgentPipeReader)

	c := bpf.DefaultPerfEventConfig()
	c.NumPages = npages

	me, err := bpf.NewPerCpuEvents(c)
	if err != nil {
		log.WithError(err).Error("Error while starting monitor")
		return
	}
	monitorEvents = me

	last := time.Now()
	// Main event loop
	for {
		todo, err := monitorEvents.Poll(pollTimeout)
		if err != nil {
			log.WithError(err).Error("Error in Poll")
			if err == syscall.EBADF {
				break
			}
		}
		if todo > 0 {
			if err := monitorEvents.ReadAll(m.receiveEvent, m.lostEvent); err != nil {
				log.WithError(err).Warn("Error received while reading from perf buffer")
			}
		}

		if time.Since(last) > 5*time.Second {
			last = time.Now()
			m.dumpStat()
		}
	}
}

// dumpStat prints out the monitor status in JSON.
func (m *Monitor) dumpStat() {
	c := int64(monitorEvents.Cpus)
	n := int64(monitorEvents.Npages)
	p := int64(monitorEvents.Pagesize)
	l, u := monitorEvents.Stats()
	ms := models.MonitorStatus{Cpus: c, Npages: n, Pagesize: p, Lost: int64(l), Unknown: int64(u)}

	mp, err := json.Marshal(ms)
	if err != nil {
		log.WithError(err).Error("error marshalling JSON")
		return
	}
	fmt.Println(string(mp))
}

// handleConnection handles all the incoming connections.
func (m *Monitor) handleConnection(server net.Listener) {
	for {
		conn, err := server.Accept()
		if err != nil {
			log.WithError(err).Warn("error accepting connection")
			continue
		}

		mutex.Lock()
		listeners[newMonitorListener(conn)] = struct{}{}
		log.WithField("count.listener", len(listeners)).Info("New monitor connected.")
		mutex.Unlock()
	}
}

// send writes the payload.Meta and the actual payload to the active
// connections.
func (m *Monitor) send(pl payload.Payload) {
	mutex.Lock()
	defer mutex.Unlock()
	if len(listeners) == 0 {
		return
	}

	buf, err := pl.BuildMessage()
	if err != nil {
		log.WithError(err).Error("Unable to send notification to listeners")
	}

	for ml := range listeners {
		ml.enqueue(buf)
	}
}

func (ml *monitorListener) remove() {
	mutex.Lock()
	delete(listeners, ml)
	mutex.Unlock()
}

func (ml *monitorListener) enqueue(msg []byte) {
	select {
	case ml.queue <- msg:
	default:
		log.Debugf("Per listener queue is full, dropping message")
	}
}

func (ml *monitorListener) drainQueue() {
	for {
		msgBuf := <-ml.queue
		if _, err := ml.conn.Write(msgBuf); err != nil {
			ml.conn.Close()
			ml.remove()

			if op, ok := err.(*net.OpError); ok {
				if syscerr, ok := op.Err.(*os.SyscallError); ok {
					if errn, ok := syscerr.Err.(syscall.Errno); ok {
						if errn == syscall.EPIPE {
							log.Info("Monitor client disconnected")
							return
						}
					}
				}
			}
			log.WithError(err).Warn("Monitor removed due to write failure")
			return
		}
	}
}

func (m *Monitor) receiveEvent(es *bpf.PerfEventSample, c int) {
	pl := payload.Payload{Data: es.DataCopy(), CPU: c, Lost: 0, Type: payload.EventSample}
	m.send(pl)
}

func (m *Monitor) lostEvent(el *bpf.PerfEventLost, c int) {
	pl := payload.Payload{Data: []byte{}, CPU: c, Lost: el.Lost, Type: payload.RecordLost}
	m.send(pl)
}
<|endoftext|>