<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy

import (
	"time"

	"github.com/googlecloudplatform/gcsfuse/lease"
	"github.com/googlecloudplatform/gcsfuse/timeutil"
	"golang.org/x/net/context"
)

// A mutable view on some content. Created with an initial read-only view,
// which then can be modified by the user and read back. Keeps track of which
// portion of the content has been dirtied.
//
// External synchronization is required.
type MutableContent struct {
}

type StatResult struct {
	// The current size in bytes of the content.
	Size int64

	// It is guaranteed that all bytes in the range [0, DirtyThreshold) are
	// unmodified from the original content with which the mutable content object
	// was created.
	DirtyThreshold int64

	// The time at which the content was last updated, or nil if we've never
	// changed it.
	Mtime *time.Time
}

// Create a mutable content object whose initial contents are given by the
// supplied read proxy.
func NewMutableContent(
	initialContents lease.ReadProxy,
	clock timeutil.Clock) (mc *MutableContent)

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// Panic if any internal invariants are violated. Careful users can call this
// at appropriate times to help debug weirdness. Consider using
// syncutil.InvariantMutex to automate the process.
func (mc *MutableContent) CheckInvariants()

// Destroy any state used by the object, putting it into an indeterminate
// state. The object must not be used again.
func (mc *MutableContent) Destroy()

// Read part of the content, with semantics equivalent to io.ReaderAt aside
// from context support.
func (mc *MutableContent) ReadAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error)

// Return information about the current state of the content.
func (mc *MutableContent) Stat(ctx context.Context) (sr StatResult, err error)

// Write into the content, with semantics equivalent to io.WriterAt aside from
// context support.
func (mc *MutableContent) WriteAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error)

// Truncate our the content to the given number of bytes, extending if n is
// greater than the current size.
func (mc *MutableContent) Truncate(
	ctx context.Context,
	n int64) (err error)
<commit_msg>Defined the contents of MutableContent.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy

import (
	"time"

	"github.com/googlecloudplatform/gcsfuse/lease"
	"github.com/googlecloudplatform/gcsfuse/timeutil"
	"golang.org/x/net/context"
)

// A mutable view on some content. Created with an initial read-only view,
// which then can be modified by the user and read back. Keeps track of which
// portion of the content has been dirtied.
//
// External synchronization is required.
type MutableContent struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	clock timeutil.Clock

	/////////////////////////
	// Mutable state
	/////////////////////////

	destroyed bool

	// The initial contents with which this object was created, or nil if it has
	// been dirtied.
	//
	// INVARIANT: When non-nil, initialContents.CheckInvariants() does not panic.
	initialContents lease.ReadProxy

	// When dirty, a read/write lease containing our current contents. When
	// clean, nil.
	//
	// INVARIANT: (initialContents == nil) != (readWriteLease == nil)
	readWriteLease lease.ReadWriteLease

	// The time at which a method that modifies our contents was last called, or
	// nil if never.
	//
	// INVARIANT: If dirty(), then mtime != nil
	mtime *time.Time
}

type StatResult struct {
	// The current size in bytes of the content.
	Size int64

	// It is guaranteed that all bytes in the range [0, DirtyThreshold) are
	// unmodified from the original content with which the mutable content object
	// was created.
	DirtyThreshold int64

	// The time at which the content was last updated, or nil if we've never
	// changed it.
	Mtime *time.Time
}

// Create a mutable content object whose initial contents are given by the
// supplied read proxy.
func NewMutableContent(
	initialContents lease.ReadProxy,
	clock timeutil.Clock) (mc *MutableContent)

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// Panic if any internal invariants are violated. Careful users can call this
// at appropriate times to help debug weirdness. Consider using
// syncutil.InvariantMutex to automate the process.
func (mc *MutableContent) CheckInvariants()

// Destroy any state used by the object, putting it into an indeterminate
// state. The object must not be used again.
func (mc *MutableContent) Destroy()

// Read part of the content, with semantics equivalent to io.ReaderAt aside
// from context support.
func (mc *MutableContent) ReadAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error)

// Return information about the current state of the content.
func (mc *MutableContent) Stat(ctx context.Context) (sr StatResult, err error)

// Write into the content, with semantics equivalent to io.WriterAt aside from
// context support.
func (mc *MutableContent) WriteAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error)

// Truncate our the content to the given number of bytes, extending if n is
// greater than the current size.
func (mc *MutableContent) Truncate(
	ctx context.Context,
	n int64) (err error)

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func (mc *MutableContent) dirty() bool {
	return mc.readWriteLease != nil
}
<|endoftext|>


<commit_before>package goose

import (
	"io/ioutil"
	"log"
	"net/http"
	"net/http/cookiejar"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
)

// Crawler can fetch the target HTML page
type Crawler struct {
	config  Configuration
	url     string
	RawHTML string
	Charset string
}

// NewCrawler returns a crawler object initialised with the URL and the [optional] raw HTML body
func NewCrawler(config Configuration, url string, RawHTML string) Crawler {
	return Crawler{
		config:  config,
		url:     url,
		RawHTML: RawHTML,
		Charset: "",
	}
}

func getCharsetFromContentType(cs string) string {
	cs = strings.ToLower(strings.Replace(cs, " ", "", -1))
	cs = strings.TrimPrefix(cs, "text/html;charset=")
	cs = strings.TrimPrefix(cs, "text/xhtml;charset=")
	cs = strings.TrimPrefix(cs, "application/xhtml+xml;charset=")
	return NormaliseCharset(cs)
}

// SetCharset can be used to force a charset (e.g. when read from the HTTP headers)
// rather than relying on the detection from the HTML meta tags
func (c *Crawler) SetCharset(cs string) {
	c.Charset = getCharsetFromContentType(cs)
}

// GetContentType returns the Content-Type string extracted from the meta tags
func (c Crawler) GetContentType(document *goquery.Document) string {
	var attr string
	// <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
	document.Find("meta[http-equiv#=(?i)^Content\\-type$]").Each(func(i int, s *goquery.Selection) {
		attr, _ = s.Attr("content")
	})
	return attr
}

// GetCharset returns a normalised charset string extracted from the meta tags
func (c Crawler) GetCharset(document *goquery.Document) string {
	// manually-provided charset (from HTTP headers?) takes priority
	if "" != c.Charset {
		return c.Charset
	}

	// <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
	ct := c.GetContentType(document)
	if "" != ct && strings.Contains(strings.ToLower(ct), "charset") {
		return getCharsetFromContentType(ct)
	}

	// <meta charset="utf-8">
	selection := document.Find("meta").EachWithBreak(func(i int, s *goquery.Selection) bool {
		_, exists := s.Attr("charset")
		return !exists
	})

	if selection != nil {
		cs, _ := selection.Attr("charset")
		return NormaliseCharset(cs)
	}

	return ""
}

// Preprocess fetches the HTML page if needed, converts it to UTF-8 and applies
// some text normalisation to guarantee better results when extracting the content
func (c *Crawler) Preprocess() (*goquery.Document, error) {
	if c.RawHTML == "" {
		c.RawHTML = c.fetchHTML(c.url, c.config.timeout)
	}
	if c.RawHTML == "" {
		return nil, nil
	}

	c.RawHTML = c.addSpacesBetweenTags(c.RawHTML)

	reader := strings.NewReader(c.RawHTML)
	document, err := goquery.NewDocumentFromReader(reader)

	if err != nil {
		return nil, err
	}

	cs := c.GetCharset(document)
	//log.Println("-------------------------------------------CHARSET:", cs)
	if "" != cs && "UTF-8" != cs {
		// the net/html parser and goquery require UTF-8 data
		c.RawHTML = UTF8encode(c.RawHTML, cs)
		reader = strings.NewReader(c.RawHTML)
		document, err = goquery.NewDocumentFromReader(reader)

		if nil != err {
			return nil, err
		}
	}

	return document, nil
}

// Crawl fetches the HTML body and returns an Article
func (c Crawler) Crawl() (*Article, error) {
	article := new(Article)

	document, err := c.Preprocess()
	if nil != err {
		return nil, err
	}
	if nil == document {
		return article, nil
	}

	extractor := NewExtractor(c.config)

	startTime := time.Now().UnixNano()

	article.RawHTML, err = document.Html()
	if nil != err {
		return nil, err
	}
	article.FinalURL = c.url
	article.Doc = document

	article.Title = extractor.GetTitle(document)
	article.MetaLang = extractor.GetMetaLanguage(document)
	article.MetaFavicon = extractor.GetFavicon(document)

	article.MetaDescription = extractor.GetMetaContentWithSelector(document, "meta[name#=(?i)^description$]")
	article.MetaKeywords = extractor.GetMetaContentWithSelector(document, "meta[name#=(?i)^keywords$]")
	article.CanonicalLink = extractor.GetCanonicalLink(document)
	if "" == article.CanonicalLink {
		article.CanonicalLink = article.FinalURL
	}
	article.Domain = extractor.GetDomain(article.CanonicalLink)
	article.Tags = extractor.GetTags(document)

	cleaner := NewCleaner(c.config)
	article.Doc = cleaner.Clean(article.Doc)

	article.TopImage = OpenGraphResolver(document)
	if article.TopImage == "" {
		article.TopImage = WebPageResolver(article)
	}

	article.TopNode = extractor.CalculateBestNode(document)
	if article.TopNode != nil {
		article.TopNode = extractor.PostCleanup(article.TopNode)

		article.CleanedText, article.Links = extractor.GetCleanTextAndLinks(article.TopNode, article.MetaLang)

		videoExtractor := NewVideoExtractor()
		article.Movies = videoExtractor.GetVideos(document)
	}

	article.Delta = time.Now().UnixNano() - startTime

	return article, nil
}

// In many cases, like at the end of each
 element or between  tags,
// we need to add spaces, otherwise the text on either side will get joined together into one word.
// This method also adds newlines after each
 tag to preserve paragraphs.
func (c Crawler) addSpacesBetweenTags(text string) string {
	text = strings.Replace(text, "><", "> <", -1)
	text = strings.Replace(text, "
", "
\n", -1)
	text = strings.Replace(text, "<img ", "\n<img ", -1)
	text = strings.Replace(text, "
", "
\n", -1)
	return strings.Replace(text, "
", "
\n", -1)
}

func (c *Crawler) fetchHTML(u string, timeout time.Duration) string {
	cookieJar, _ := cookiejar.New(nil)
	client := &http.Client{
		Jar:     cookieJar,
		Timeout: timeout,
	}
	req, err := http.NewRequest("GET", u, nil)
	if err != nil {
		log.Println(err.Error())
		return ""
	}

	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.91 Safari/534.30")
	resp, err := client.Do(req)
	if err != nil {
		log.Println(err.Error())
		return ""
	}
	contents, err := ioutil.ReadAll(resp.Body)
	if err == nil {
		c.RawHTML = string(contents)
	} else {
		log.Println(err.Error())
	}
	err = resp.Body.Close()
	if err != nil {
		log.Println(err.Error())
	}

	return c.RawHTML
}
<commit_msg>Return crawl errors for handling by pkg consumer instead of logging and swallowing.<commit_after>package goose

import (
	"errors"
	"io/ioutil"
	"net/http"
	"net/http/cookiejar"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
)

// Crawler can fetch the target HTML page
type Crawler struct {
	config  Configuration
	url     string
	RawHTML string
	Charset string
}

// NewCrawler returns a crawler object initialised with the URL and the [optional] raw HTML body
func NewCrawler(config Configuration, url string, RawHTML string) Crawler {
	return Crawler{
		config:  config,
		url:     url,
		RawHTML: RawHTML,
		Charset: "",
	}
}

func getCharsetFromContentType(cs string) string {
	cs = strings.ToLower(strings.Replace(cs, " ", "", -1))
	cs = strings.TrimPrefix(cs, "text/html;charset=")
	cs = strings.TrimPrefix(cs, "text/xhtml;charset=")
	cs = strings.TrimPrefix(cs, "application/xhtml+xml;charset=")
	return NormaliseCharset(cs)
}

// SetCharset can be used to force a charset (e.g. when read from the HTTP headers)
// rather than relying on the detection from the HTML meta tags
func (c *Crawler) SetCharset(cs string) {
	c.Charset = getCharsetFromContentType(cs)
}

// GetContentType returns the Content-Type string extracted from the meta tags
func (c Crawler) GetContentType(document *goquery.Document) string {
	var attr string
	// <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
	document.Find("meta[http-equiv#=(?i)^Content\\-type$]").Each(func(i int, s *goquery.Selection) {
		attr, _ = s.Attr("content")
	})
	return attr
}

// GetCharset returns a normalised charset string extracted from the meta tags
func (c Crawler) GetCharset(document *goquery.Document) string {
	// manually-provided charset (from HTTP headers?) takes priority
	if "" != c.Charset {
		return c.Charset
	}

	// <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
	ct := c.GetContentType(document)
	if "" != ct && strings.Contains(strings.ToLower(ct), "charset") {
		return getCharsetFromContentType(ct)
	}

	// <meta charset="utf-8">
	selection := document.Find("meta").EachWithBreak(func(i int, s *goquery.Selection) bool {
		_, exists := s.Attr("charset")
		return !exists
	})

	if selection != nil {
		cs, _ := selection.Attr("charset")
		return NormaliseCharset(cs)
	}

	return ""
}

// Preprocess fetches the HTML page if needed, converts it to UTF-8 and applies
// some text normalisation to guarantee better results when extracting the content
func (c *Crawler) Preprocess() (*goquery.Document, error) {
	var err error

	if c.RawHTML == "" {
		if c.RawHTML, err = c.fetchHTML(c.url, c.config.timeout); err != nil {
			return nil, err
		}
	}
	if c.RawHTML == "" {
		return nil, errors.New("cannot process empty HTML content")
	}

	c.RawHTML = c.addSpacesBetweenTags(c.RawHTML)

	reader := strings.NewReader(c.RawHTML)
	document, err := goquery.NewDocumentFromReader(reader)
	if err != nil {
		return nil, err
	}

	cs := c.GetCharset(document)
	//log.Println("-------------------------------------------CHARSET:", cs)
	if "" != cs && "UTF-8" != cs {
		// the net/html parser and goquery require UTF-8 data
		c.RawHTML = UTF8encode(c.RawHTML, cs)
		reader = strings.NewReader(c.RawHTML)
		if document, err = goquery.NewDocumentFromReader(reader); err != nil {
			return nil, err
		}
	}

	return document, nil
}

// Crawl fetches the HTML body and returns an Article
func (c Crawler) Crawl() (*Article, error) {
	article := new(Article)

	document, err := c.Preprocess()
	if nil != err {
		return nil, err
	}
	if nil == document {
		return article, nil
	}

	extractor := NewExtractor(c.config)

	startTime := time.Now().UnixNano()

	article.RawHTML, err = document.Html()
	if nil != err {
		return nil, err
	}
	article.FinalURL = c.url
	article.Doc = document

	article.Title = extractor.GetTitle(document)
	article.MetaLang = extractor.GetMetaLanguage(document)
	article.MetaFavicon = extractor.GetFavicon(document)

	article.MetaDescription = extractor.GetMetaContentWithSelector(document, "meta[name#=(?i)^description$]")
	article.MetaKeywords = extractor.GetMetaContentWithSelector(document, "meta[name#=(?i)^keywords$]")
	article.CanonicalLink = extractor.GetCanonicalLink(document)
	if "" == article.CanonicalLink {
		article.CanonicalLink = article.FinalURL
	}
	article.Domain = extractor.GetDomain(article.CanonicalLink)
	article.Tags = extractor.GetTags(document)

	cleaner := NewCleaner(c.config)
	article.Doc = cleaner.Clean(article.Doc)

	article.TopImage = OpenGraphResolver(document)
	if article.TopImage == "" {
		article.TopImage = WebPageResolver(article)
	}

	article.TopNode = extractor.CalculateBestNode(document)
	if article.TopNode != nil {
		article.TopNode = extractor.PostCleanup(article.TopNode)

		article.CleanedText, article.Links = extractor.GetCleanTextAndLinks(article.TopNode, article.MetaLang)

		videoExtractor := NewVideoExtractor()
		article.Movies = videoExtractor.GetVideos(document)
	}

	article.Delta = time.Now().UnixNano() - startTime

	return article, nil
}

// In many cases, like at the end of each
 element or between  tags,
// we need to add spaces, otherwise the text on either side will get joined together into one word.
// This method also adds newlines after each
 tag to preserve paragraphs.
func (c Crawler) addSpacesBetweenTags(text string) string {
	text = strings.Replace(text, "><", "> <", -1)
	text = strings.Replace(text, "
", "
\n", -1)
	text = strings.Replace(text, "<img ", "\n<img ", -1)
	text = strings.Replace(text, "
", "
\n", -1)
	return strings.Replace(text, "
", "
\n", -1)
}

func (c *Crawler) fetchHTML(u string, timeout time.Duration) (string, error) {
	cookieJar, err := cookiejar.New(nil)
	if err != nil {
		return "", err
	}

	client := &http.Client{
		Jar:     cookieJar,
		Timeout: timeout,
	}

	req, err := http.NewRequest("GET", u, nil)
	if err != nil {
		return "", err
	}

	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.91 Safari/534.30")
	resp, err := client.Do(req)
	if err != nil {
		return "", err
	}
	contents, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	c.RawHTML = string(contents)

	if err = resp.Body.Close(); err != nil {
		return "", err
	}

	return c.RawHTML, nil
}
<|endoftext|>


<commit_before><commit_msg>Throw a fatal error if can not connect to the database on startup<commit_after><|endoftext|>


<commit_before>package add_tests

import (
	"github.com/BytemarkHosting/bytemark-client/lib"
	"github.com/BytemarkHosting/bytemark-client/lib/brain"
)

var DefVM = lib.VirtualMachineName{Group: "default", Account: "default-account"}
var DefGroup = lib.GroupName{Group: "default", Account: "default-account"}
<commit_msg>remove s from test<commit_after>package add_test

import (
	"github.com/BytemarkHosting/bytemark-client/lib"
	"github.com/BytemarkHosting/bytemark-client/lib/brain"
)

var DefVM = lib.VirtualMachineName{Group: "default", Account: "default-account"}
var DefGroup = lib.GroupName{Group: "default", Account: "default-account"}
<|endoftext|>


<commit_before>package kateway

import (
	"sync"
	"time"

	"github.com/funkygao/gafka/cmd/kguard/monitor"
	"github.com/funkygao/gafka/zk"
	log "github.com/funkygao/log4go"
	"github.com/rcrowley/go-metrics"
)

func init() {
	monitor.RegisterWatcher("kateway.sublag", func() monitor.Watcher {
		return &WatchSubLag{
			Tick: time.Minute,
		}
	})
}

// SubLag monitors aliveness of kateway cluster.
type WatchSubLag struct {
	Zkzone *zk.ZkZone
	Stop   <-chan struct{}
	Tick   time.Duration
	Wg     *sync.WaitGroup

	zkcluster *zk.ZkCluster

	suspects map[string]struct{}
}

func (this *WatchSubLag) Init(ctx monitor.Context) {
	this.Zkzone = ctx.ZkZone()
	this.Stop = ctx.StopChan()
	this.Wg = ctx.WaitGroup()
	this.suspects = make(map[string]struct{})
}

func (this *WatchSubLag) Run() {
	defer this.Wg.Done()

	this.zkcluster = this.Zkzone.NewCluster("bigtopic") // TODO

	ticker := time.NewTicker(this.Tick)
	defer ticker.Stop()

	subLagGroups := metrics.NewRegisteredGauge("sub.lags", nil)
	for {
		select {
		case <-this.Stop:
			log.Info("kateway.sublag stopped")
			return

		case <-ticker.C:
			subLagGroups.Update(int64(this.report()))

		}
	}
}

func (this *WatchSubLag) isSuspect(group string, topic string) bool {
	if _, present := this.suspects[group+"|"+topic]; present {
		return true
	}

	return false
}

func (this *WatchSubLag) suspect(group, topic string) {
	this.suspects[group+"|"+topic] = struct{}{}
}

func (this *WatchSubLag) unsuspect(group string, topic string) {
	delete(this.suspects, group+"|"+topic)
}

func (this *WatchSubLag) report() (lags int) {
	for group, consumers := range this.zkcluster.ConsumersByGroup("") {
		for _, c := range consumers {
			if !c.Online {
				continue
			}

			if c.ConsumerZnode == nil {
				log.Warn("group[%s] topic[%s/%s] unrecognized consumer", group, c.Topic, c.PartitionId)

				continue
			}

			if time.Since(c.ConsumerZnode.Uptime()) < time.Minute*2 {
				log.Info("group[%s] just started, topic[%s/%s]", group, c.Topic, c.PartitionId)

				this.unsuspect(group, c.Topic)
				continue
			}

			// offset commit every 1m, sublag runs every 1m, so the gap might be 2m
			// TODO lag too much, even if it's still alive, emit alarm
			elapsed := time.Since(c.Mtime.Time())
			if c.Lag == 0 || elapsed < time.Minute*3 {
				this.unsuspect(group, c.Topic)
				continue
			}

			// it might be lagging, but need confirm with last round
			if !this.isSuspect(group, c.Topic) {
				// suspect it, next round if it is still lagging, put on trial
				this.suspect(group, c.Topic)
				continue
			}

			// bingo! it IS lagging
			log.Warn("group[%s] topic[%s/%s] %d - %d = %d, offset commit elapsed: %s",
				group, c.Topic, c.PartitionId, c.ProducerOffset, c.ConsumerOffset, c.Lag, elapsed.String())

			lags++
		}
	}

	return
}
<commit_msg>log the suspect<commit_after>package kateway

import (
	"sync"
	"time"

	"github.com/funkygao/gafka/cmd/kguard/monitor"
	"github.com/funkygao/gafka/zk"
	log "github.com/funkygao/log4go"
	"github.com/rcrowley/go-metrics"
)

func init() {
	monitor.RegisterWatcher("kateway.sublag", func() monitor.Watcher {
		return &WatchSubLag{
			Tick: time.Minute,
		}
	})
}

// SubLag monitors aliveness of kateway cluster.
type WatchSubLag struct {
	Zkzone *zk.ZkZone
	Stop   <-chan struct{}
	Tick   time.Duration
	Wg     *sync.WaitGroup

	zkcluster *zk.ZkCluster

	suspects map[string]struct{}
}

func (this *WatchSubLag) Init(ctx monitor.Context) {
	this.Zkzone = ctx.ZkZone()
	this.Stop = ctx.StopChan()
	this.Wg = ctx.WaitGroup()
	this.suspects = make(map[string]struct{})
}

func (this *WatchSubLag) Run() {
	defer this.Wg.Done()

	this.zkcluster = this.Zkzone.NewCluster("bigtopic") // TODO

	ticker := time.NewTicker(this.Tick)
	defer ticker.Stop()

	subLagGroups := metrics.NewRegisteredGauge("sub.lags", nil)
	for {
		select {
		case <-this.Stop:
			log.Info("kateway.sublag stopped")
			return

		case <-ticker.C:
			subLagGroups.Update(int64(this.report()))

		}
	}
}

func (this *WatchSubLag) isSuspect(group string, topic string) bool {
	if _, present := this.suspects[group+"|"+topic]; present {
		return true
	}

	return false
}

func (this *WatchSubLag) suspect(group, topic string) {
	this.suspects[group+"|"+topic] = struct{}{}
}

func (this *WatchSubLag) unsuspect(group string, topic string) {
	delete(this.suspects, group+"|"+topic)
}

func (this *WatchSubLag) report() (lags int) {
	for group, consumers := range this.zkcluster.ConsumersByGroup("") {
		for _, c := range consumers {
			if !c.Online {
				continue
			}

			if c.ConsumerZnode == nil {
				log.Warn("group[%s] topic[%s/%s] unrecognized consumer", group, c.Topic, c.PartitionId)

				continue
			}

			if time.Since(c.ConsumerZnode.Uptime()) < time.Minute*2 {
				log.Info("group[%s] just started, topic[%s/%s]", group, c.Topic, c.PartitionId)

				this.unsuspect(group, c.Topic)
				continue
			}

			// offset commit every 1m, sublag runs every 1m, so the gap might be 2m
			// TODO lag too much, even if it's still alive, emit alarm
			elapsed := time.Since(c.Mtime.Time())
			if c.Lag == 0 || elapsed < time.Minute*3 {
				this.unsuspect(group, c.Topic)
				continue
			}

			// it might be lagging, but need confirm with last round
			if !this.isSuspect(group, c.Topic) {
				// suspect it, next round if it is still lagging, put on trial
				log.Warn("group[%s] suspected topic[%s/%s] %d - %d = %d, offset commit elapsed: %s",
					group, c.Topic, c.PartitionId, c.ProducerOffset, c.ConsumerOffset, c.Lag, elapsed.String())

				this.suspect(group, c.Topic)
				continue
			}

			// bingo! it IS lagging
			log.Warn("group[%s] confirmed topic[%s/%s] %d - %d = %d, offset commit elapsed: %s",
				group, c.Topic, c.PartitionId, c.ProducerOffset, c.ConsumerOffset, c.Lag, elapsed.String())

			lags++
		}
	}

	return
}
<|endoftext|>


<commit_before>package actor

import (
	"fmt"
	"log"
	"reflect"
	"runtime"
	"strings"
	"time"

	"github.com/emirpasic/gods/stacks/linkedliststack"
)

type messageSender struct {
	Message interface{}
	Sender  *PID
}

type localContext struct {
	message        interface{}
	parent         *PID
	self           *PID
	actor          Actor
	supervisor     SupervisorStrategy
	producer       Producer
	middleware     ReceiveFunc
	behavior       behaviorStack
	receive        ReceiveFunc
	children       PIDSet
	watchers       PIDSet
	watching       PIDSet
	stash          *linkedliststack.Stack
	stopping       bool
	restarting     bool
	receiveTimeout time.Duration
	t              *time.Timer
	restartStats   *ChildRestartStats
}

func newLocalContext(producer Producer, supervisor SupervisorStrategy, middleware ReceiveFunc, parent *PID) *localContext {
	cell := &localContext{
		parent:     parent,
		producer:   producer,
		supervisor: supervisor,
		middleware: middleware,
	}
	cell.incarnateActor()
	return cell
}

func (ctx *localContext) Actor() Actor {
	return ctx.actor
}

func (ctx *localContext) Message() interface{} {
	userMessage, ok := ctx.message.(*messageSender)
	if ok {
		return userMessage.Message
	}
	return ctx.message
}

func (ctx *localContext) Sender() *PID {
	userMessage, ok := ctx.message.(*messageSender)
	if ok {
		return userMessage.Sender
	}
	return nil
}

func (ctx *localContext) Stash() {
	if ctx.stash == nil {
		ctx.stash = linkedliststack.New()
	}

	ctx.stash.Push(ctx.message)
}

func (ctx *localContext) cancelTimer() {
	if ctx.t != nil {
		ctx.t.Stop()
		ctx.t = nil
	}
}

func (ctx *localContext) receiveTimeoutHandler() {
	ctx.self.Request(receiveTimeoutMessage, nil)
}

func (ctx *localContext) SetReceiveTimeout(d time.Duration) {
	if d == ctx.receiveTimeout {
		return
	}
	if ctx.t != nil {
		ctx.t.Stop()
	}

	if d < time.Millisecond {
		// anything less than than 1 millisecond is set to zero
		d = 0
	}

	ctx.receiveTimeout = d
	if d > 0 {
		if ctx.t == nil {
			ctx.t = time.AfterFunc(d, ctx.receiveTimeoutHandler)
		} else {
			ctx.t.Reset(d)
		}
	}
}

func (ctx *localContext) ReceiveTimeout() time.Duration {
	return ctx.receiveTimeout
}

func (ctx *localContext) Children() []*PID {
	r := make([]*PID, ctx.children.Len())
	ctx.children.ForEach(func(i int, p PID) {
		r[i] = &p
	})
	return r
}

func (ctx *localContext) Self() *PID {
	return ctx.self
}

func (ctx *localContext) Parent() *PID {
	return ctx.parent
}

func (ctx *localContext) Receive(message interface{}) {
	ctx.processMessage(message)
}

// localContextReceiver is used when middleware chain is required
func localContextReceiver(ctx Context) {
	a := ctx.(*localContext)
	if _, ok := a.message.(*PoisonPill); ok {
		a.self.Stop()
	} else {
		a.receive(ctx)
	}
}

func (ctx *localContext) processMessage(m interface{}) {
	ctx.message = m

	if ctx.middleware != nil {
		ctx.middleware(ctx)
	} else {
		if _, ok := m.(*PoisonPill); ok {
			ctx.self.Stop()
		} else {
			ctx.receive(ctx)
		}
	}
}

func (ctx *localContext) incarnateActor() {
	actor := ctx.producer()
	ctx.restarting = false
	ctx.stopping = false
	ctx.actor = actor
	ctx.receive = actor.Receive
}

func (ctx *localContext) InvokeSystemMessage(message SystemMessage) {
	switch msg := message.(interface{}).(type) {
	case *Started:
		ctx.InvokeUserMessage(msg) // forward
	case *Watch:
		ctx.watchers.Add(msg.Watcher)
	case *Unwatch:
		ctx.watchers.Remove(msg.Watcher)
	case *SuspendMailbox, *ResumeMailbox:
	//pass
	case *Stop:
		ctx.handleStop(msg)
	case *Terminated:
		ctx.handleTerminated(msg)
	case *Failure:
		ctx.handleFailure(msg)
	case *Restart:
		ctx.handleRestart(msg)
	default:
		log.Printf("Unknown system message %T", msg)
	}
}

func (ctx *localContext) handleRestart(msg *Restart) {
	ctx.stopping = false
	ctx.restarting = true
	ctx.InvokeUserMessage(restartingMessage)
	ctx.children.ForEach(func(_ int, pid PID) {
		pid.Stop()
	})
	ctx.tryRestartOrTerminate()
}

//I am stopping
func (ctx *localContext) handleStop(msg *Stop) {
	ctx.stopping = true
	ctx.restarting = false

	ctx.InvokeUserMessage(stoppingMessage)
	ctx.children.ForEach(func(_ int, pid PID) {
		pid.Stop()
	})
	ctx.tryRestartOrTerminate()
}

//child stopped, check if we can stop or restart (if needed)
func (ctx *localContext) handleTerminated(msg *Terminated) {
	ctx.children.Remove(msg.Who)
	ctx.watching.Remove(msg.Who)

	ctx.InvokeUserMessage(msg)
	ctx.tryRestartOrTerminate()
}

//offload the supervision completely to the supervisor strategy
func (ctx *localContext) handleFailure(msg *Failure) {
	if strategy, ok := ctx.actor.(SupervisorStrategy); ok {
		strategy.HandleFailure(ctx, msg.Who, msg.ChildStats, msg.Reason)
		return
	}
	ctx.supervisor.HandleFailure(ctx, msg.Who, msg.ChildStats, msg.Reason)
}

func (ctx *localContext) EscalateFailure(who *PID, reason interface{}) {
	if ctx.Parent() == nil {
		log.Printf("[ACTOR] '%v' Cannot escalate failure from root actor; stopping instead", ctx.debugString())
		ctx.Self().sendSystemMessage(stopMessage)
		return
	}
	//suspend self
	ctx.Self().sendSystemMessage(suspendMailboxMessage)
	//send failure to parent
	ctx.Parent().sendSystemMessage(&Failure{Reason: reason, Who: who})
}

func (ctx *localContext) tryRestartOrTerminate() {
	if ctx.t != nil {
		ctx.t.Stop()
		ctx.t = nil
		ctx.receiveTimeout = 0
	}

	if !ctx.children.Empty() {
		return
	}

	if ctx.restarting {
		ctx.restart()
		return
	}

	if ctx.stopping {
		ctx.stopped()
	}
}

func (ctx *localContext) restart() {
	ctx.incarnateActor()
	//create a new childRestartStats with the current failure settings
	ctx.restartStats = &ChildRestartStats{
		FailureCount:    ctx.restartStats.FailureCount + 1,
		LastFailureTime: time.Now(),
	}
	ctx.InvokeUserMessage(startedMessage)
	if ctx.stash != nil {
		for !ctx.stash.Empty() {
			msg, _ := ctx.stash.Pop()
			ctx.InvokeUserMessage(msg)
		}
	}
	ctx.self.sendSystemMessage(resumeMailboxMessage)
}

func (ctx *localContext) stopped() {
	ProcessRegistry.Remove(ctx.self)
	ctx.InvokeUserMessage(stoppedMessage)
	otherStopped := &Terminated{Who: ctx.self}
	ctx.watchers.ForEach(func(i int, pid PID) {
		pid.sendSystemMessage(otherStopped)
	})
}

func identifyPanic() string {
	var name, file string
	var line int
	var pc [16]uintptr

	n := runtime.Callers(3, pc[:])
	for _, pc := range pc[:n] {
		log.Printf("%d", pc)
		fn := runtime.FuncForPC(pc)
		if fn == nil {
			continue
		}
		file, line = fn.FileLine(pc)
		fmt.Printf(file, line, pc)
		name = fn.Name()
		if !strings.HasPrefix(name, "runtime.") {
			break
		}
	}

	switch {
	case name != "":
		return fmt.Sprintf("%v:%v", name, line)
	case file != "":
		return fmt.Sprintf("%v:%v", file, line)
	}

	return fmt.Sprintf("pc:%x", pc)
}

func (ctx *localContext) InvokeUserMessage(md interface{}) {
	defer func() {
		if r := recover(); r != nil {
			log.Printf("[ACTOR] '%v' Recovering from: %v. Detailed stack: %v", ctx.debugString(), r, identifyPanic())

			//lazy initialize the child restart stats if this is the first time
			//further mutations are handled within "restart"
			if ctx.restartStats == nil {
				ctx.restartStats = &ChildRestartStats{
					FailureCount: 1,
				}
			}
			failure := &Failure{Reason: r, Who: ctx.self, ChildStats: ctx.restartStats}
			if ctx.parent == nil {
				handleRootFailure(failure)
			} else {
				//TODO: Akka recursively suspends all children also on failure
				//Not sure if I think this is the right way to go, why do children need to wait for their parents failed state to recover?
				ctx.self.sendSystemMessage(suspendMailboxMessage)
				ctx.parent.sendSystemMessage(failure)
			}
		}
	}()

	if md == nil {
		log.Printf("[ACTOR] '%v' got nil message", ctx.Self().String())
		return
	}

	influenceTimeout := true
	if ctx.receiveTimeout > 0 {
		_, influenceTimeout = md.(NotInfluenceReceiveTimeout)
		influenceTimeout = !influenceTimeout
		if influenceTimeout {
			ctx.t.Stop()
		}
	}

	ctx.processMessage(md)

	if ctx.receiveTimeout > 0 && influenceTimeout {
		ctx.t.Reset(ctx.receiveTimeout)
	}
}

func (ctx *localContext) Become(behavior ReceiveFunc) {
	ctx.behavior.Clear()
	ctx.receive = behavior
}

func (ctx *localContext) BecomeStacked(behavior ReceiveFunc) {
	ctx.behavior.Push(ctx.receive)
	ctx.receive = behavior
}

func (ctx *localContext) UnbecomeStacked() {
	if ctx.behavior.Len() == 0 {
		panic("Cannot unbecome actor base behavior")
	}
	ctx.receive, _ = ctx.behavior.Pop()
}

func (ctx *localContext) Watch(who *PID) {
	who.sendSystemMessage(&Watch{
		Watcher: ctx.self,
	})
	ctx.watching.Add(who)
}

func (ctx *localContext) Unwatch(who *PID) {
	who.sendSystemMessage(&Unwatch{
		Watcher: ctx.self,
	})
	ctx.watching.Remove(who)
}

func (ctx *localContext) Respond(response interface{}) {
	if ctx.Sender() == nil {
		log.Fatal("[ACTOR] No sender")
	}
	ctx.Sender().Tell(response)
}

func (ctx *localContext) Spawn(props Props) *PID {
	return ctx.SpawnNamed(props, ProcessRegistry.NextId())
}

func (ctx *localContext) SpawnNamed(props Props, name string) *PID {
	pid := props.spawn(ctx.self.Id+"/"+name, ctx.self)
	ctx.children.Add(pid)
	ctx.Watch(pid)
	return pid
}

func (ctx *localContext) debugString() string {
	return fmt.Sprintf("%v/%v:%v", ctx.self.Address, ctx.self.Id, reflect.TypeOf(ctx.actor))
}

func handleRootFailure(msg *Failure) {
	defaultSupervisionStrategy.HandleFailure(nil, msg.Who, msg.ChildStats, msg.Reason)
}
<commit_msg>identifypanic fix<commit_after>package actor

import (
	"fmt"
	"log"
	"reflect"
	"runtime"
	"strings"
	"time"

	"github.com/emirpasic/gods/stacks/linkedliststack"
)

type messageSender struct {
	Message interface{}
	Sender  *PID
}

type localContext struct {
	message        interface{}
	parent         *PID
	self           *PID
	actor          Actor
	supervisor     SupervisorStrategy
	producer       Producer
	middleware     ReceiveFunc
	behavior       behaviorStack
	receive        ReceiveFunc
	children       PIDSet
	watchers       PIDSet
	watching       PIDSet
	stash          *linkedliststack.Stack
	stopping       bool
	restarting     bool
	receiveTimeout time.Duration
	t              *time.Timer
	restartStats   *ChildRestartStats
}

func newLocalContext(producer Producer, supervisor SupervisorStrategy, middleware ReceiveFunc, parent *PID) *localContext {
	cell := &localContext{
		parent:     parent,
		producer:   producer,
		supervisor: supervisor,
		middleware: middleware,
	}
	cell.incarnateActor()
	return cell
}

func (ctx *localContext) Actor() Actor {
	return ctx.actor
}

func (ctx *localContext) Message() interface{} {
	userMessage, ok := ctx.message.(*messageSender)
	if ok {
		return userMessage.Message
	}
	return ctx.message
}

func (ctx *localContext) Sender() *PID {
	userMessage, ok := ctx.message.(*messageSender)
	if ok {
		return userMessage.Sender
	}
	return nil
}

func (ctx *localContext) Stash() {
	if ctx.stash == nil {
		ctx.stash = linkedliststack.New()
	}

	ctx.stash.Push(ctx.message)
}

func (ctx *localContext) cancelTimer() {
	if ctx.t != nil {
		ctx.t.Stop()
		ctx.t = nil
	}
}

func (ctx *localContext) receiveTimeoutHandler() {
	ctx.self.Request(receiveTimeoutMessage, nil)
}

func (ctx *localContext) SetReceiveTimeout(d time.Duration) {
	if d == ctx.receiveTimeout {
		return
	}
	if ctx.t != nil {
		ctx.t.Stop()
	}

	if d < time.Millisecond {
		// anything less than than 1 millisecond is set to zero
		d = 0
	}

	ctx.receiveTimeout = d
	if d > 0 {
		if ctx.t == nil {
			ctx.t = time.AfterFunc(d, ctx.receiveTimeoutHandler)
		} else {
			ctx.t.Reset(d)
		}
	}
}

func (ctx *localContext) ReceiveTimeout() time.Duration {
	return ctx.receiveTimeout
}

func (ctx *localContext) Children() []*PID {
	r := make([]*PID, ctx.children.Len())
	ctx.children.ForEach(func(i int, p PID) {
		r[i] = &p
	})
	return r
}

func (ctx *localContext) Self() *PID {
	return ctx.self
}

func (ctx *localContext) Parent() *PID {
	return ctx.parent
}

func (ctx *localContext) Receive(message interface{}) {
	ctx.processMessage(message)
}

// localContextReceiver is used when middleware chain is required
func localContextReceiver(ctx Context) {
	a := ctx.(*localContext)
	if _, ok := a.message.(*PoisonPill); ok {
		a.self.Stop()
	} else {
		a.receive(ctx)
	}
}

func (ctx *localContext) processMessage(m interface{}) {
	ctx.message = m

	if ctx.middleware != nil {
		ctx.middleware(ctx)
	} else {
		if _, ok := m.(*PoisonPill); ok {
			ctx.self.Stop()
		} else {
			ctx.receive(ctx)
		}
	}
}

func (ctx *localContext) incarnateActor() {
	actor := ctx.producer()
	ctx.restarting = false
	ctx.stopping = false
	ctx.actor = actor
	ctx.receive = actor.Receive
}

func (ctx *localContext) InvokeSystemMessage(message SystemMessage) {
	switch msg := message.(interface{}).(type) {
	case *Started:
		ctx.InvokeUserMessage(msg) // forward
	case *Watch:
		ctx.watchers.Add(msg.Watcher)
	case *Unwatch:
		ctx.watchers.Remove(msg.Watcher)
	case *SuspendMailbox, *ResumeMailbox:
	//pass
	case *Stop:
		ctx.handleStop(msg)
	case *Terminated:
		ctx.handleTerminated(msg)
	case *Failure:
		ctx.handleFailure(msg)
	case *Restart:
		ctx.handleRestart(msg)
	default:
		log.Printf("Unknown system message %T", msg)
	}
}

func (ctx *localContext) handleRestart(msg *Restart) {
	ctx.stopping = false
	ctx.restarting = true
	ctx.InvokeUserMessage(restartingMessage)
	ctx.children.ForEach(func(_ int, pid PID) {
		pid.Stop()
	})
	ctx.tryRestartOrTerminate()
}

//I am stopping
func (ctx *localContext) handleStop(msg *Stop) {
	ctx.stopping = true
	ctx.restarting = false

	ctx.InvokeUserMessage(stoppingMessage)
	ctx.children.ForEach(func(_ int, pid PID) {
		pid.Stop()
	})
	ctx.tryRestartOrTerminate()
}

//child stopped, check if we can stop or restart (if needed)
func (ctx *localContext) handleTerminated(msg *Terminated) {
	ctx.children.Remove(msg.Who)
	ctx.watching.Remove(msg.Who)

	ctx.InvokeUserMessage(msg)
	ctx.tryRestartOrTerminate()
}

//offload the supervision completely to the supervisor strategy
func (ctx *localContext) handleFailure(msg *Failure) {
	if strategy, ok := ctx.actor.(SupervisorStrategy); ok {
		strategy.HandleFailure(ctx, msg.Who, msg.ChildStats, msg.Reason)
		return
	}
	ctx.supervisor.HandleFailure(ctx, msg.Who, msg.ChildStats, msg.Reason)
}

func (ctx *localContext) EscalateFailure(who *PID, reason interface{}) {
	if ctx.Parent() == nil {
		log.Printf("[ACTOR] '%v' Cannot escalate failure from root actor; stopping instead", ctx.debugString())
		ctx.Self().sendSystemMessage(stopMessage)
		return
	}
	//suspend self
	ctx.Self().sendSystemMessage(suspendMailboxMessage)
	//send failure to parent
	ctx.Parent().sendSystemMessage(&Failure{Reason: reason, Who: who})
}

func (ctx *localContext) tryRestartOrTerminate() {
	if ctx.t != nil {
		ctx.t.Stop()
		ctx.t = nil
		ctx.receiveTimeout = 0
	}

	if !ctx.children.Empty() {
		return
	}

	if ctx.restarting {
		ctx.restart()
		return
	}

	if ctx.stopping {
		ctx.stopped()
	}
}

func (ctx *localContext) restart() {
	ctx.incarnateActor()
	//create a new childRestartStats with the current failure settings
	ctx.restartStats = &ChildRestartStats{
		FailureCount:    ctx.restartStats.FailureCount + 1,
		LastFailureTime: time.Now(),
	}
	ctx.InvokeUserMessage(startedMessage)
	if ctx.stash != nil {
		for !ctx.stash.Empty() {
			msg, _ := ctx.stash.Pop()
			ctx.InvokeUserMessage(msg)
		}
	}
	ctx.self.sendSystemMessage(resumeMailboxMessage)
}

func (ctx *localContext) stopped() {
	ProcessRegistry.Remove(ctx.self)
	ctx.InvokeUserMessage(stoppedMessage)
	otherStopped := &Terminated{Who: ctx.self}
	ctx.watchers.ForEach(func(i int, pid PID) {
		pid.sendSystemMessage(otherStopped)
	})
}

func identifyPanic() string {
	var name, file string
	var line int
	var pc [16]uintptr

	n := runtime.Callers(3, pc[:])
	for _, pc := range pc[:n] {
		log.Printf("%d", pc)
		fn := runtime.FuncForPC(pc)
		if fn == nil {
			continue
		}
		file, line = fn.FileLine(pc)
		name = fn.Name()
		if !strings.HasPrefix(name, "runtime.") {
			break
		}
	}

	switch {
	case name != "":
		return fmt.Sprintf("%v:%v", name, line)
	case file != "":
		return fmt.Sprintf("%v:%v", file, line)
	}

	return fmt.Sprintf("pc:%x", pc)
}

func (ctx *localContext) InvokeUserMessage(md interface{}) {
	defer func() {
		if r := recover(); r != nil {
			log.Printf("[ACTOR] '%v' Recovering from: %v. Detailed stack: %v", ctx.debugString(), r, identifyPanic())

			//lazy initialize the child restart stats if this is the first time
			//further mutations are handled within "restart"
			if ctx.restartStats == nil {
				ctx.restartStats = &ChildRestartStats{
					FailureCount: 1,
				}
			}
			failure := &Failure{Reason: r, Who: ctx.self, ChildStats: ctx.restartStats}
			if ctx.parent == nil {
				handleRootFailure(failure)
			} else {
				//TODO: Akka recursively suspends all children also on failure
				//Not sure if I think this is the right way to go, why do children need to wait for their parents failed state to recover?
				ctx.self.sendSystemMessage(suspendMailboxMessage)
				ctx.parent.sendSystemMessage(failure)
			}
		}
	}()

	if md == nil {
		log.Printf("[ACTOR] '%v' got nil message", ctx.Self().String())
		return
	}

	influenceTimeout := true
	if ctx.receiveTimeout > 0 {
		_, influenceTimeout = md.(NotInfluenceReceiveTimeout)
		influenceTimeout = !influenceTimeout
		if influenceTimeout {
			ctx.t.Stop()
		}
	}

	ctx.processMessage(md)

	if ctx.receiveTimeout > 0 && influenceTimeout {
		ctx.t.Reset(ctx.receiveTimeout)
	}
}

func (ctx *localContext) Become(behavior ReceiveFunc) {
	ctx.behavior.Clear()
	ctx.receive = behavior
}

func (ctx *localContext) BecomeStacked(behavior ReceiveFunc) {
	ctx.behavior.Push(ctx.receive)
	ctx.receive = behavior
}

func (ctx *localContext) UnbecomeStacked() {
	if ctx.behavior.Len() == 0 {
		panic("Cannot unbecome actor base behavior")
	}
	ctx.receive, _ = ctx.behavior.Pop()
}

func (ctx *localContext) Watch(who *PID) {
	who.sendSystemMessage(&Watch{
		Watcher: ctx.self,
	})
	ctx.watching.Add(who)
}

func (ctx *localContext) Unwatch(who *PID) {
	who.sendSystemMessage(&Unwatch{
		Watcher: ctx.self,
	})
	ctx.watching.Remove(who)
}

func (ctx *localContext) Respond(response interface{}) {
	if ctx.Sender() == nil {
		log.Fatal("[ACTOR] No sender")
	}
	ctx.Sender().Tell(response)
}

func (ctx *localContext) Spawn(props Props) *PID {
	return ctx.SpawnNamed(props, ProcessRegistry.NextId())
}

func (ctx *localContext) SpawnNamed(props Props, name string) *PID {
	pid := props.spawn(ctx.self.Id+"/"+name, ctx.self)
	ctx.children.Add(pid)
	ctx.Watch(pid)
	return pid
}

func (ctx *localContext) debugString() string {
	return fmt.Sprintf("%v/%v:%v", ctx.self.Address, ctx.self.Id, reflect.TypeOf(ctx.actor))
}

func handleRootFailure(msg *Failure) {
	defaultSupervisionStrategy.HandleFailure(nil, msg.Who, msg.ChildStats, msg.Reason)
}
<|endoftext|>


<commit_before>package render

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"net/http"
	"reflect"
)

// Respond handles streaming JSON and XML responses, automatically setting the
// Content-Type based on request headers. It will default to a JSON response.
func Respond(w http.ResponseWriter, r *http.Request, v interface{}) {
	// Present the object.
	if presenter, ok := r.Context().Value(presenterCtxKey).(Presenter); ok {
		r, v = presenter.Present(r, v)
	}

	switch reflect.TypeOf(v).Kind() {
	case reflect.Chan:
		switch getResponseContentType(r) {
		case ContentTypeEventStream:
			channelEventStream(w, r, v)
			return
		default:
			v = channelIntoSlice(w, r, v)
		}
	}

	// Format data based on Content-Type.
	switch getResponseContentType(r) {
	case ContentTypeJSON:
		JSON(w, r, v)
	case ContentTypeXML:
		XML(w, r, v)
	default:
		JSON(w, r, v)
	}
}

// PlainText writes a string to the response, setting the Content-Type as
// text/plain.
func PlainText(w http.ResponseWriter, r *http.Request, v string) {
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	if status, ok := r.Context().Value(statusCtxKey).(int); ok {
		w.WriteHeader(status)
	}
	w.Write([]byte(v))
}

// Data writes raw bytes to the response, setting the Content-Type as
// application/octet-stream.
func Data(w http.ResponseWriter, r *http.Request, v []byte) {
	w.Header().Set("Content-Type", "application/octet-stream")
	if status, ok := r.Context().Value(statusCtxKey).(int); ok {
		w.WriteHeader(status)
	}
	w.Write(v)
}

// HTML writes a string to the response, setting the Content-Type as text/html.
func HTML(w http.ResponseWriter, r *http.Request, v string) {
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	if status, ok := r.Context().Value(statusCtxKey).(int); ok {
		w.WriteHeader(status)
	}
	w.Write([]byte(v))
}

// JSON marshals 'v' to JSON, automatically escaping HTML and setting the
// Content-Type as application/json.
func JSON(w http.ResponseWriter, r *http.Request, v interface{}) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	if status, ok := r.Context().Value(statusCtxKey).(int); ok {
		w.WriteHeader(status)
	}

	enc := json.NewEncoder(w)
	enc.SetEscapeHTML(true)
	if err := enc.Encode(v); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
}

// XML marshals 'v' to JSON, setting the Content-Type as application/xml. It
// will automatically prepend a generic XML header (see encoding/xml.Header) if
// one is not found in the first 100 bytes of 'v'.
func XML(w http.ResponseWriter, r *http.Request, v interface{}) {
	b, err := xml.Marshal(v)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
	if status, ok := r.Context().Value(statusCtxKey).(int); ok {
		w.WriteHeader(status)
	}

	// Try to find <?xml header in first 100 bytes (just in case there're some XML comments).
	findHeaderUntil := len(b)
	if findHeaderUntil > 100 {
		findHeaderUntil = 100
	}
	if !bytes.Contains(b[:findHeaderUntil], []byte("<?xml")) {
		// No header found. Print it out first.
		w.Write([]byte(xml.Header))
	}

	w.Write(b)
}

// NoContent returns a HTTP 204 "No Content" response.
func NoContent(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(204)
}

func channelEventStream(w http.ResponseWriter, r *http.Request, v interface{}) {
	if reflect.TypeOf(v).Kind() != reflect.Chan {
		panic(fmt.Sprintf("render.EventStream() expects channel, not %v", reflect.TypeOf(v).Kind()))
	}

	w.Header().Set("Content-Type", "text/event-stream; charset=utf-8")
	w.Header().Set("Cache-Control", "no-cache")
	w.Header().Set("Connection", "keep-alive")
	w.WriteHeader(200)

	ctx := r.Context()
	for {
		switch chosen, recv, ok := reflect.Select([]reflect.SelectCase{
			{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(ctx.Done())},
			{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(v)},
		}); chosen {
		case 0: // equivalent to: case <-ctx.Done()
			w.Write([]byte("event: error\ndata: {\"error\":\"Server Timeout\"}\n\n"))
			return

		default: // equivalent to: case v, ok := <-stream
			if !ok {
				w.Write([]byte("event: EOF\n\n"))
				return
			}
			v := recv.Interface()

			// Present each channel item.
			if presenter, ok := r.Context().Value(presenterCtxKey).(Presenter); ok {
				r, v = presenter.Present(r, v)
			}

			// TODO: Can't use json Encoder - it panics on bufio.Flush(). Why?!
			// enc := json.NewEncoder(w)
			// enc.Encode(v.Interface())
			bytes, err := json.Marshal(v)
			if err != nil {
				w.Write([]byte(fmt.Sprintf("event: error\ndata: {\"error\":\"%v\"}\n\n", err)))
				if f, ok := w.(http.Flusher); ok {
					f.Flush()
				}
				continue
			}
			w.Write([]byte(fmt.Sprintf("event: data\ndata: %s\n\n", bytes)))
			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			}
		}
	}
}

// channelIntoSlice buffers channel data into a slice.
func channelIntoSlice(w http.ResponseWriter, r *http.Request, from interface{}) interface{} {
	ctx := r.Context()

	var to []interface{}
	for {
		switch chosen, recv, ok := reflect.Select([]reflect.SelectCase{
			{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(ctx.Done())},
			{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(from)},
		}); chosen {
		case 0: // equivalent to: case <-ctx.Done()
			http.Error(w, "Server Timeout", 504)
			return nil

		default: // equivalent to: case v, ok := <-stream
			if !ok {
				return to
			}
			v := recv.Interface()

			// Present each channel item.
			if presenter, ok := r.Context().Value(presenterCtxKey).(Presenter); ok {
				r, v = presenter.Present(r, v)
			}

			to = append(to, v)
		}
	}
}

// contextKey is a value for use with context.WithValue. It's used as
// a pointer so it fits in an interface{} without allocation. This technique
// for defining context keys was copied from Go 1.7's new use of context in net/http.
type contextKey struct {
	name string
}

func (k *contextKey) String() string {
	return "chi render context value " + k.name
}
<commit_msg>fix typo in godoc for xml marshaler<commit_after>package render

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"net/http"
	"reflect"
)

// Respond handles streaming JSON and XML responses, automatically setting the
// Content-Type based on request headers. It will default to a JSON response.
func Respond(w http.ResponseWriter, r *http.Request, v interface{}) {
	// Present the object.
	if presenter, ok := r.Context().Value(presenterCtxKey).(Presenter); ok {
		r, v = presenter.Present(r, v)
	}

	switch reflect.TypeOf(v).Kind() {
	case reflect.Chan:
		switch getResponseContentType(r) {
		case ContentTypeEventStream:
			channelEventStream(w, r, v)
			return
		default:
			v = channelIntoSlice(w, r, v)
		}
	}

	// Format data based on Content-Type.
	switch getResponseContentType(r) {
	case ContentTypeJSON:
		JSON(w, r, v)
	case ContentTypeXML:
		XML(w, r, v)
	default:
		JSON(w, r, v)
	}
}

// PlainText writes a string to the response, setting the Content-Type as
// text/plain.
func PlainText(w http.ResponseWriter, r *http.Request, v string) {
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	if status, ok := r.Context().Value(statusCtxKey).(int); ok {
		w.WriteHeader(status)
	}
	w.Write([]byte(v))
}

// Data writes raw bytes to the response, setting the Content-Type as
// application/octet-stream.
func Data(w http.ResponseWriter, r *http.Request, v []byte) {
	w.Header().Set("Content-Type", "application/octet-stream")
	if status, ok := r.Context().Value(statusCtxKey).(int); ok {
		w.WriteHeader(status)
	}
	w.Write(v)
}

// HTML writes a string to the response, setting the Content-Type as text/html.
func HTML(w http.ResponseWriter, r *http.Request, v string) {
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	if status, ok := r.Context().Value(statusCtxKey).(int); ok {
		w.WriteHeader(status)
	}
	w.Write([]byte(v))
}

// JSON marshals 'v' to JSON, automatically escaping HTML and setting the
// Content-Type as application/json.
func JSON(w http.ResponseWriter, r *http.Request, v interface{}) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	if status, ok := r.Context().Value(statusCtxKey).(int); ok {
		w.WriteHeader(status)
	}

	enc := json.NewEncoder(w)
	enc.SetEscapeHTML(true)
	if err := enc.Encode(v); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
}

// XML marshals 'v' to XML, setting the Content-Type as application/xml. It
// will automatically prepend a generic XML header (see encoding/xml.Header) if
// one is not found in the first 100 bytes of 'v'.
func XML(w http.ResponseWriter, r *http.Request, v interface{}) {
	b, err := xml.Marshal(v)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
	if status, ok := r.Context().Value(statusCtxKey).(int); ok {
		w.WriteHeader(status)
	}

	// Try to find <?xml header in first 100 bytes (just in case there're some XML comments).
	findHeaderUntil := len(b)
	if findHeaderUntil > 100 {
		findHeaderUntil = 100
	}
	if !bytes.Contains(b[:findHeaderUntil], []byte("<?xml")) {
		// No header found. Print it out first.
		w.Write([]byte(xml.Header))
	}

	w.Write(b)
}

// NoContent returns a HTTP 204 "No Content" response.
func NoContent(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(204)
}

func channelEventStream(w http.ResponseWriter, r *http.Request, v interface{}) {
	if reflect.TypeOf(v).Kind() != reflect.Chan {
		panic(fmt.Sprintf("render.EventStream() expects channel, not %v", reflect.TypeOf(v).Kind()))
	}

	w.Header().Set("Content-Type", "text/event-stream; charset=utf-8")
	w.Header().Set("Cache-Control", "no-cache")
	w.Header().Set("Connection", "keep-alive")
	w.WriteHeader(200)

	ctx := r.Context()
	for {
		switch chosen, recv, ok := reflect.Select([]reflect.SelectCase{
			{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(ctx.Done())},
			{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(v)},
		}); chosen {
		case 0: // equivalent to: case <-ctx.Done()
			w.Write([]byte("event: error\ndata: {\"error\":\"Server Timeout\"}\n\n"))
			return

		default: // equivalent to: case v, ok := <-stream
			if !ok {
				w.Write([]byte("event: EOF\n\n"))
				return
			}
			v := recv.Interface()

			// Present each channel item.
			if presenter, ok := r.Context().Value(presenterCtxKey).(Presenter); ok {
				r, v = presenter.Present(r, v)
			}

			// TODO: Can't use json Encoder - it panics on bufio.Flush(). Why?!
			// enc := json.NewEncoder(w)
			// enc.Encode(v.Interface())
			bytes, err := json.Marshal(v)
			if err != nil {
				w.Write([]byte(fmt.Sprintf("event: error\ndata: {\"error\":\"%v\"}\n\n", err)))
				if f, ok := w.(http.Flusher); ok {
					f.Flush()
				}
				continue
			}
			w.Write([]byte(fmt.Sprintf("event: data\ndata: %s\n\n", bytes)))
			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			}
		}
	}
}

// channelIntoSlice buffers channel data into a slice.
func channelIntoSlice(w http.ResponseWriter, r *http.Request, from interface{}) interface{} {
	ctx := r.Context()

	var to []interface{}
	for {
		switch chosen, recv, ok := reflect.Select([]reflect.SelectCase{
			{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(ctx.Done())},
			{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(from)},
		}); chosen {
		case 0: // equivalent to: case <-ctx.Done()
			http.Error(w, "Server Timeout", 504)
			return nil

		default: // equivalent to: case v, ok := <-stream
			if !ok {
				return to
			}
			v := recv.Interface()

			// Present each channel item.
			if presenter, ok := r.Context().Value(presenterCtxKey).(Presenter); ok {
				r, v = presenter.Present(r, v)
			}

			to = append(to, v)
		}
	}
}

// contextKey is a value for use with context.WithValue. It's used as
// a pointer so it fits in an interface{} without allocation. This technique
// for defining context keys was copied from Go 1.7's new use of context in net/http.
type contextKey struct {
	name string
}

func (k *contextKey) String() string {
	return "chi render context value " + k.name
}
<|endoftext|>


<commit_before>package render

import (
	"encoding/json"
	"html/template"
	"log"
	"net/http"
	"path/filepath"
	"strings"
	"sync"
)

// An interface of suggested methods for rendering output in Go
type MiloRenderer interface {
	RenderTemplates(w http.ResponseWriter, r *http.Request, data map[string]interface{}, tpls ...string)
	RenderJson(w http.ResponseWriter, r *http.Request, data interface{})
	RegisterTemplateFunc(key string, fn interface{})
	Redirect(w http.ResponseWriter, r *http.Request, url string, code int)
}

// Default milo renderer that can cache templates, sets a base template directory.
type DefaultMiloRenderer struct {
	templateCache map[string]*template.Template
	tplDir        string
	tplFuncs      map[string]interface{}
	cacheTpls     bool
	sync.RWMutex
}

// Create a new default milo renderer.
func NewDefaultMiloRenderer(tplDir string, cache bool) MiloRenderer {
	r := &DefaultMiloRenderer{templateCache: make(map[string]*template.Template), tplDir: tplDir, tplFuncs: make(map[string]interface{}), cacheTpls: cache}
	r.tplFuncs["host"] = Host
	r.tplFuncs["marshal"] = Marshal
	return r
}

// Takes care of rendering templates from file.
func (mr *DefaultMiloRenderer) RenderTemplates(w http.ResponseWriter, r *http.Request, data map[string]interface{}, tpls ...string) {
	if len(tpls) < 1 {
		w.WriteHeader(500)
		w.Write([]byte("Error: Template required!"))
		return
	}

	log.Println("Rendering the templates")

	list := make([]string, 0)
	for _, elem := range tpls {
		list = append(list, filepath.Join(mr.tplDir, elem))
	}

	if tpl, loadErr := mr.acquireTemplate(strings.Join(tpls, ""), list...); loadErr != nil {
		w.WriteHeader(500)
		w.Write([]byte(loadErr.Error()))
	} else {
		tpl.Execute(w, data)
	}
}

// Unexported method to help handle template parsing.  If the cache template bool is set on the config
// struct this method with look in the cache & load the cache upon subsequent encounters.
// This should lower disk access penalties useful for production instances.
func (mr *DefaultMiloRenderer) acquireTemplate(key string, tpls ...string) (*template.Template, error) {
	var tpl *template.Template
	var loadErr error
	var ok bool

	if mr.cacheTpls {
		mr.RLock()
		tpl, ok = mr.templateCache[key]
		mr.RUnlock()
		if ok {
			return tpl, nil
		}
	}

	tpl, loadErr = template.New(filepath.Base(tpls[0])).Funcs(mr.tplFuncs).ParseFiles(tpls...)
	if loadErr != nil {
		return nil, loadErr
	}

	if mr.cacheTpls {
		mr.Lock()
		mr.templateCache[key] = tpl
		mr.Unlock()
	}
	return tpl, nil
}

// Render json output
func (mr *DefaultMiloRenderer) RenderJson(w http.ResponseWriter, r *http.Request, data interface{}) {
	if data, err := json.Marshal(data); err != nil {
		w.WriteHeader(500)
		w.Write([]byte(err.Error()))
	} else {
		w.Header().Add("Content-Type", "application/json")
		w.WriteHeader(http.StatusOK)
		w.Write(data)
	}
}

// Register a template function with the MiloRenderer
func (mr *DefaultMiloRenderer) RegisterTemplateFunc(key string, fn interface{}) {
	mr.tplFuncs[key] = fn
}

// Setup an http redirect on the request.
func (mr *DefaultMiloRenderer) Redirect(w http.ResponseWriter, r *http.Request, url string, code int) {
	http.Redirect(w, r, url, code)
}
<commit_msg>Updated render to help with error logging on template execution.<commit_after>package render

import (
	"bytes"
	"encoding/json"
	"html/template"
	"log"
	"net/http"
	"path/filepath"
	"strings"
	"sync"
)

// An interface of suggested methods for rendering output in Go
type MiloRenderer interface {
	RenderTemplates(w http.ResponseWriter, r *http.Request, data map[string]interface{}, tpls ...string)
	RenderJson(w http.ResponseWriter, r *http.Request, data interface{})
	RegisterTemplateFunc(key string, fn interface{})
	Redirect(w http.ResponseWriter, r *http.Request, url string, code int)
}

// Default milo renderer that can cache templates, sets a base template directory.
type DefaultMiloRenderer struct {
	templateCache map[string]*template.Template
	tplDir        string
	tplFuncs      map[string]interface{}
	cacheTpls     bool
	sync.RWMutex
}

// Create a new default milo renderer.
func NewDefaultMiloRenderer(tplDir string, cache bool) MiloRenderer {
	r := &DefaultMiloRenderer{templateCache: make(map[string]*template.Template), tplDir: tplDir, tplFuncs: make(map[string]interface{}), cacheTpls: cache}
	r.tplFuncs["host"] = Host
	r.tplFuncs["marshal"] = Marshal
	return r
}

// Takes care of rendering templates from file.
func (mr *DefaultMiloRenderer) RenderTemplates(w http.ResponseWriter, r *http.Request, data map[string]interface{}, tpls ...string) {
	if len(tpls) < 1 {
		w.WriteHeader(500)
		w.Write([]byte("Error: Template required!"))
		return
	}

	log.Println("Rendering the templates")

	list := make([]string, 0)
	for _, elem := range tpls {
		list = append(list, filepath.Join(mr.tplDir, elem))
	}

	if tpl, loadErr := mr.acquireTemplate(strings.Join(tpls, ""), list...); loadErr != nil {
		w.WriteHeader(500)
		w.Write([]byte(loadErr.Error()))
	} else {
		var doc bytes.Buffer
		err := tpl.Execute(&doc, data)
		if err == nil {
			w.WriteHeader(200)
			w.Write(doc.Bytes())
		} else {
			w.WriteHeader(500)
			w.Write([]byte(err.Error()))
		}
	}
}

// Unexported method to help handle template parsing.  If the cache template bool is set on the config
// struct this method with look in the cache & load the cache upon subsequent encounters.
// This should lower disk access penalties useful for production instances.
func (mr *DefaultMiloRenderer) acquireTemplate(key string, tpls ...string) (*template.Template, error) {
	var tpl *template.Template
	var loadErr error
	var ok bool

	if mr.cacheTpls {
		mr.RLock()
		tpl, ok = mr.templateCache[key]
		mr.RUnlock()
		if ok {
			return tpl, nil
		}
	}

	tpl, loadErr = template.New(filepath.Base(tpls[0])).Funcs(mr.tplFuncs).ParseFiles(tpls...)
	if loadErr != nil {
		return nil, loadErr
	}

	if mr.cacheTpls {
		mr.Lock()
		mr.templateCache[key] = tpl
		mr.Unlock()
	}
	return tpl, nil
}

// Render json output
func (mr *DefaultMiloRenderer) RenderJson(w http.ResponseWriter, r *http.Request, data interface{}) {
	if data, err := json.Marshal(data); err != nil {
		w.WriteHeader(500)
		w.Write([]byte(err.Error()))
	} else {
		w.Header().Add("Content-Type", "application/json")
		w.WriteHeader(http.StatusOK)
		w.Write(data)
	}
}

// Register a template function with the MiloRenderer
func (mr *DefaultMiloRenderer) RegisterTemplateFunc(key string, fn interface{}) {
	mr.tplFuncs[key] = fn
}

// Setup an http redirect on the request.
func (mr *DefaultMiloRenderer) Redirect(w http.ResponseWriter, r *http.Request, url string, code int) {
	http.Redirect(w, r, url, code)
}
<|endoftext|>


<commit_before>// Copyright 2015 Matthew Collins
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package render

import (
	"math"
	"os"

	"github.com/go-gl/mathgl/mgl32"
	"github.com/thinkofdeath/steven/console"
	"github.com/thinkofdeath/steven/render/gl"
	"github.com/thinkofdeath/steven/render/glsl"
	"github.com/thinkofdeath/steven/type/direction"
	"github.com/thinkofdeath/steven/type/vmath"
)

var (
	chunkProgram  gl.Program
	shaderChunk   *chunkShader
	chunkProgramT gl.Program
	shaderChunkT  *chunkShader
	lineProgram   gl.Program
	shaderLine    *lineShader

	FOV = console.NewIntVar("r_fov", 90, console.Mutable, console.Serializable).Doc(`
r_fov controls the field of view of the camera. Measured
in degrees.
`)
	lastFOV               int = 90
	lastWidth, lastHeight int = -1, -1
	perspectiveMatrix         = mgl32.Mat4{}
	cameraMatrix              = mgl32.Mat4{}
	frustum                   = vmath.NewFrustum()

	syncChan = make(chan func(), 500)

	glTexture       gl.Texture
	textureDepth    int
	texturesCreated bool

	debugFramebuffers = console.NewBoolVar("r_debug_buffers", false, console.Mutable).Doc(`
r_debug_buffers blits all frame buffers to the screen for
debugging.
`)

	LightLevel, SkyOffset float32 = 0.8, 1.0
	ClearColour                   = struct{ R, G, B float32 }{
		122.0 / 255.0, 165.0 / 255.0, 247.0 / 255.0,
	}
)

// Start starts the renderer
func Start() {
	if os.Getenv("STEVEN_DEBUG") == "true" {
		gl.DebugLog()
	}

	gl.Enable(gl.DepthTest)
	gl.Enable(gl.CullFaceFlag)
	gl.CullFace(gl.Back)
	gl.FrontFace(gl.ClockWise)

	chunkProgram = CreateProgram(
		glsl.Get("chunk_vertex"),
		glsl.Get("chunk_frag"),
	)
	shaderChunk = &chunkShader{}
	InitStruct(shaderChunk, chunkProgram)

	chunkProgramT = CreateProgram(
		glsl.Get("chunk_vertex"),
		glsl.Get("chunk_frag", "alpha"),
	)
	shaderChunkT = &chunkShader{}
	InitStruct(shaderChunkT, chunkProgramT)

	initUI()
	initLineDraw()
	initModels()
	clouds.init()

	gl.BlendFunc(gl.SrcAlpha, gl.OneMinusSrcAlpha)

	elementBuffer = gl.CreateBuffer()
}

var (
	textureIds    []int
	frameID       uint
	nearestBuffer *ChunkBuffer
	viewVector    mgl32.Vec3
)

// Draw draws a single frame
func Draw(width, height int, delta float64) {
	tickAnimatedTextures(delta)
	frameID++
sync:
	for {
		select {
		case f := <-syncChan:
			f()
		default:
			break sync
		}
	}

	// Only update the viewport if the window was resized
	if lastHeight != height || lastWidth != width || lastFOV != FOV.Value() {
		lastWidth = width
		lastHeight = height
		lastFOV = FOV.Value()

		perspectiveMatrix = mgl32.Perspective(
			(math.Pi/180)*float32(lastFOV),
			float32(width)/float32(height),
			0.1,
			500.0,
		)
		gl.Viewport(0, 0, width, height)
		frustum.SetPerspective(
			(math.Pi/180)*float32(lastFOV),
			float32(width)/float32(height),
			0.1,
			500.0,
		)
		initTrans()
	}

	mainFramebuffer.Bind()
	gl.Enable(gl.Multisample)

	gl.ActiveTexture(0)
	glTexture.Bind(gl.Texture2DArray)

	gl.ClearColor(ClearColour.R, ClearColour.G, ClearColour.B, 1.0)
	gl.Clear(gl.ColorBufferBit | gl.DepthBufferBit)

	chunkProgram.Use()

	viewVector = mgl32.Vec3{
		float32(math.Cos(Camera.Yaw-math.Pi/2) * -math.Cos(Camera.Pitch)),
		float32(-math.Sin(Camera.Pitch)),
		float32(-math.Sin(Camera.Yaw-math.Pi/2) * -math.Cos(Camera.Pitch)),
	}
	cam := mgl32.Vec3{-float32(Camera.X), -float32(Camera.Y), float32(Camera.Z)}
	cameraMatrix = mgl32.LookAtV(
		cam,
		cam.Add(mgl32.Vec3{-viewVector.X(), -viewVector.Y(), viewVector.Z()}),
		mgl32.Vec3{0, -1, 0},
	)
	cameraMatrix = cameraMatrix.Mul4(mgl32.Scale3D(-1.0, 1.0, 1.0))

	frustum.SetCamera(
		cam,
		cam.Add(mgl32.Vec3{-viewVector.X(), -viewVector.Y(), viewVector.Z()}),
		mgl32.Vec3{0, -1, 0},
	)

	shaderChunk.PerspectiveMatrix.Matrix4(&perspectiveMatrix)
	shaderChunk.CameraMatrix.Matrix4(&cameraMatrix)
	shaderChunk.Texture.Int(0)
	shaderChunk.LightLevel.Float(LightLevel)
	shaderChunk.SkyOffset.Float(SkyOffset)

	chunkPos := position{
		X: int(Camera.X) >> 4,
		Y: int(Camera.Y) >> 4,
		Z: int(Camera.Z) >> 4,
	}
	nearestBuffer = buffers[chunkPos]

	for _, dir := range direction.Values {
		validDirs[dir] = viewVector.Dot(dir.AsVec()) > -0.8
	}

	renderOrder = renderOrder[:0]
	renderBuffer(nearestBuffer, chunkPos, direction.Invalid, delta)

	drawLines()
	drawModels()
	clouds.tick(delta)

	chunkProgramT.Use()
	shaderChunkT.PerspectiveMatrix.Matrix4(&perspectiveMatrix)
	shaderChunkT.CameraMatrix.Matrix4(&cameraMatrix)
	shaderChunkT.Texture.Int(0)
	shaderChunkT.LightLevel.Float(LightLevel)
	shaderChunkT.SkyOffset.Float(SkyOffset)

	// Copy the depth buffer
	mainFramebuffer.BindRead()
	transFramebuffer.BindDraw()
	gl.BlitFramebuffer(
		0, 0, lastWidth, lastHeight,
		0, 0, lastWidth, lastHeight,
		gl.DepthBufferBit, gl.Nearest,
	)

	gl.Enable(gl.Blend)
	gl.DepthMask(false)
	transFramebuffer.Bind()
	gl.ClearColor(0, 0, 0, 1)
	gl.Clear(gl.ColorBufferBit)
	gl.ClearBuffer(gl.Color, 0, []float32{0, 0, 0, 1})
	gl.ClearBuffer(gl.Color, 1, []float32{0, 0, 0, 0})
	gl.BlendFuncSeparate(gl.OneFactor, gl.OneFactor, gl.ZeroFactor, gl.OneMinusSrcAlpha)
	for _, chunk := range renderOrder {
		if chunk.countT > 0 && chunk.bufferT.IsValid() {
			shaderChunkT.Offset.Int3(chunk.X, chunk.Y*4096-chunk.Y*int(4096*(1-chunk.progress)), chunk.Z)

			chunk.arrayT.Bind()
			gl.DrawElements(gl.Triangles, chunk.countT, elementBufferType, 0)
		}
	}

	gl.UnbindFramebuffer()
	gl.Disable(gl.DepthTest)
	gl.Clear(gl.ColorBufferBit)
	gl.Disable(gl.Blend)

	transDraw()

	gl.Enable(gl.DepthTest)
	gl.DepthMask(true)
	gl.BlendFunc(gl.SrcAlpha, gl.OneMinusSrcAlpha)
	gl.Disable(gl.Multisample)

	drawUI()

	if debugFramebuffers.Value() {
		gl.Enable(gl.Multisample)
		blitBuffers()
		gl.Disable(gl.Multisample)
	}
}

var (
	renderOrder []*ChunkBuffer
	validDirs   = make([]bool, len(direction.Values))
)

type renderRequest struct {
	chunk *ChunkBuffer
	pos   position
	from  direction.Type
}

const (
	renderQueueSize = 5000
)

var rQueue renderQueue

func renderBuffer(ch *ChunkBuffer, po position, fr direction.Type, delta float64) {
	if ch == nil {
		return
	}
	rQueue.Append(renderRequest{ch, po, fr})

	for !rQueue.Empty() {
		req := rQueue.Take()
		if req.chunk.renderedOn == frameID {
			continue
		}
		req.chunk.renderedOn = frameID

		aabb := vmath.NewAABB(
			-float32((req.pos.X<<4)+16), -float32((req.pos.Y<<4)+16), float32((req.pos.Z<<4)),
			-float32((req.pos.X<<4)), -float32((req.pos.Y<<4)), float32((req.pos.Z<<4)+16),
		).Grow(1, 1, 1)
		if !frustum.IsAABBInside(aabb) {
			continue
		}
		renderOrder = append(renderOrder, req.chunk)

		req.chunk.Rendered = true

		dx := req.pos.X - int(Camera.X)>>4
		dz := req.pos.Z - int(Camera.Z)>>4

		if req.chunk.progress < 1 && math.Sqrt(float64(dx*dx+dz*dz)) > 6 {
			req.chunk.progress += delta * 0.015
		} else {
			req.chunk.progress = 1
		}

		if req.chunk.count != 0 && req.chunk.buffer.IsValid() {
			shaderChunk.Offset.Int3(req.chunk.X, req.chunk.Y*4096-req.chunk.Y*int(4096*(1-req.chunk.progress)), req.chunk.Z)

			req.chunk.array.Bind()
			gl.DrawElements(gl.Triangles, req.chunk.count, elementBufferType, 0)
		}

		for _, dir := range direction.Values {
			c := req.chunk.neighborChunks[dir]
			if dir != req.from && c != nil && c.renderedOn != frameID &&
				(req.from == direction.Invalid || (req.chunk.IsVisible(req.from, dir) && validDirs[dir])) {
				ox, oy, oz := dir.Offset()
				pos := position{req.pos.X + ox, req.pos.Y + oy, req.pos.Z + oz}
				rQueue.Append(renderRequest{c, pos, dir.Opposite()})
			}
		}
	}
}

// Sync runs the passed function on the next frame on the same goroutine
// as the renderer.
func Sync(f func()) {
	syncChan <- f
}
<commit_msg>render: put slidy chunks behind a 'r_slidy_chunks' flag<commit_after>// Copyright 2015 Matthew Collins
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package render

import (
	"math"
	"os"

	"github.com/go-gl/mathgl/mgl32"
	"github.com/thinkofdeath/steven/console"
	"github.com/thinkofdeath/steven/render/gl"
	"github.com/thinkofdeath/steven/render/glsl"
	"github.com/thinkofdeath/steven/type/direction"
	"github.com/thinkofdeath/steven/type/vmath"
)

var (
	chunkProgram  gl.Program
	shaderChunk   *chunkShader
	chunkProgramT gl.Program
	shaderChunkT  *chunkShader
	lineProgram   gl.Program
	shaderLine    *lineShader

	FOV = console.NewIntVar("r_fov", 90, console.Mutable, console.Serializable).Doc(`
r_fov controls the field of view of the camera. Measured
in degrees.
`)
	lastFOV               int = 90
	lastWidth, lastHeight int = -1, -1
	perspectiveMatrix         = mgl32.Mat4{}
	cameraMatrix              = mgl32.Mat4{}
	frustum                   = vmath.NewFrustum()

	syncChan = make(chan func(), 500)

	glTexture       gl.Texture
	textureDepth    int
	texturesCreated bool

	debugFramebuffers = console.NewBoolVar("r_debug_buffers", false, console.Mutable).Doc(`
r_debug_buffers blits all frame buffers to the screen for
debugging.
`)

	LightLevel, SkyOffset float32 = 0.8, 1.0
	ClearColour                   = struct{ R, G, B float32 }{
		122.0 / 255.0, 165.0 / 255.0, 247.0 / 255.0,
	}

	slidyChunks = console.NewBoolVar("r_slidy_chunks", false, console.Mutable, console.Serializable).Doc(`
r_slidy_chunks makes chunks slide into view instead of just
popping in.
`)
)

// Start starts the renderer
func Start() {
	if os.Getenv("STEVEN_DEBUG") == "true" {
		gl.DebugLog()
	}

	gl.Enable(gl.DepthTest)
	gl.Enable(gl.CullFaceFlag)
	gl.CullFace(gl.Back)
	gl.FrontFace(gl.ClockWise)

	chunkProgram = CreateProgram(
		glsl.Get("chunk_vertex"),
		glsl.Get("chunk_frag"),
	)
	shaderChunk = &chunkShader{}
	InitStruct(shaderChunk, chunkProgram)

	chunkProgramT = CreateProgram(
		glsl.Get("chunk_vertex"),
		glsl.Get("chunk_frag", "alpha"),
	)
	shaderChunkT = &chunkShader{}
	InitStruct(shaderChunkT, chunkProgramT)

	initUI()
	initLineDraw()
	initModels()
	clouds.init()

	gl.BlendFunc(gl.SrcAlpha, gl.OneMinusSrcAlpha)

	elementBuffer = gl.CreateBuffer()
}

var (
	textureIds    []int
	frameID       uint
	nearestBuffer *ChunkBuffer
	viewVector    mgl32.Vec3
)

// Draw draws a single frame
func Draw(width, height int, delta float64) {
	tickAnimatedTextures(delta)
	frameID++
sync:
	for {
		select {
		case f := <-syncChan:
			f()
		default:
			break sync
		}
	}

	// Only update the viewport if the window was resized
	if lastHeight != height || lastWidth != width || lastFOV != FOV.Value() {
		lastWidth = width
		lastHeight = height
		lastFOV = FOV.Value()

		perspectiveMatrix = mgl32.Perspective(
			(math.Pi/180)*float32(lastFOV),
			float32(width)/float32(height),
			0.1,
			500.0,
		)
		gl.Viewport(0, 0, width, height)
		frustum.SetPerspective(
			(math.Pi/180)*float32(lastFOV),
			float32(width)/float32(height),
			0.1,
			500.0,
		)
		initTrans()
	}

	mainFramebuffer.Bind()
	gl.Enable(gl.Multisample)

	gl.ActiveTexture(0)
	glTexture.Bind(gl.Texture2DArray)

	gl.ClearColor(ClearColour.R, ClearColour.G, ClearColour.B, 1.0)
	gl.Clear(gl.ColorBufferBit | gl.DepthBufferBit)

	chunkProgram.Use()

	viewVector = mgl32.Vec3{
		float32(math.Cos(Camera.Yaw-math.Pi/2) * -math.Cos(Camera.Pitch)),
		float32(-math.Sin(Camera.Pitch)),
		float32(-math.Sin(Camera.Yaw-math.Pi/2) * -math.Cos(Camera.Pitch)),
	}
	cam := mgl32.Vec3{-float32(Camera.X), -float32(Camera.Y), float32(Camera.Z)}
	cameraMatrix = mgl32.LookAtV(
		cam,
		cam.Add(mgl32.Vec3{-viewVector.X(), -viewVector.Y(), viewVector.Z()}),
		mgl32.Vec3{0, -1, 0},
	)
	cameraMatrix = cameraMatrix.Mul4(mgl32.Scale3D(-1.0, 1.0, 1.0))

	frustum.SetCamera(
		cam,
		cam.Add(mgl32.Vec3{-viewVector.X(), -viewVector.Y(), viewVector.Z()}),
		mgl32.Vec3{0, -1, 0},
	)

	shaderChunk.PerspectiveMatrix.Matrix4(&perspectiveMatrix)
	shaderChunk.CameraMatrix.Matrix4(&cameraMatrix)
	shaderChunk.Texture.Int(0)
	shaderChunk.LightLevel.Float(LightLevel)
	shaderChunk.SkyOffset.Float(SkyOffset)

	chunkPos := position{
		X: int(Camera.X) >> 4,
		Y: int(Camera.Y) >> 4,
		Z: int(Camera.Z) >> 4,
	}
	nearestBuffer = buffers[chunkPos]

	for _, dir := range direction.Values {
		validDirs[dir] = viewVector.Dot(dir.AsVec()) > -0.8
	}

	renderOrder = renderOrder[:0]
	renderBuffer(nearestBuffer, chunkPos, direction.Invalid, delta)

	drawLines()
	drawModels()
	clouds.tick(delta)

	chunkProgramT.Use()
	shaderChunkT.PerspectiveMatrix.Matrix4(&perspectiveMatrix)
	shaderChunkT.CameraMatrix.Matrix4(&cameraMatrix)
	shaderChunkT.Texture.Int(0)
	shaderChunkT.LightLevel.Float(LightLevel)
	shaderChunkT.SkyOffset.Float(SkyOffset)

	// Copy the depth buffer
	mainFramebuffer.BindRead()
	transFramebuffer.BindDraw()
	gl.BlitFramebuffer(
		0, 0, lastWidth, lastHeight,
		0, 0, lastWidth, lastHeight,
		gl.DepthBufferBit, gl.Nearest,
	)

	gl.Enable(gl.Blend)
	gl.DepthMask(false)
	transFramebuffer.Bind()
	gl.ClearColor(0, 0, 0, 1)
	gl.Clear(gl.ColorBufferBit)
	gl.ClearBuffer(gl.Color, 0, []float32{0, 0, 0, 1})
	gl.ClearBuffer(gl.Color, 1, []float32{0, 0, 0, 0})
	gl.BlendFuncSeparate(gl.OneFactor, gl.OneFactor, gl.ZeroFactor, gl.OneMinusSrcAlpha)
	for _, chunk := range renderOrder {
		if chunk.countT > 0 && chunk.bufferT.IsValid() {
			shaderChunkT.Offset.Int3(chunk.X, chunk.Y*4096-chunk.Y*int(4096*(1-chunk.progress)), chunk.Z)

			chunk.arrayT.Bind()
			gl.DrawElements(gl.Triangles, chunk.countT, elementBufferType, 0)
		}
	}

	gl.UnbindFramebuffer()
	gl.Disable(gl.DepthTest)
	gl.Clear(gl.ColorBufferBit)
	gl.Disable(gl.Blend)

	transDraw()

	gl.Enable(gl.DepthTest)
	gl.DepthMask(true)
	gl.BlendFunc(gl.SrcAlpha, gl.OneMinusSrcAlpha)
	gl.Disable(gl.Multisample)

	drawUI()

	if debugFramebuffers.Value() {
		gl.Enable(gl.Multisample)
		blitBuffers()
		gl.Disable(gl.Multisample)
	}
}

var (
	renderOrder []*ChunkBuffer
	validDirs   = make([]bool, len(direction.Values))
)

type renderRequest struct {
	chunk *ChunkBuffer
	pos   position
	from  direction.Type
}

const (
	renderQueueSize = 5000
)

var rQueue renderQueue

func renderBuffer(ch *ChunkBuffer, po position, fr direction.Type, delta float64) {
	if ch == nil {
		return
	}
	rQueue.Append(renderRequest{ch, po, fr})

	slidy := slidyChunks.Value()

	for !rQueue.Empty() {
		req := rQueue.Take()
		if req.chunk.renderedOn == frameID {
			continue
		}
		req.chunk.renderedOn = frameID

		aabb := vmath.NewAABB(
			-float32((req.pos.X<<4)+16), -float32((req.pos.Y<<4)+16), float32((req.pos.Z<<4)),
			-float32((req.pos.X<<4)), -float32((req.pos.Y<<4)), float32((req.pos.Z<<4)+16),
		).Grow(1, 1, 1)
		if !frustum.IsAABBInside(aabb) {
			continue
		}
		renderOrder = append(renderOrder, req.chunk)

		req.chunk.Rendered = true

		if slidy {
			dx := req.pos.X - int(Camera.X)>>4
			dz := req.pos.Z - int(Camera.Z)>>4

			if req.chunk.progress < 1 && dx*dx+dz*dz > 6*6 {
				req.chunk.progress += delta * 0.015
			} else {
				req.chunk.progress = 1
			}
		} else {
			req.chunk.progress = 1
		}

		if req.chunk.count != 0 && req.chunk.buffer.IsValid() {
			shaderChunk.Offset.Int3(req.chunk.X, req.chunk.Y*4096-req.chunk.Y*int(4096*(1-req.chunk.progress)), req.chunk.Z)

			req.chunk.array.Bind()
			gl.DrawElements(gl.Triangles, req.chunk.count, elementBufferType, 0)
		}

		for _, dir := range direction.Values {
			c := req.chunk.neighborChunks[dir]
			if dir != req.from && c != nil && c.renderedOn != frameID &&
				(req.from == direction.Invalid || (req.chunk.IsVisible(req.from, dir) && validDirs[dir])) {
				ox, oy, oz := dir.Offset()
				pos := position{req.pos.X + ox, req.pos.Y + oy, req.pos.Z + oz}
				rQueue.Append(renderRequest{c, pos, dir.Opposite()})
			}
		}
	}
}

// Sync runs the passed function on the next frame on the same goroutine
// as the renderer.
func Sync(f func()) {
	syncChan <- f
}
<|endoftext|>


<commit_before><commit_msg>Make `SyscallTable.lookup` be a fixed-size array rather than a slice.<commit_after><|endoftext|>


<commit_before>/*
Package reportsockets implements a websocket interface where all the clients
get the same data.

It internally works like a publish/subscribe model where each websocket
connection subscribe to a publisher.

It implements the http.Handler interface so you can use it with the standard net/http.

To use it, you first need to declare the channel or exchange.

	exchange := reportsockets.New()

To tie it to a url, use the standard http.

	http.Handle("/report", exchange.Handler()

To send messages to all the clients:

	msg := "Hello world"
	exchange.Publish(&msg)

Other thing that you can do is declare a handler for reciving client messages.

	func myFunc(msg []byte, ws *websocket.Conn, exchange *reportsockets.Exchange){
	  ...
	  // You can responds to the specific client:
	  ws.Write(myData)
	  /e/ Or send a messages to other clients.
	  exchagne.Publish(msg)
	}

	exchange.ClientMessageHandler = myFunc

If you don't define that method, all the client messages will be ignored.
*/
package reportsockets

import (
	"code.google.com/p/go.net/websocket"
	"net/http"
)

type Exchange struct {
	ClientMessageHandler func(msg []byte, ws *websocket.Conn, exchange *Exchange)
	clients              []*websocket.Conn
	newClientChan        chan *websocket.Conn
	publishChan          chan []byte
	removeClientChan     chan *websocket.Conn
	stopChan             chan bool
}

func New() *Exchange {
	e := new(Exchange)
	e.clients = make([]*websocket.Conn, 0)
	e.newClientChan = make(chan *websocket.Conn)
	e.publishChan = make(chan []byte)
	e.stopChan = make(chan bool)
	e.removeClientChan = make(chan *websocket.Conn)
	go e.loop()
	return e
}

func (e *Exchange) loop() {

forLoop:
	for {
		select {
		case newClient := <-e.newClientChan:
			e.clients = append(e.clients, newClient)
		case msg := <-e.publishChan:
			for i, client := range e.clients {
				_, err := client.Write(msg)
				if err != nil {
					client.Close()
					e.clients = append(e.clients[:i], e.clients[(i+1):]...)
				}
			}
		case <-e.stopChan:
			for _, client := range e.clients {
				client.Close()
			}
			break forLoop
		case oldWs := <-e.removeClientChan:
			for i, ws := range e.clients {
				if oldWs == ws {
					e.clients = append(e.clients[:i], e.clients[(i+1):]...)
				}
			}
		}
	}
	for _ = range e.clients {
		<-e.removeClientChan
	}
	e.clients = e.clients[0:0]
}

func (e *Exchange) Handler() http.Handler {
	// ws.Close will be called when returning from this func
	handler := func(ws *websocket.Conn) {
		e.newClientChan <- ws
		for {
			data := make([]byte, 1024*10)
			_, err := ws.Read(data)
			if err != nil {
				e.removeClientChan <- ws
				break
			}
			if e.ClientMessageHandler != nil {
				e.ClientMessageHandler(data, ws, e)
			}
		}
	}
	return websocket.Handler(handler)
}

func (e *Exchange) Publish(msg []byte) {
	e.publishChan <- msg
}

func (e *Exchange) Stop() {
	e.stopChan <- true
}
<commit_msg>Improve docs<commit_after>/*
Package reportsockets implements a websocket interface where all the clients
get the same data.

It internally works like a publish/subscribe model where each websocket
connection subscribe to a publisher.

It implements the http.Handler interface so you can use it with the standard net/http.

To use it, you first need to declare the channel or exchange.

	exchange := reportsockets.New()

To tie it to a url, use the standard http.

	http.Handle("/report", exchange.Handler())

To send messages to all the clients:

	msg := "Hello world"
	exchange.Publish(&msg)

Other thing that you can do is declare a handler for reciving client messages.

	func myFunc(msg []byte, ws *websocket.Conn, exchange *reportsockets.Exchange){
	  ...
	  // You can responds to the specific client:
	  ws.Write(myData)
	  /e/ Or send a messages to other clients.
	  exchagne.Publish(msg)
	}

	exchange.ClientMessageHandler = myFunc

If you don't define that method, all the client messages will be ignored.

Once you are done, you can close it with Stop.

	exchange.Stop()

This will close all the websocket connections
*/
package reportsockets

import (
	"code.google.com/p/go.net/websocket"
	"net/http"
)

// Exchange represent the main point to subscribe
type Exchange struct {
	ClientMessageHandler func(msg []byte, ws *websocket.Conn, exchange *Exchange)
	clients              []*websocket.Conn
	newClientChan        chan *websocket.Conn
	publishChan          chan []byte
	removeClientChan     chan *websocket.Conn
	stopChan             chan bool
}

// New return a new exchange
func New() *Exchange {
	e := new(Exchange)
	e.clients = make([]*websocket.Conn, 0)
	e.newClientChan = make(chan *websocket.Conn)
	e.publishChan = make(chan []byte)
	e.stopChan = make(chan bool)
	e.removeClientChan = make(chan *websocket.Conn)
	go e.loop()
	return e
}

func (e *Exchange) loop() {
forLoop:
	for {
		select {
		case newClient := <-e.newClientChan:
			e.clients = append(e.clients, newClient)
		case msg := <-e.publishChan:
			for i, client := range e.clients {
				_, err := client.Write(msg)
				if err != nil {
					client.Close()
					e.clients = append(e.clients[:i], e.clients[(i+1):]...)
				}
			}
		case <-e.stopChan:
			for _, client := range e.clients {
				client.Close()
			}
			break forLoop
		case oldWs := <-e.removeClientChan:
			for i, ws := range e.clients {
				if oldWs == ws {
					e.clients = append(e.clients[:i], e.clients[(i+1):]...)
				}
			}
		}
	}
	for _ = range e.clients {
		<-e.removeClientChan
	}
	e.clients = e.clients[0:0]
}

// Handler return the http handler for this exchange.
func (e *Exchange) Handler() http.Handler {
	// ws.Close will be called when returning from this func
	handler := func(ws *websocket.Conn) {
		e.newClientChan <- ws
		for {
			data := make([]byte, 1024*10)
			_, err := ws.Read(data)
			if err != nil {
				e.removeClientChan <- ws
				break
			}
			if e.ClientMessageHandler != nil {
				e.ClientMessageHandler(data, ws, e)
			}
		}
	}
	return websocket.Handler(handler)
}

// Publish Broadcast the message to all the clients
func (e *Exchange) Publish(msg []byte) {
	e.publishChan <- msg
}

// Stop will stop the exchange and close all the ws connections
func (e *Exchange) Stop() {
	e.stopChan <- true
}
<|endoftext|>


<commit_before>package main

import (
	"os"

	"gopkg.in/src-d/go-kallax.v1/generator/cli/kallax/cmd"

	"gopkg.in/urfave/cli.v1"
)

const version = "1.2.2"

func main() {
	newApp().Run(os.Args)
}

func newApp() *cli.App {
	app := cli.NewApp()
	app.Name = "kallax"
	app.Version = version
	app.Usage = "generate kallax models"
	app.Flags = cmd.Generate.Flags
	app.Action = cmd.Generate.Action
	app.Commands = cli.Commands{
		cmd.Generate,
		cmd.Migrate,
	}

	return app
}
<commit_msg>Update cmd.go<commit_after>package main

import (
	"os"

	"gopkg.in/src-d/go-kallax.v1/generator/cli/kallax/cmd"

	"gopkg.in/urfave/cli.v1"
)

const version = "1.2.3"

func main() {
	newApp().Run(os.Args)
}

func newApp() *cli.App {
	app := cli.NewApp()
	app.Name = "kallax"
	app.Version = version
	app.Usage = "generate kallax models"
	app.Flags = cmd.Generate.Flags
	app.Action = cmd.Generate.Action
	app.Commands = cli.Commands{
		cmd.Generate,
		cmd.Migrate,
	}

	return app
}
<|endoftext|>


<commit_before>package drum

import (
	"encoding/binary"
	"errors"
	"io"
	"log"
	"os"
	"strings"
)

// DecodeFile decodes the drum machine file found at the provided path
// and returns a pointer to a parsed pattern which is the entry point to the
// rest of the data.
func DecodeFile(path string) (*Pattern, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}

	p := &Pattern{}
	if err := NewDecoder(file).Decode(p); err != nil {
		log.Fatal(err)
		return nil, err
	}

	return p, nil
}

// Decoder reads and decodes splice patterns from an input stream
type Decoder struct {
	io.Reader
}

// NewDecoder returns a new decoder that reads from r
func NewDecoder(r io.Reader) *Decoder {
	return &Decoder{r}
}

// Decode reads the input stream and decodes it's values into a Pattern
func (d *Decoder) Decode(p *Pattern) error {
	n, err := d.spliceHeaderInfo()
	if err != nil {
		if err != io.EOF && err != io.ErrUnexpectedEOF {
			return errors.New("unable to decode .splice file header: " + err.Error())
		}
	}

	if err := d.decodeBody(p, n); err != nil {
		if err != io.EOF && err != io.ErrUnexpectedEOF {
			return errors.New("unable to decode .splice format: " + err.Error())
		}
	}
	return nil
}

// spliceHeaderInfo will read the inital bites
// and returns the size of the file contents or error if it's
// unable check the SPLICE portion or read the headers
func (d *Decoder) spliceHeaderInfo() (int64, error) {
	hdr := make([]byte, 6)
	if _, err := io.ReadFull(d, hdr); err != nil {
		return 0, err
	}
	if string(hdr) != "SPLICE" {
		return 0, errors.New("unable to decode non SPLICE files")
	}

	var size int64
	if err := binary.Read(d, binary.BigEndian, &size); err != nil {
		return 0, err
	}

	return size, nil
}

// decodeBody reads the contents of the input stream up to size
// and into p. It returns EOF or ErrUnexpectedEOF if it reads over
// size.
func (d *Decoder) decodeBody(p *Pattern, size int64) error {
	version := make([]byte, 32)
	if _, err := io.ReadFull(d, version); err != nil {
		return errors.New("unable to decode hw version: " + err.Error())
	}
	p.Version = strings.Trim(string(version), "\x00")

	if err := binary.Read(d, binary.LittleEndian, &p.Tempo); err != nil {
		return errors.New("unable to decode tempo: " + err.Error())
	}

	// version and tempo
	size -= (32 + 4)

	// decode the tracks
	for size >= 0 {
		t := Track{}
		if err := binary.Read(d, binary.BigEndian, &t.ID); err != nil {
			return err
		}

		var nameLen int32
		if err := binary.Read(d, binary.BigEndian, &nameLen); err != nil {
			return err
		}

		name := make([]byte, nameLen)
		if _, err := io.ReadFull(d, name); err != nil {
			return err
		}
		t.Name = string(name)

		steps := make([]byte, measureCount)
		if err := binary.Read(d, binary.BigEndian, steps); err != nil {
			return err
		}

		// convert steps into 'x' or '-' runes
		for idx, step := range steps {
			if step == 1 {
				steps[idx] = 'x'
			} else {
				steps[idx] = '-'
			}
		}
		t.Steps = steps

		p.Tracks = append(p.Tracks, t)
		// ID, nameLen, measureCount, and size of name
		size -= (1 + 4 + measureCount + int64(len(name)))
	}

	return nil
}
<commit_msg>Remove leftover log.Fatal<commit_after>package drum

import (
	"encoding/binary"
	"errors"
	"io"
	"os"
	"strings"
)

// DecodeFile decodes the drum machine file found at the provided path
// and returns a pointer to a parsed pattern which is the entry point to the
// rest of the data.
func DecodeFile(path string) (*Pattern, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}

	p := &Pattern{}
	if err := NewDecoder(file).Decode(p); err != nil {
		return nil, err
	}

	return p, nil
}

// Decoder reads and decodes splice patterns from an input stream
type Decoder struct {
	io.Reader
}

// NewDecoder returns a new decoder that reads from r
func NewDecoder(r io.Reader) *Decoder {
	return &Decoder{r}
}

// Decode reads the input stream and decodes it's values into a Pattern
func (d *Decoder) Decode(p *Pattern) error {
	n, err := d.spliceHeaderInfo()
	if err != nil {
		if err != io.EOF && err != io.ErrUnexpectedEOF {
			return errors.New("unable to decode .splice file header: " + err.Error())
		}
	}

	if err := d.decodeBody(p, n); err != nil {
		if err != io.EOF && err != io.ErrUnexpectedEOF {
			return errors.New("unable to decode .splice format: " + err.Error())
		}
	}
	return nil
}

// spliceHeaderInfo will read the inital bites
// and returns the size of the file contents or error if it's
// unable check the SPLICE portion or read the headers
func (d *Decoder) spliceHeaderInfo() (int64, error) {
	hdr := make([]byte, 6)
	if _, err := io.ReadFull(d, hdr); err != nil {
		return 0, err
	}
	if string(hdr) != "SPLICE" {
		return 0, errors.New("unable to decode non SPLICE files")
	}

	var size int64
	if err := binary.Read(d, binary.BigEndian, &size); err != nil {
		return 0, err
	}

	return size, nil
}

// decodeBody reads the contents of the input stream up to size
// and into p. It returns EOF or ErrUnexpectedEOF if it reads over
// size.
func (d *Decoder) decodeBody(p *Pattern, size int64) error {
	version := make([]byte, 32)
	if _, err := io.ReadFull(d, version); err != nil {
		return errors.New("unable to decode hw version: " + err.Error())
	}
	p.Version = strings.Trim(string(version), "\x00")

	if err := binary.Read(d, binary.LittleEndian, &p.Tempo); err != nil {
		return errors.New("unable to decode tempo: " + err.Error())
	}

	// version and tempo
	size -= (32 + 4)

	// decode the tracks
	for size >= 0 {
		t := Track{}
		if err := binary.Read(d, binary.BigEndian, &t.ID); err != nil {
			return err
		}

		var nameLen int32
		if err := binary.Read(d, binary.BigEndian, &nameLen); err != nil {
			return err
		}

		name := make([]byte, nameLen)
		if _, err := io.ReadFull(d, name); err != nil {
			return err
		}
		t.Name = string(name)

		steps := make([]byte, measureCount)
		if err := binary.Read(d, binary.BigEndian, steps); err != nil {
			return err
		}

		// convert steps into 'x' or '-' runes
		for idx, step := range steps {
			if step == 1 {
				steps[idx] = 'x'
			} else {
				steps[idx] = '-'
			}
		}
		t.Steps = steps

		p.Tracks = append(p.Tracks, t)
		// ID, nameLen, measureCount, and size of name
		size -= (1 + 4 + measureCount + int64(len(name)))
	}

	return nil
}
<|endoftext|>