<commit_before>package bdiscord

import (
	"github.com/42wim/matterbridge/bridge/config"
	log "github.com/Sirupsen/logrus"
	"github.com/bwmarrin/discordgo"
	"regexp"
	"strings"
	"sync"
)

type bdiscord struct {
	c             *discordgo.Session
	Config        *config.Protocol
	Remote        chan config.Message
	Account       string
	Channels      []*discordgo.Channel
	Nick          string
	UseChannelID  bool
	userMemberMap map[string]*discordgo.Member
	guildID       string
	sync.RWMutex
}

var flog *log.Entry
var protocol = "discord"

func init() {
	flog = log.WithFields(log.Fields{"module": protocol})
}

func New(cfg config.Protocol, account string, c chan config.Message) *bdiscord {
	b := &bdiscord{}
	b.Config = &cfg
	b.Remote = c
	b.Account = account
	b.userMemberMap = make(map[string]*discordgo.Member)
	return b
}

func (b *bdiscord) Connect() error {
	var err error
	flog.Info("Connecting")
	if !strings.HasPrefix(b.Config.Token, "Bot ") {
		b.Config.Token = "Bot " + b.Config.Token
	}
	b.c, err = discordgo.New(b.Config.Token)
	if err != nil {
		flog.Debugf("%#v", err)
		return err
	}
	flog.Info("Connection succeeded")
	b.c.AddHandler(b.messageCreate)
	b.c.AddHandler(b.memberUpdate)
	b.c.AddHandler(b.messageUpdate)
	err = b.c.Open()
	if err != nil {
		flog.Debugf("%#v", err)
		return err
	}
	guilds, err := b.c.UserGuilds()
	if err != nil {
		flog.Debugf("%#v", err)
		return err
	}
	userinfo, err := b.c.User("@me")
	if err != nil {
		flog.Debugf("%#v", err)
		return err
	}
	b.Nick = userinfo.Username
	for _, guild := range guilds {
		if guild.Name == b.Config.Server {
			b.Channels, err = b.c.GuildChannels(guild.ID)
			b.guildID = guild.ID
			if err != nil {
				flog.Debugf("%#v", err)
				return err
			}
		}
	}
	return nil
}

func (b *bdiscord) Disconnect() error {
	return nil
}

func (b *bdiscord) JoinChannel(channel string) error {
	idcheck := strings.Split(channel, "ID:")
	if len(idcheck) > 1 {
		b.UseChannelID = true
	}
	return nil
}

func (b *bdiscord) Send(msg config.Message) error {
	flog.Debugf("Receiving %#v", msg)
	channelID := b.getChannelID(msg.Channel)
	if channelID == "" {
		flog.Errorf("Could not find channelID for %v", msg.Channel)
		return nil
	}
	b.c.ChannelMessageSend(channelID, msg.Username+msg.Text)
	return nil
}

func (b *bdiscord) messageUpdate(s *discordgo.Session, m *discordgo.MessageUpdate) {
	if b.Config.EditDisable {
		return
	}
	// only when message is actually edited
	if m.Message.EditedTimestamp != "" {
		flog.Debugf("Sending edit message")
		m.Content = m.Content + b.Config.EditSuffix
		b.messageCreate(s, (*discordgo.MessageCreate)(m))
	}
}

func (b *bdiscord) messageCreate(s *discordgo.Session, m *discordgo.MessageCreate) {
	// not relay our own messages
	if m.Author.Username == b.Nick {
		return
	}
	if len(m.Attachments) > 0 {
		for _, attach := range m.Attachments {
			m.Content = m.Content + "\n" + attach.URL
		}
	}
	if m.Content == "" {
		return
	}
	flog.Debugf("Sending message from %s on %s to gateway", m.Author.Username, b.Account)
	channelName := b.getChannelName(m.ChannelID)
	if b.UseChannelID {
		channelName = "ID:" + m.ChannelID
	}
	username := b.getNick(m.Author)
	if len(m.MentionRoles) > 0 {
		m.Message.Content = b.replaceRoleMentions(m.Message.Content)
	}
	m.Message.Content = b.stripCustomoji(m.Message.Content)
	b.Remote <- config.Message{Username: username, Text: m.ContentWithMentionsReplaced(), Channel: channelName,
		Account: b.Account, Avatar: "https://cdn.discordapp.com/avatars/" + m.Author.ID + "/" + m.Author.Avatar + ".jpg"}
}

func (b *bdiscord) memberUpdate(s *discordgo.Session, m *discordgo.GuildMemberUpdate) {
	b.Lock()
	if _, ok := b.userMemberMap[m.Member.User.ID]; ok {
		flog.Debugf("%s: memberupdate: user %s (nick %s) changes nick to %s", b.Account, m.Member.User.Username, b.userMemberMap[m.Member.User.ID].Nick, m.Member.Nick)
	}
	b.userMemberMap[m.Member.User.ID] = m.Member
	b.Unlock()
}

func (b *bdiscord) getNick(user *discordgo.User) string {
	var err error
	b.Lock()
	defer b.Unlock()
	if _, ok := b.userMemberMap[user.ID]; ok {
		if b.userMemberMap[user.ID].Nick != "" {
			// only return if nick is set
			return b.userMemberMap[user.ID].Nick
		}
		// otherwise return username
		return user.Username
	}
	// if we didn't find nick, search for it
	b.userMemberMap[user.ID], err = b.c.GuildMember(b.guildID, user.ID)
	if err != nil {
		return user.Username
	}
	// only return if nick is set
	if b.userMemberMap[user.ID].Nick != "" {
		return b.userMemberMap[user.ID].Nick
	}
	return user.Username
}

func (b *bdiscord) getChannelID(name string) string {
	idcheck := strings.Split(name, "ID:")
	if len(idcheck) > 1 {
		return idcheck[1]
	}
	for _, channel := range b.Channels {
		if channel.Name == name {
			return channel.ID
		}
	}
	return ""
}

func (b *bdiscord) getChannelName(id string) string {
	for _, channel := range b.Channels {
		if channel.ID == id {
			return channel.Name
		}
	}
	return ""
}

func (b *bdiscord) replaceRoleMentions(text string) string {
	roles, err := b.c.GuildRoles(b.guildID)
	if err != nil {
		flog.Debugf("%#v", string(err.(*discordgo.RESTError).ResponseBody))
		return text
	}
	for _, role := range roles {
		text = strings.Replace(text, "<@&"+role.ID+">", "@"+role.Name, -1)
	}
	return text
}

func (b *bdiscord) stripCustomoji(text string) string {
	// <:doge:302803592035958784>
	re := regexp.MustCompile("<(:.*?:)[0-9]+>")
	return re.ReplaceAllString(text, `$1`)
}
<commit_msg>Fix possible crash on nil (discord)<commit_after>package bdiscord

import (
	"github.com/42wim/matterbridge/bridge/config"
	log "github.com/Sirupsen/logrus"
	"github.com/bwmarrin/discordgo"
	"regexp"
	"strings"
	"sync"
)

type bdiscord struct {
	c             *discordgo.Session
	Config        *config.Protocol
	Remote        chan config.Message
	Account       string
	Channels      []*discordgo.Channel
	Nick          string
	UseChannelID  bool
	userMemberMap map[string]*discordgo.Member
	guildID       string
	sync.RWMutex
}

var flog *log.Entry
var protocol = "discord"

func init() {
	flog = log.WithFields(log.Fields{"module": protocol})
}

func New(cfg config.Protocol, account string, c chan config.Message) *bdiscord {
	b := &bdiscord{}
	b.Config = &cfg
	b.Remote = c
	b.Account = account
	b.userMemberMap = make(map[string]*discordgo.Member)
	return b
}

func (b *bdiscord) Connect() error {
	var err error
	flog.Info("Connecting")
	if !strings.HasPrefix(b.Config.Token, "Bot ") {
		b.Config.Token = "Bot " + b.Config.Token
	}
	b.c, err = discordgo.New(b.Config.Token)
	if err != nil {
		flog.Debugf("%#v", err)
		return err
	}
	flog.Info("Connection succeeded")
	b.c.AddHandler(b.messageCreate)
	b.c.AddHandler(b.memberUpdate)
	b.c.AddHandler(b.messageUpdate)
	err = b.c.Open()
	if err != nil {
		flog.Debugf("%#v", err)
		return err
	}
	guilds, err := b.c.UserGuilds()
	if err != nil {
		flog.Debugf("%#v", err)
		return err
	}
	userinfo, err := b.c.User("@me")
	if err != nil {
		flog.Debugf("%#v", err)
		return err
	}
	b.Nick = userinfo.Username
	for _, guild := range guilds {
		if guild.Name == b.Config.Server {
			b.Channels, err = b.c.GuildChannels(guild.ID)
			b.guildID = guild.ID
			if err != nil {
				flog.Debugf("%#v", err)
				return err
			}
		}
	}
	return nil
}

func (b *bdiscord) Disconnect() error {
	return nil
}

func (b *bdiscord) JoinChannel(channel string) error {
	idcheck := strings.Split(channel, "ID:")
	if len(idcheck) > 1 {
		b.UseChannelID = true
	}
	return nil
}

func (b *bdiscord) Send(msg config.Message) error {
	flog.Debugf("Receiving %#v", msg)
	channelID := b.getChannelID(msg.Channel)
	if channelID == "" {
		flog.Errorf("Could not find channelID for %v", msg.Channel)
		return nil
	}
	b.c.ChannelMessageSend(channelID, msg.Username+msg.Text)
	return nil
}

func (b *bdiscord) messageUpdate(s *discordgo.Session, m *discordgo.MessageUpdate) {
	if b.Config.EditDisable {
		return
	}
	// only when message is actually edited
	if m.Message.EditedTimestamp != "" {
		flog.Debugf("Sending edit message")
		m.Content = m.Content + b.Config.EditSuffix
		b.messageCreate(s, (*discordgo.MessageCreate)(m))
	}
}

func (b *bdiscord) messageCreate(s *discordgo.Session, m *discordgo.MessageCreate) {
	// not relay our own messages
	if m.Author.Username == b.Nick {
		return
	}
	if len(m.Attachments) > 0 {
		for _, attach := range m.Attachments {
			m.Content = m.Content + "\n" + attach.URL
		}
	}
	if m.Content == "" {
		return
	}
	flog.Debugf("Sending message from %s on %s to gateway", m.Author.Username, b.Account)
	channelName := b.getChannelName(m.ChannelID)
	if b.UseChannelID {
		channelName = "ID:" + m.ChannelID
	}
	username := b.getNick(m.Author)
	if len(m.MentionRoles) > 0 {
		m.Message.Content = b.replaceRoleMentions(m.Message.Content)
	}
	m.Message.Content = b.stripCustomoji(m.Message.Content)
	b.Remote <- config.Message{Username: username, Text: m.ContentWithMentionsReplaced(), Channel: channelName,
		Account: b.Account, Avatar: "https://cdn.discordapp.com/avatars/" + m.Author.ID + "/" + m.Author.Avatar + ".jpg"}
}

func (b *bdiscord) memberUpdate(s *discordgo.Session, m *discordgo.GuildMemberUpdate) {
	b.Lock()
	if _, ok := b.userMemberMap[m.Member.User.ID]; ok {
		flog.Debugf("%s: memberupdate: user %s (nick %s) changes nick to %s", b.Account, m.Member.User.Username, b.userMemberMap[m.Member.User.ID].Nick, m.Member.Nick)
	}
	b.userMemberMap[m.Member.User.ID] = m.Member
	b.Unlock()
}

func (b *bdiscord) getNick(user *discordgo.User) string {
	var err error
	b.Lock()
	defer b.Unlock()
	if _, ok := b.userMemberMap[user.ID]; ok {
		if b.userMemberMap[user.ID] != nil {
			if b.userMemberMap[user.ID].Nick != "" {
				// only return if nick is set
				return b.userMemberMap[user.ID].Nick
			}
			// otherwise return username
			return user.Username
		}
	}
	// if we didn't find nick, search for it
	b.userMemberMap[user.ID], err = b.c.GuildMember(b.guildID, user.ID)
	if err != nil {
		return user.Username
	}
	// only return if nick is set
	if b.userMemberMap[user.ID].Nick != "" {
		return b.userMemberMap[user.ID].Nick
	}
	return user.Username
}

func (b *bdiscord) getChannelID(name string) string {
	idcheck := strings.Split(name, "ID:")
	if len(idcheck) > 1 {
		return idcheck[1]
	}
	for _, channel := range b.Channels {
		if channel.Name == name {
			return channel.ID
		}
	}
	return ""
}

func (b *bdiscord) getChannelName(id string) string {
	for _, channel := range b.Channels {
		if channel.ID == id {
			return channel.Name
		}
	}
	return ""
}

func (b *bdiscord) replaceRoleMentions(text string) string {
	roles, err := b.c.GuildRoles(b.guildID)
	if err != nil {
		flog.Debugf("%#v", string(err.(*discordgo.RESTError).ResponseBody))
		return text
	}
	for _, role := range roles {
		text = strings.Replace(text, "<@&"+role.ID+">", "@"+role.Name, -1)
	}
	return text
}

func (b *bdiscord) stripCustomoji(text string) string {
	// <:doge:302803592035958784>
	re := regexp.MustCompile("<(:.*?:)[0-9]+>")
	return re.ReplaceAllString(text, `$1`)
}
<|endoftext|>


<commit_before>package numgo

import (
	"fmt"
	"math"
	"runtime"
)

// Flatten reshapes the data to a 1-D array.
func (a *Array64) Flatten() *Array64 {
	if a.HasErr() {
		return a
	}
	return a.Reshape(a.strides[0])
}

// C will return a deep copy of the source array.
func (a *Array64) C() (b *Array64) {
	if a.HasErr() {
		return a
	}

	b = &Array64{
		shape:   make([]int, len(a.shape)),
		strides: make([]int, len(a.strides)),
		data:    make([]float64, a.strides[0]),
		err:     nil,
		debug:   "",
		stack:   "",
	}

	copy(b.shape, a.shape)
	copy(b.strides, a.strides)
	copy(b.data, a.data)
	return b
}

// Shape returns a copy of the array shape
func (a *Array64) Shape() []int {
	if a.HasErr() {
		return nil
	}

	res := make([]int, 0, len(a.shape))
	copy(res, a.shape)

	return res
}

// At returns the element at the given index.
// There should be one index per axis.  Generates a ShapeError if incorrect index.
func (a *Array64) At(index ...int) float64 {
	idx := a.valIdx(index, "At")
	if a.HasErr() {
		return math.NaN()
	}

	return a.data[idx]
}

func (a *Array64) at(index []int) float64 {
	var idx int
	for i, v := range index {
		idx += v * a.strides[i+1]
	}
	return a.data[idx]
}

func (a *Array64) valIdx(index []int, mthd string) (idx int) {
	if a.HasErr() {
		return 0
	}
	if len(index) > len(a.shape) {
		a.err = InvIndexError
		if debug {
			a.debug = fmt.Sprintf("Incorrect number of indicies received by %s().  Shape: %v  Index: %v", mthd, a.shape, index)
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return 0
	}
	for i, v := range index {
		if v >= a.shape[i] || v < 0 {
			a.err = IndexError
			if debug {
				a.debug = fmt.Sprintf("Index received by %s() does not exist shape: %v index: %v", mthd, a.shape, index)
				a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
			}
			return 0
		}
		idx += v * a.strides[i+1]
	}
	return
}

// SliceElement returns the element group at one axis above the leaf elements.
// Data is returned as a copy  in a float slice.
func (a *Array64) SliceElement(index ...int) (ret []float64) {
	idx := a.valIdx(index, "SliceElement")
	switch {
	case a.HasErr():
		return nil
	case len(a.shape)-1 != len(index):
		a.err = InvIndexError
		if debug {
			a.debug = fmt.Sprintf("Incorrect number of indicies received by SliceElement().  Shape: %v  Index: %v", a.shape, index)
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return nil
	}

	return append(ret, a.data[idx:idx+a.strides[len(a.strides)-2]]...)
}

// SubArr slices the array at a given index.
func (a *Array64) SubArr(index ...int) (ret *Array64) {
	idx := a.valIdx(index, "SubArr")
	if a.HasErr() {
		return a
	}

	ret = newArray64(a.shape[len(index):]...)
	copy(ret.data, a.data[idx:idx+a.strides[len(index)]])

	return
}

// Set sets the element at the given index.
// There should be one index per axis.  Generates a ShapeError if incorrect index.
func (a *Array64) Set(val float64, index ...int) *Array64 {
	idx := a.valIdx(index, "Set")
	if a.HasErr() {
		return a
	}

	a.data[idx] = val
	return a
}

// SetSliceElement sets the element group at one axis above the leaf elements.
// Source Array is returned, for function-chaining design.
func (a *Array64) SetSliceElement(vals []float64, index ...int) *Array64 {
	idx := a.valIdx(index, "SetSliceElement")
	switch {
	case a.HasErr():
		return a
	case len(a.shape)-1 != len(index):
		if debug {
			a.debug = fmt.Sprintf("Incorrect number of indicies received by SetSliceElement().  Shape: %v  Index: %v", a.shape, index)
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		fallthrough
	case len(vals) != a.shape[len(a.shape)-1]:
		a.err = InvIndexError
		if debug {
			a.debug = fmt.Sprintf("Incorrect slice length received by SetSliceElement().  Shape: %v  Index: %v", a.shape, len(index))
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return a
	}

	copy(a.data[idx:idx+a.strides[len(a.strides)-2]], vals[:a.strides[len(a.strides)-2]])
	return a
}

// SetSubArr sets the array below a given index to the values in vals.
// Values will be broadcast up multiple axes if the shapes match.
func (a *Array64) SetSubArr(vals *Array64, index ...int) *Array64 {
	idx := a.valIdx(index, "SetSubArr")
	switch {
	case a.HasErr():
		return a
	case vals.HasErr():
		a.err = vals.getErr()
		if debug {
			a.debug = "Array received by SetSubArr() is in error."
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return a
	case len(vals.shape)+len(index) > len(a.shape):
		a.err = InvIndexError
		if debug {
			a.debug = fmt.Sprintf("Array received by SetSubArr() cant be broadcast.  Shape: %v  Vals shape: %v index: %v", a.shape, vals.shape, index)
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return a
	}

	for i, j := len(a.shape)-1, len(vals.shape)-1; j >= 0; i, j = i-1, j-1 {
		if a.shape[i] != vals.shape[j] {
			a.err = ShapeError
			if debug {
				a.debug = fmt.Sprintf("Shape of array recieved by SetSubArr() doesn't match receiver.  Shape: %v  Vals Shape: %v", a.shape, vals.shape)
				a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
			}
			return a
		}
	}

	if len(a.shape)-len(index)-len(vals.shape) == 0 {
		copy(a.data[idx:idx+len(vals.data)], vals.data)
		return a
	}

	reps := 1
	for i := len(index); i < len(a.shape)-len(vals.shape); i++ {
		reps *= a.shape[i]
	}

	ln := len(vals.data)
	for i := 1; i <= reps; i++ {
		copy(a.data[idx+ln*(i-1):idx+ln*i], vals.data)
	}
	return a
}

// Resize will change the underlying array size.
//
// Make a copy C() if the original array needs to remain unchanged.
// Element location in the underlying slice will not be adjusted to the new shape.
func (a *Array64) Resize(shape ...int) *Array64 {
	switch {
	case a.HasErr():
		return a
	case len(shape) == 0:
		tmp := newArray64(0)
		a.shape, a.strides = tmp.shape, tmp.strides
		a.data = tmp.data
		return a
	}

	var sz int = 1
	for _, v := range shape {
		if v >= 0 {
			sz *= v
			continue
		}

		a.err = NegativeAxis
		if debug {
			a.debug = fmt.Sprintf("Negative axis length received by Resize.  Shape: %v", shape)
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return a
	}

	ln, cp := len(shape), cap(a.shape)
	if ln > cp {
		a.shape = append(a.shape[:cp], make([]int, ln-cp)...)
	} else {
		a.shape = a.shape[:ln]
	}

	ln, cp = ln+1, cap(a.strides)
	if ln > cp {
		a.strides = append(a.strides[:cp], make([]int, ln-cp)...)
	} else {
		a.strides = a.strides[:ln]
	}

	a.strides[ln-1] = 1
	for i := ln - 2; i >= 0; i-- {
		a.shape[i] = shape[i]
		a.strides[i] = a.shape[i] * a.strides[i+1]
	}

	cp = cap(a.data)
	if sz > cp {
		a.data = append(a.data[:cp], make([]float64, sz-cp)...)
	} else {
		a.data = a.data[:sz]
	}

	return a
}

// Append will concatenate a and val at the given axis.
//
// Source array will be changed, so use C() if the original data is needed.
// All axes must be the same except the appending axis.
func (a *Array64) Append(val *Array64, axis int) *Array64 {
	switch {
	case a.HasErr():
		return a
	case axis >= len(a.shape), axis < 0:
		a.err = IndexError
		if debug {
			a.debug = fmt.Sprintf("Axis received by Append() out of range.  Shape: %v  Axis: %v", a.shape, axis)
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return a
	case val.HasErr():
		a.err = val.GetErr()
		if debug {
			a.debug = "Array received by Append() is in error."
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return a
	case len(a.shape) != len(val.shape):
		a.err = ShapeError
		if debug {
			a.debug = fmt.Sprintf("Array received by Append() can not be matched.  Shape: %v  Val shape: %v", a.shape, val.shape)
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return a
	}

	for k, v := range a.shape {
		if v != val.shape[k] && k != axis {
			a.err = ShapeError
			if debug {
				a.debug = fmt.Sprintf("Array received by Append() can not be matched.  Shape: %v  Val shape: %v", a.shape, val.shape)
				a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
			}
			return a
		}
	}

	ln := len(a.data) + len(val.data)
	var dat []float64
	cp := cap(a.data)
	if ln > cp {
		dat = make([]float64, ln)
	} else {
		dat = a.data[:ln]
	}

	as, vs := a.strides[axis], val.strides[axis]
	for i, j := a.strides[0], val.strides[0]; i > 0; i, j = i-as, j-vs {
		copy(dat[i+j-vs:i+j], val.data[j-vs:j])
		copy(dat[i+j-as-vs:i+j-vs], a.data[i-as:i])
	}

	a.data = dat
	a.shape[axis] += val.shape[axis]

	for i := axis; i >= 0; i-- {
		a.strides[i] = a.strides[i+1] * a.shape[i]
	}

	return a
}
<commit_msg>Revert "for loop assignment to copy call"<commit_after>package numgo

import (
	"fmt"
	"math"
	"runtime"
)

// Flatten reshapes the data to a 1-D array.
func (a *Array64) Flatten() *Array64 {
	if a.HasErr() {
		return a
	}
	return a.Reshape(a.strides[0])
}

// C will return a deep copy of the source array.
func (a *Array64) C() (b *Array64) {
	if a.HasErr() {
		return a
	}

	b = &Array64{
		shape:   make([]int, len(a.shape)),
		strides: make([]int, len(a.strides)),
		data:    make([]float64, a.strides[0]),
		err:     nil,
		debug:   "",
		stack:   "",
	}

	copy(b.shape, a.shape)
	copy(b.strides, a.strides)
	copy(b.data, a.data)
	return b
}

// Shape returns a copy of the array shape
func (a *Array64) Shape() []int {
	if a.HasErr() {
		return nil
	}

	res := make([]int, 0, len(a.shape))
	for _, v := range a.shape {
		res = append(res, v)
	}

	return res
}

// At returns the element at the given index.
// There should be one index per axis.  Generates a ShapeError if incorrect index.
func (a *Array64) At(index ...int) float64 {
	idx := a.valIdx(index, "At")
	if a.HasErr() {
		return math.NaN()
	}

	return a.data[idx]
}

func (a *Array64) at(index []int) float64 {
	var idx int
	for i, v := range index {
		idx += v * a.strides[i+1]
	}
	return a.data[idx]
}

func (a *Array64) valIdx(index []int, mthd string) (idx int) {
	if a.HasErr() {
		return 0
	}
	if len(index) > len(a.shape) {
		a.err = InvIndexError
		if debug {
			a.debug = fmt.Sprintf("Incorrect number of indicies received by %s().  Shape: %v  Index: %v", mthd, a.shape, index)
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return 0
	}
	for i, v := range index {
		if v >= a.shape[i] || v < 0 {
			a.err = IndexError
			if debug {
				a.debug = fmt.Sprintf("Index received by %s() does not exist shape: %v index: %v", mthd, a.shape, index)
				a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
			}
			return 0
		}
		idx += v * a.strides[i+1]
	}
	return
}

// SliceElement returns the element group at one axis above the leaf elements.
// Data is returned as a copy  in a float slice.
func (a *Array64) SliceElement(index ...int) (ret []float64) {
	idx := a.valIdx(index, "SliceElement")
	switch {
	case a.HasErr():
		return nil
	case len(a.shape)-1 != len(index):
		a.err = InvIndexError
		if debug {
			a.debug = fmt.Sprintf("Incorrect number of indicies received by SliceElement().  Shape: %v  Index: %v", a.shape, index)
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return nil
	}

	return append(ret, a.data[idx:idx+a.strides[len(a.strides)-2]]...)
}

// SubArr slices the array at a given index.
func (a *Array64) SubArr(index ...int) (ret *Array64) {
	idx := a.valIdx(index, "SubArr")
	if a.HasErr() {
		return a
	}

	ret = newArray64(a.shape[len(index):]...)
	copy(ret.data, a.data[idx:idx+a.strides[len(index)]])

	return
}

// Set sets the element at the given index.
// There should be one index per axis.  Generates a ShapeError if incorrect index.
func (a *Array64) Set(val float64, index ...int) *Array64 {
	idx := a.valIdx(index, "Set")
	if a.HasErr() {
		return a
	}

	a.data[idx] = val
	return a
}

// SetSliceElement sets the element group at one axis above the leaf elements.
// Source Array is returned, for function-chaining design.
func (a *Array64) SetSliceElement(vals []float64, index ...int) *Array64 {
	idx := a.valIdx(index, "SetSliceElement")
	switch {
	case a.HasErr():
		return a
	case len(a.shape)-1 != len(index):
		if debug {
			a.debug = fmt.Sprintf("Incorrect number of indicies received by SetSliceElement().  Shape: %v  Index: %v", a.shape, index)
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		fallthrough
	case len(vals) != a.shape[len(a.shape)-1]:
		a.err = InvIndexError
		if debug {
			a.debug = fmt.Sprintf("Incorrect slice length received by SetSliceElement().  Shape: %v  Index: %v", a.shape, len(index))
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return a
	}

	copy(a.data[idx:idx+a.strides[len(a.strides)-2]], vals[:a.strides[len(a.strides)-2]])
	return a
}

// SetSubArr sets the array below a given index to the values in vals.
// Values will be broadcast up multiple axes if the shapes match.
func (a *Array64) SetSubArr(vals *Array64, index ...int) *Array64 {
	idx := a.valIdx(index, "SetSubArr")
	switch {
	case a.HasErr():
		return a
	case vals.HasErr():
		a.err = vals.getErr()
		if debug {
			a.debug = "Array received by SetSubArr() is in error."
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return a
	case len(vals.shape)+len(index) > len(a.shape):
		a.err = InvIndexError
		if debug {
			a.debug = fmt.Sprintf("Array received by SetSubArr() cant be broadcast.  Shape: %v  Vals shape: %v index: %v", a.shape, vals.shape, index)
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return a
	}

	for i, j := len(a.shape)-1, len(vals.shape)-1; j >= 0; i, j = i-1, j-1 {
		if a.shape[i] != vals.shape[j] {
			a.err = ShapeError
			if debug {
				a.debug = fmt.Sprintf("Shape of array recieved by SetSubArr() doesn't match receiver.  Shape: %v  Vals Shape: %v", a.shape, vals.shape)
				a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
			}
			return a
		}
	}

	if len(a.shape)-len(index)-len(vals.shape) == 0 {
		copy(a.data[idx:idx+len(vals.data)], vals.data)
		return a
	}

	reps := 1
	for i := len(index); i < len(a.shape)-len(vals.shape); i++ {
		reps *= a.shape[i]
	}

	ln := len(vals.data)
	for i := 1; i <= reps; i++ {
		copy(a.data[idx+ln*(i-1):idx+ln*i], vals.data)
	}
	return a
}

// Resize will change the underlying array size.
//
// Make a copy C() if the original array needs to remain unchanged.
// Element location in the underlying slice will not be adjusted to the new shape.
func (a *Array64) Resize(shape ...int) *Array64 {
	switch {
	case a.HasErr():
		return a
	case len(shape) == 0:
		tmp := newArray64(0)
		a.shape, a.strides = tmp.shape, tmp.strides
		a.data = tmp.data
		return a
	}

	var sz int = 1
	for _, v := range shape {
		if v >= 0 {
			sz *= v
			continue
		}

		a.err = NegativeAxis
		if debug {
			a.debug = fmt.Sprintf("Negative axis length received by Resize.  Shape: %v", shape)
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return a
	}

	ln, cp := len(shape), cap(a.shape)
	if ln > cp {
		a.shape = append(a.shape[:cp], make([]int, ln-cp)...)
	} else {
		a.shape = a.shape[:ln]
	}

	ln, cp = ln+1, cap(a.strides)
	if ln > cp {
		a.strides = append(a.strides[:cp], make([]int, ln-cp)...)
	} else {
		a.strides = a.strides[:ln]
	}

	a.strides[ln-1] = 1
	for i := ln - 2; i >= 0; i-- {
		a.shape[i] = shape[i]
		a.strides[i] = a.shape[i] * a.strides[i+1]
	}

	cp = cap(a.data)
	if sz > cp {
		a.data = append(a.data[:cp], make([]float64, sz-cp)...)
	} else {
		a.data = a.data[:sz]
	}

	return a
}

// Append will concatenate a and val at the given axis.
//
// Source array will be changed, so use C() if the original data is needed.
// All axes must be the same except the appending axis.
func (a *Array64) Append(val *Array64, axis int) *Array64 {
	switch {
	case a.HasErr():
		return a
	case axis >= len(a.shape), axis < 0:
		a.err = IndexError
		if debug {
			a.debug = fmt.Sprintf("Axis received by Append() out of range.  Shape: %v  Axis: %v", a.shape, axis)
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return a
	case val.HasErr():
		a.err = val.GetErr()
		if debug {
			a.debug = "Array received by Append() is in error."
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return a
	case len(a.shape) != len(val.shape):
		a.err = ShapeError
		if debug {
			a.debug = fmt.Sprintf("Array received by Append() can not be matched.  Shape: %v  Val shape: %v", a.shape, val.shape)
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return a
	}

	for k, v := range a.shape {
		if v != val.shape[k] && k != axis {
			a.err = ShapeError
			if debug {
				a.debug = fmt.Sprintf("Array received by Append() can not be matched.  Shape: %v  Val shape: %v", a.shape, val.shape)
				a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
			}
			return a
		}
	}

	ln := len(a.data) + len(val.data)
	var dat []float64
	cp := cap(a.data)
	if ln > cp {
		dat = make([]float64, ln)
	} else {
		dat = a.data[:ln]
	}

	as, vs := a.strides[axis], val.strides[axis]
	for i, j := a.strides[0], val.strides[0]; i > 0; i, j = i-as, j-vs {
		copy(dat[i+j-vs:i+j], val.data[j-vs:j])
		copy(dat[i+j-as-vs:i+j-vs], a.data[i-as:i])
	}

	a.data = dat
	a.shape[axis] += val.shape[axis]

	for i := axis; i >= 0; i-- {
		a.strides[i] = a.strides[i+1] * a.shape[i]
	}

	return a
}
<|endoftext|>


<commit_before>package tmdb

import (
	. "gopkg.in/check.v1"
)

func (s *TmdbSuite) TestConfiguration(c *C) {
	result, err := s.tmdb.Configuration()
	s.baseTest(&result, err, c)
	c.Assert(result.Images.BaseURL, Equals, "http://image.tmdb.org/t/p/")
	c.Assert(result.Images.SecureBaseURL, Equals, "https://image.tmdb.org/t/p/")
	c.Assert(len(result.Images.BackdropSizes), Equals, 4)
	c.Assert(len(result.ChangeKeys), Equals, 53)
}
<commit_msg>Switch assert to HasLen checker<commit_after>package tmdb

import (
	. "gopkg.in/check.v1"
)

func (s *TmdbSuite) TestConfiguration(c *C) {
	result, err := s.tmdb.Configuration()
	s.baseTest(&result, err, c)
	c.Assert(result.Images.BaseURL, Equals, "http://image.tmdb.org/t/p/")
	c.Assert(result.Images.SecureBaseURL, Equals, "https://image.tmdb.org/t/p/")
	c.Assert(result.Images.BackdropSizes, HasLen, 4)
	c.Assert(result.ChangeKeys, HasLen, 53)
}
<|endoftext|>


<commit_before>package main

import (
	"os"

	"github.com/codegangsta/cli"
	"github.com/miclle/brandy/action"
	"github.com/op/go-logging"
	// "gopkg.in/fsnotify.v1"
	// "gopkg.in/yaml.v2"
)

var version = "0.0.1-dev"

const usage = `Full stack build system.

Usage: brandy COMMAND [ARGS]

The most common brandy commands are:
 generate    Generate new code (short-cut alias: "g")
 server      Start the brandy server (short-cut alias: "s")
 new         Create a new brandy application. "brandy new my_app" creates a
             new application called MyApp in "./my_app"

In addition to those, there are:
 destroy      Undo code generated with "generate" (short-cut alias: "d")

All commands can be run with -h (or --help) for more information.

More info https://github.com/miclle/brandy
`

var log = logging.MustGetLogger("brandy")

// Example format string. Everything except the message has a custom color
// which is dependent on the log level. Many fields have a custom output
// formatting too, eg. the time returns the hour down to the milli second.
var format = logging.MustStringFormatter(
	`%{color}%{time:15:04:05.000} %{shortfunc} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}`,
)

func init() {
	// For demo purposes, create two backend for os.Stderr.
	backend1 := logging.NewLogBackend(os.Stderr, "", 0)
	backend2 := logging.NewLogBackend(os.Stderr, "", 0)

	// For messages written to backend2 we want to add some additional
	// information to the output, including the used log level and the name of
	// the function.
	backend2Formatter := logging.NewBackendFormatter(backend2, format)

	// Only errors and more severe messages should be sent to backend1
	backend1Leveled := logging.AddModuleLevel(backend1)
	backend1Leveled.SetLevel(logging.ERROR, "")

	// Set the backends to be used.
	logging.SetBackend(backend1Leveled, backend2Formatter)
}

func main() {

	app := cli.NewApp()
	app.Name = "brandy"
	app.Usage = usage
	app.Version = version
	app.Author = "Miclle Zheng"
	app.Email = "miclle.zheng@gmail.com"

	app.CommandNotFound = func(c *cli.Context, command string) {
		log.Errorf("Command %s does not exist.", command)
	}

	app.Before = startup

	app.Commands = commands()

	if err := app.Run(os.Args); err != nil {
		log.Error(err.Error())
		os.Exit(1)
	}
}

func startup(c *cli.Context) error {
	// TODO
	return nil
}

func commands() []cli.Command {
	return []cli.Command{
		{
			Name:  "about",
			Usage: "Learn about brandy",
			Action: func(c *cli.Context) {
				action.About()
			},
		},
		{
			Name:        "server",
			Usage:       "Start a static file server",
			Description: "Start the brandy server",
			Action: func(c *cli.Context) {
				// TODO
			},
		},
		{
			Name:  "build",
			Usage: "Build the project",
			Action: func(c *cli.Context) {
				// TODO
			},
		},
		{
			Name:  "init",
			Usage: "Initialize the configuration",
			Action: func(c *cli.Context) {
				// TODO
			},
		},
	}
}
<commit_msg>Update main init func<commit_after>package main

import (
	"os"

	"github.com/codegangsta/cli"
	"github.com/miclle/brandy/action"
	"github.com/op/go-logging"
	// "gopkg.in/fsnotify.v1"
	// "gopkg.in/yaml.v2"
)

var version = "0.0.1-dev"

const usage = `Full stack build system.

Usage: brandy COMMAND [ARGS]

The most common brandy commands are:
 generate    Generate new code (short-cut alias: "g")
 server      Start the brandy server (short-cut alias: "s")
 new         Create a new brandy application. "brandy new my_app" creates a
             new application called MyApp in "./my_app"

In addition to those, there are:
 destroy      Undo code generated with "generate" (short-cut alias: "d")

All commands can be run with -h (or --help) for more information.

More info https://github.com/miclle/brandy
`

var log = logging.MustGetLogger("brandy")

var format = logging.MustStringFormatter(`%{color}%{time:15:04:05.000} %{shortfunc} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}`)

func init() {
	backend1 := logging.NewLogBackend(os.Stderr, "", 0)
	backend2 := logging.NewLogBackend(os.Stderr, "", 0)
	backend2Formatter := logging.NewBackendFormatter(backend2, format)
	backend1Leveled := logging.AddModuleLevel(backend1)
	backend1Leveled.SetLevel(logging.ERROR, "")
	logging.SetBackend(backend1Leveled, backend2Formatter)
}

func main() {

	app := cli.NewApp()
	app.Name = "brandy"
	app.Usage = usage
	app.Version = version
	app.Author = "Miclle Zheng"
	app.Email = "miclle.zheng@gmail.com"

	app.CommandNotFound = func(c *cli.Context, command string) {
		log.Errorf("Command %s does not exist.", command)
	}

	app.Before = startup

	app.Commands = commands()

	if err := app.Run(os.Args); err != nil {
		log.Error(err.Error())
		os.Exit(1)
	}
}

func startup(c *cli.Context) error {
	// TODO
	return nil
}

func commands() []cli.Command {
	return []cli.Command{
		{
			Name:  "about",
			Usage: "Learn about brandy",
			Action: func(c *cli.Context) {
				action.About()
			},
		},
		{
			Name:        "server",
			Usage:       "Start a static file server",
			Description: "Start the brandy server",
			Action: func(c *cli.Context) {
				// TODO
			},
		},
		{
			Name:  "build",
			Usage: "Build the project",
			Action: func(c *cli.Context) {
				// TODO
			},
		},
		{
			Name:  "init",
			Usage: "Initialize the configuration",
			Action: func(c *cli.Context) {
				// TODO
			},
		},
	}
}
<|endoftext|>


<commit_before>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package exec

import (
	"context"
	"fmt"
	"io"
	"sync/atomic"
	"time"

	"github.com/apache/beam/sdks/go/pkg/beam/core/graph/coder"
	"github.com/apache/beam/sdks/go/pkg/beam/log"
)

// DataSource is a Root execution unit.
type DataSource struct {
	UID    UnitID
	Port   Port
	Target Target
	Coder  *coder.Coder
	Out    Node

	sid    StreamID
	source DataReader
	count  int64
	start  time.Time
}

func (n *DataSource) ID() UnitID {
	return n.UID
}

func (n *DataSource) Up(ctx context.Context) error {
	return nil
}

func (n *DataSource) StartBundle(ctx context.Context, id string, data DataManager) error {
	n.sid = StreamID{Port: n.Port, Target: n.Target, InstID: id}
	n.source = data
	n.start = time.Now()
	atomic.StoreInt64(&n.count, 0)
	return n.Out.StartBundle(ctx, id, data)
}

func (n *DataSource) Process(ctx context.Context) error {
	r, err := n.source.OpenRead(ctx, n.sid)
	if err != nil {
		return err
	}
	defer r.Close()

	c := coder.SkipW(n.Coder)
	switch {
	case coder.IsCoGBK(c):
		ck := MakeElementDecoder(c.Components[0])
		cv := MakeElementDecoder(c.Components[1])

		for {
			t, err := DecodeWindowedValueHeader(r)
			if err != nil {
				if err == io.EOF {
					return nil
				}
				return fmt.Errorf("source failed: %v", err)
			}

			// Decode key

			key, err := ck.Decode(r)
			if err != nil {
				return fmt.Errorf("source decode failed: %v", err)
			}
			key.Timestamp = t

			// TODO(herohde) 4/30/2017: the State API will be handle re-iterations
			// and only "small" value streams would be inline. Presumably, that
			// would entail buffering the whole stream. We do that for now.

			var buf []FullValue

			size, err := coder.DecodeInt32(r)
			if err != nil {
				return fmt.Errorf("stream size decoding failed: %v", err)
			}

			if size > -1 {
				// Single chunk stream.

				// log.Printf("Fixed size=%v", size)
				atomic.AddInt64(&n.count, int64(size))

				for i := int32(0); i < size; i++ {
					value, err := cv.Decode(r)
					if err != nil {
						return fmt.Errorf("stream value decode failed: %v", err)
					}
					buf = append(buf, value)
				}
			} else {
				// Multi-chunked stream.

				for {
					chunk, err := coder.DecodeVarUint64(r)
					if err != nil {
						return fmt.Errorf("stream chunk size decoding failed: %v", err)
					}

					// log.Printf("Chunk size=%v", chunk)

					if chunk == 0 {
						break
					}

					atomic.AddInt64(&n.count, int64(chunk))
					for i := uint64(0); i < chunk; i++ {
						value, err := cv.Decode(r)
						if err != nil {
							return fmt.Errorf("stream value decode failed: %v", err)
						}
						buf = append(buf, value)
					}
				}
			}

			values := &FixedReStream{Buf: buf}
			if err := n.Out.ProcessElement(ctx, key, values); err != nil {
				return err
			}
		}

	default:
		ec := MakeElementDecoder(c)

		for {
			atomic.AddInt64(&n.count, 1)
			t, err := DecodeWindowedValueHeader(r)
			if err != nil {
				if err == io.EOF {
					return nil
				}
				return fmt.Errorf("source failed: %v", err)
			}

			elm, err := ec.Decode(r)
			if err != nil {
				return fmt.Errorf("source decode failed: %v", err)
			}
			elm.Timestamp = t

			// log.Printf("READ: %v %v", elm.Key.Type(), elm.Key.Interface())

			if err := n.Out.ProcessElement(ctx, elm); err != nil {
				return err
			}
		}
	}
}

func (n *DataSource) FinishBundle(ctx context.Context) error {
	log.Infof(context.Background(), "DataSource: %d elements in %d ns", n.count, time.Now().Sub(n.start))
	n.sid = StreamID{}
	n.source = nil
	return n.Out.FinishBundle(ctx)
}

func (n *DataSource) Down(ctx context.Context) error {
	n.sid = StreamID{}
	n.source = nil
	return nil
}

func (n *DataSource) String() string {
	sid := StreamID{Port: n.Port, Target: n.Target}
	return fmt.Sprintf("DataSource[%v] Out:%v", sid, n.Out.ID())
}

// ProgressReportSnapshot captures the progress reading an input source.
type ProgressReportSnapshot struct {
	ID, Name string
	Count    int64
}

// Progress returns a snapshot of the source's progress.
func (n *DataSource) Progress() ProgressReportSnapshot {
	if n == nil {
		return ProgressReportSnapshot{}
	}
	return ProgressReportSnapshot{n.sid.Target.ID, n.sid.Target.Name, atomic.LoadInt64(&n.count)}
}
<commit_msg>Fix incorrect read of atomic counter.<commit_after>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package exec

import (
	"context"
	"fmt"
	"io"
	"sync/atomic"
	"time"

	"github.com/apache/beam/sdks/go/pkg/beam/core/graph/coder"
	"github.com/apache/beam/sdks/go/pkg/beam/log"
)

// DataSource is a Root execution unit.
type DataSource struct {
	UID    UnitID
	Port   Port
	Target Target
	Coder  *coder.Coder
	Out    Node

	sid    StreamID
	source DataReader
	count  int64
	start  time.Time
}

func (n *DataSource) ID() UnitID {
	return n.UID
}

func (n *DataSource) Up(ctx context.Context) error {
	return nil
}

func (n *DataSource) StartBundle(ctx context.Context, id string, data DataManager) error {
	n.sid = StreamID{Port: n.Port, Target: n.Target, InstID: id}
	n.source = data
	n.start = time.Now()
	atomic.StoreInt64(&n.count, 0)
	return n.Out.StartBundle(ctx, id, data)
}

func (n *DataSource) Process(ctx context.Context) error {
	r, err := n.source.OpenRead(ctx, n.sid)
	if err != nil {
		return err
	}
	defer r.Close()

	c := coder.SkipW(n.Coder)
	switch {
	case coder.IsCoGBK(c):
		ck := MakeElementDecoder(c.Components[0])
		cv := MakeElementDecoder(c.Components[1])

		for {
			t, err := DecodeWindowedValueHeader(r)
			if err != nil {
				if err == io.EOF {
					return nil
				}
				return fmt.Errorf("source failed: %v", err)
			}

			// Decode key

			key, err := ck.Decode(r)
			if err != nil {
				return fmt.Errorf("source decode failed: %v", err)
			}
			key.Timestamp = t

			// TODO(herohde) 4/30/2017: the State API will be handle re-iterations
			// and only "small" value streams would be inline. Presumably, that
			// would entail buffering the whole stream. We do that for now.

			var buf []FullValue

			size, err := coder.DecodeInt32(r)
			if err != nil {
				return fmt.Errorf("stream size decoding failed: %v", err)
			}

			if size > -1 {
				// Single chunk stream.

				// log.Printf("Fixed size=%v", size)
				atomic.AddInt64(&n.count, int64(size))

				for i := int32(0); i < size; i++ {
					value, err := cv.Decode(r)
					if err != nil {
						return fmt.Errorf("stream value decode failed: %v", err)
					}
					buf = append(buf, value)
				}
			} else {
				// Multi-chunked stream.

				for {
					chunk, err := coder.DecodeVarUint64(r)
					if err != nil {
						return fmt.Errorf("stream chunk size decoding failed: %v", err)
					}

					// log.Printf("Chunk size=%v", chunk)

					if chunk == 0 {
						break
					}

					atomic.AddInt64(&n.count, int64(chunk))
					for i := uint64(0); i < chunk; i++ {
						value, err := cv.Decode(r)
						if err != nil {
							return fmt.Errorf("stream value decode failed: %v", err)
						}
						buf = append(buf, value)
					}
				}
			}

			values := &FixedReStream{Buf: buf}
			if err := n.Out.ProcessElement(ctx, key, values); err != nil {
				return err
			}
		}

	default:
		ec := MakeElementDecoder(c)

		for {
			atomic.AddInt64(&n.count, 1)
			t, err := DecodeWindowedValueHeader(r)
			if err != nil {
				if err == io.EOF {
					return nil
				}
				return fmt.Errorf("source failed: %v", err)
			}

			elm, err := ec.Decode(r)
			if err != nil {
				return fmt.Errorf("source decode failed: %v", err)
			}
			elm.Timestamp = t

			// log.Printf("READ: %v %v", elm.Key.Type(), elm.Key.Interface())

			if err := n.Out.ProcessElement(ctx, elm); err != nil {
				return err
			}
		}
	}
}

func (n *DataSource) FinishBundle(ctx context.Context) error {
	log.Infof(context.Background(), "DataSource: %d elements in %d ns", atomic.LoadInt64(&n.count), time.Now().Sub(n.start))
	n.sid = StreamID{}
	n.source = nil
	return n.Out.FinishBundle(ctx)
}

func (n *DataSource) Down(ctx context.Context) error {
	n.sid = StreamID{}
	n.source = nil
	return nil
}

func (n *DataSource) String() string {
	sid := StreamID{Port: n.Port, Target: n.Target}
	return fmt.Sprintf("DataSource[%v] Out:%v", sid, n.Out.ID())
}

// ProgressReportSnapshot captures the progress reading an input source.
type ProgressReportSnapshot struct {
	ID, Name string
	Count    int64
}

// Progress returns a snapshot of the source's progress.
func (n *DataSource) Progress() ProgressReportSnapshot {
	if n == nil {
		return ProgressReportSnapshot{}
	}
	return ProgressReportSnapshot{n.sid.Target.ID, n.sid.Target.Name, atomic.LoadInt64(&n.count)}
}
<|endoftext|>


<commit_before>package common

import (
	"bytes"
	"fmt"
	"os"
	"text/template"

	"github.com/Masterminds/sprig"
	"github.com/flant/dapp/pkg/config"
	"github.com/flant/dapp/pkg/slug"
)

func GetHelmRelease(releaseOption string, environmentOption string, dappfile *config.Dappfile) (string, error) {
	if releaseOption != "" {
		err := slug.ValidateHelmRelease(releaseOption)
		if err != nil {
			return "", fmt.Errorf("bad Helm release specified '%s': %s", releaseOption, err)
		}
		return releaseOption, nil
	}

	releaseTemplate := dappfile.Meta.DeployTemplates.HelmRelease
	if releaseTemplate == "" {
		releaseTemplate = "[[ project ]]-[[ environment ]]"
	}

	renderedRelease, err := renderDeployParamTemplate("release", releaseTemplate, environmentOption, dappfile)
	if err != nil {
		return "", fmt.Errorf("cannot render Helm release name by template '%s': %s", releaseTemplate, err)
	}

	if dappfile.Meta.DeployTemplates.HelmReleaseSlug {
		return slug.HelmRelease(renderedRelease), nil
	}

	err = slug.ValidateHelmRelease(renderedRelease)
	if err != nil {
		return "", fmt.Errorf("bad Helm release '%s' rendered by template '%s': %s", renderedRelease, releaseTemplate, err)
	}

	return renderedRelease, nil
}

func GetKubernetesNamespace(namespaceOption string, environmentOption string, dappfile *config.Dappfile) (string, error) {
	if namespaceOption != "" {
		err := slug.ValidateKubernetesNamespace(namespaceOption)
		if err != nil {
			return "", fmt.Errorf("bad Kubernetes namespace specified '%s': %s", namespaceOption, err)
		}
		return namespaceOption, nil
	}

	namespaceTemplate := dappfile.Meta.DeployTemplates.KubernetesNamespace
	if namespaceTemplate == "" {
		namespaceTemplate = "[[ project ]]-[[ environment ]]"
	}

	renderedNamespace, err := renderDeployParamTemplate("namespace", namespaceTemplate, environmentOption, dappfile)
	if err != nil {
		return "", fmt.Errorf("cannot render Kubernetes namespace by template '%s': %s", namespaceTemplate, err)
	}

	if dappfile.Meta.DeployTemplates.KubernetesNamespaceSlug {
		return slug.KubernetesNamespace(renderedNamespace), nil
	}

	err = slug.ValidateKubernetesNamespace(renderedNamespace)
	if err != nil {
		return "", fmt.Errorf("bad Kubernetes namespace '%s' rendered by template '%s': %s", renderedNamespace, namespaceTemplate, err)
	}

	return renderedNamespace, nil
}

func renderDeployParamTemplate(templateName, templateText string, environmentOption string, dappfile *config.Dappfile) (string, error) {
	tmpl := template.New(templateName).Delims("[[", "]]")

	funcMap := sprig.TxtFuncMap()

	funcMap["project"] = func() string {
		return dappfile.Meta.Project
	}

	funcMap["environment"] = func() (string, error) {
		environment := os.Getenv("CI_ENVIRONMENT_SLUG")

		if environment == "" {
			environment = environmentOption
		}

		if environment == "" {
			return "", fmt.Errorf("--environment option or CI_ENVIRONMENT_SLUG variable required to construct name by template '%s'", templateText)
		}

		return environment, nil
	}

	tmpl = tmpl.Funcs(template.FuncMap(funcMap))

	tmpl, err := tmpl.Parse(templateText)
	if err != nil {
		return "", fmt.Errorf("bad template: %s", err)
	}

	buf := bytes.NewBuffer(nil)
	if err := tmpl.ExecuteTemplate(buf, templateName, nil); err != nil {
		return "", err
	}

	return buf.String(), nil
}
<commit_msg>Validate empty rendered deploy params<commit_after>package common

import (
	"bytes"
	"fmt"
	"os"
	"text/template"

	"github.com/Masterminds/sprig"
	"github.com/flant/dapp/pkg/config"
	"github.com/flant/dapp/pkg/slug"
)

func GetHelmRelease(releaseOption string, environmentOption string, dappfile *config.Dappfile) (string, error) {
	if releaseOption != "" {
		err := slug.ValidateHelmRelease(releaseOption)
		if err != nil {
			return "", fmt.Errorf("bad Helm release specified '%s': %s", releaseOption, err)
		}
		return releaseOption, nil
	}

	releaseTemplate := dappfile.Meta.DeployTemplates.HelmRelease
	if releaseTemplate == "" {
		releaseTemplate = "[[ project ]]-[[ environment ]]"
	}

	renderedRelease, err := renderDeployParamTemplate("release", releaseTemplate, environmentOption, dappfile)
	if err != nil {
		return "", fmt.Errorf("cannot render Helm release name by template '%s': %s", releaseTemplate, err)
	}

	if renderedRelease == "" {
		return "", fmt.Errorf("Helm release rendered by template '%s' is empty: release name cannot be empty", releaseTemplate)
	}

	if dappfile.Meta.DeployTemplates.HelmReleaseSlug {
		return slug.HelmRelease(renderedRelease), nil
	}

	err = slug.ValidateHelmRelease(renderedRelease)
	if err != nil {
		return "", fmt.Errorf("bad Helm release '%s' rendered by template '%s': %s", renderedRelease, releaseTemplate, err)
	}

	return renderedRelease, nil
}

func GetKubernetesNamespace(namespaceOption string, environmentOption string, dappfile *config.Dappfile) (string, error) {
	if namespaceOption != "" {
		err := slug.ValidateKubernetesNamespace(namespaceOption)
		if err != nil {
			return "", fmt.Errorf("bad Kubernetes namespace specified '%s': %s", namespaceOption, err)
		}
		return namespaceOption, nil
	}

	namespaceTemplate := dappfile.Meta.DeployTemplates.KubernetesNamespace
	if namespaceTemplate == "" {
		namespaceTemplate = "[[ project ]]-[[ environment ]]"
	}

	renderedNamespace, err := renderDeployParamTemplate("namespace", namespaceTemplate, environmentOption, dappfile)
	if err != nil {
		return "", fmt.Errorf("cannot render Kubernetes namespace by template '%s': %s", namespaceTemplate, err)
	}

	if renderedNamespace == "" {
		return "", fmt.Errorf("Kubernetes namespace rendered by template '%s' is empty: namespace cannot be empty", namespaceTemplate)
	}

	if dappfile.Meta.DeployTemplates.KubernetesNamespaceSlug {
		return slug.KubernetesNamespace(renderedNamespace), nil
	}

	err = slug.ValidateKubernetesNamespace(renderedNamespace)
	if err != nil {
		return "", fmt.Errorf("bad Kubernetes namespace '%s' rendered by template '%s': %s", renderedNamespace, namespaceTemplate, err)
	}

	return renderedNamespace, nil
}

func renderDeployParamTemplate(templateName, templateText string, environmentOption string, dappfile *config.Dappfile) (string, error) {
	tmpl := template.New(templateName).Delims("[[", "]]")

	funcMap := sprig.TxtFuncMap()

	funcMap["project"] = func() string {
		return dappfile.Meta.Project
	}

	funcMap["environment"] = func() (string, error) {
		environment := os.Getenv("CI_ENVIRONMENT_SLUG")

		if environment == "" {
			environment = environmentOption
		}

		if environment == "" {
			return "", fmt.Errorf("--environment option or CI_ENVIRONMENT_SLUG variable required to construct name by template '%s'", templateText)
		}

		return environment, nil
	}

	tmpl = tmpl.Funcs(template.FuncMap(funcMap))

	tmpl, err := tmpl.Parse(templateText)
	if err != nil {
		return "", fmt.Errorf("bad template: %s", err)
	}

	buf := bytes.NewBuffer(nil)
	if err := tmpl.ExecuteTemplate(buf, templateName, nil); err != nil {
		return "", err
	}

	return buf.String(), nil
}
<|endoftext|>


<commit_before>package client

import (
	"github.com/ccding/go-logging/logging"
	"github.com/gtfierro/cs262-project/common"
	"github.com/pkg/errors"
	uuidlib "github.com/satori/go.uuid"
	"github.com/tinylib/msgp/msgp"
	"io"
	"net"
	"os"
	"sync"
	"time"
)

var log *logging.Logger
var NamespaceUUID = uuidlib.FromStringOrNil("85ce106e-0ccf-11e6-81fc-0cc47a0f7eea")

func init() {
	log, _ = logging.WriterLogger("main", logging.DEBUG, logging.BasicFormat, logging.DefaultTimeFormat, os.Stderr, true)
}

// Creates a deterministic UUID from a given name. Names are easier to remember
// than UUIDs, so this should make writing scripts easier
func UUIDFromName(name string) common.UUID {
	return common.UUID(uuidlib.NewV5(NamespaceUUID, name).String())
}

// configuration for a client
type Config struct {
	// ip:port of the initial, local broker
	BrokerAddress string
	// ip:port of the coordinator
	CoordinatorAddress string
	// the client identifier. Must be Unique!
	ID common.UUID
}

type Client struct {
	// unique client identifier
	ID common.UUID

	// Handling the connection to the local broker
	// the IP:Port of the local broker we talk to
	BrokerAddress    *net.TCPAddr
	brokerConn       *net.TCPConn
	brokerEncoder    *msgp.Writer
	brokerEncodeLock sync.Mutex

	// the IP:Port of the coordinator that we fall back to
	CoordinatorAddress *net.TCPAddr
	coordConn          *net.TCPConn
	coordEncoder       *msgp.Writer
	coordEncodeLock    sync.Mutex

	// if true, then publishHandler is non-null
	hasPublishHandler bool
	// attach a publish handler using AttachPublishHandler
	publishHandler func(m *common.PublishMessage)
	publishersLock sync.RWMutex
	publishers     map[common.UUID]*Publisher

	brokerDead bool

	// client signals on this channel when it is done
	Stop chan bool

	// the query this client is subscribed to
	query string
}

// Creates a new client with the given configuration
func NewClient(cfg *Config) (*Client, error) {
	var err error
	c := &Client{
		ID:                cfg.ID,
		Stop:              make(chan bool),
		hasPublishHandler: false,
		publishers:        make(map[common.UUID]*Publisher),
		brokerDead:        true,
	}
	c.BrokerAddress, err = net.ResolveTCPAddr("tcp", cfg.BrokerAddress)
	if err != nil {
		return c, errors.Wrap(err, "Could not resolve local broker address")
	}
	if err = c.connectBroker(c.BrokerAddress); err != nil {
		return c, errors.Wrap(err, "Could not connect to local broker")
	}

	c.CoordinatorAddress, err = net.ResolveTCPAddr("tcp", cfg.CoordinatorAddress)
	if err != nil {
		return c, errors.Wrap(err, "Could not resolve coordinator address")
	}

	// start listening for messages from the broker
	go c.listen()

	return c, nil
}

// This function is called whenever the client receives a published message
func (c *Client) AttachPublishHandler(f func(m *common.PublishMessage)) {
	c.hasPublishHandler = true
	c.publishHandler = f
}

// This should be triggered when we can no longer contact our local broker. In this
// case, we sent a BrokerRequestMessage to the coordinator
func (c *Client) doFailover() {
	c.brokerDead = true
	// establish the coordinator connection
	c.connectCoordinator()
	// prepare the BrokerRequestMessage
	brm := &common.BrokerRequestMessage{
		LocalBrokerAddr: c.BrokerAddress.String(),
		IsPublisher:     false,
		UUID:            "392c1b18-0c37-11e6-b352-1002b58053c7",
	}
	// loop until we can contact the coordinator
	err := c.sendCoordinator(brm)
	for err != nil {
		time.Sleep(1)
		err = c.sendCoordinator(brm)
	}
}

func (c *Client) connectBroker(address *net.TCPAddr) error {
	var err error
	if c.brokerConn, err = net.DialTCP("tcp", nil, address); err != nil {
		return errors.Wrap(err, "Could not dial broker")
	}
	c.brokerEncodeLock.Lock()
	c.brokerEncoder = msgp.NewWriter(c.brokerConn)
	c.brokerEncodeLock.Unlock()
	c.brokerDead = false
	return nil
}

// Loop until we can finally connect to the coordinator.
// This blocks indefinitely until it is successful
func (c *Client) connectCoordinator() {
	var (
		err      error
		waitTime = 1 * time.Second
		maxWait  = 30 * time.Second
	)
	c.coordConn, err = net.DialTCP("tcp", nil, c.CoordinatorAddress)
	for err != nil {
		log.Warningf("Retrying coordinator connection to %v with delay %v", c.CoordinatorAddress, waitTime)
		time.Sleep(waitTime)
		waitTime *= 2
		if waitTime > maxWait {
			waitTime = maxWait
		}
		c.coordConn, err = net.DialTCP("tcp", nil, c.CoordinatorAddress)
	}
	log.Debug("Connected to coordinator")
	go c.listenCoordinator()
	c.coordEncoder = msgp.NewWriter(c.coordConn)
}

//TODO: implement
// This function should contact the coordinator to get the new broker
func (c *Client) configureNewBroker(m *common.BrokerAssignmentMessage) {
	var err error
	c.BrokerAddress, err = net.ResolveTCPAddr("tcp", m.ClientBrokerAddr)
	if err != nil {
		log.Critical(errors.Wrap(err, "Could not resolve local broker address"))
	}
	if err = c.connectBroker(c.BrokerAddress); err != nil {
		log.Critical(errors.Wrap(err, "Could not connect to local broker"))
	}
	// send our subscription once we're back
	c.resubscribe()
	go c.listen()
}

// Sends message to the currently configured broker
func (c *Client) sendBroker(m common.Sendable) error {
	c.brokerEncodeLock.Lock()
	defer c.brokerEncodeLock.Unlock()
	if err := m.Encode(c.brokerEncoder); err != nil {
		return errors.Wrap(err, "Could not encode message")
	}
	if err := c.brokerEncoder.Flush(); err != nil {
		// do failover if we fail to send
		go c.doFailover()
		return errors.Wrap(err, "Could not send message to broker")
	}
	return nil
}

func (c *Client) sendCoordinator(m common.Sendable) error {
	c.coordEncodeLock.Lock()
	if err := m.Encode(c.coordEncoder); err != nil {
		c.coordEncodeLock.Unlock()
		return errors.Wrap(err, "Could not encode message")
	}
	if err := c.coordEncoder.Flush(); err != nil {
		c.coordEncodeLock.Unlock()
		c.connectCoordinator()
		return errors.Wrap(err, "Could not send message to coordinator")
	}
	c.coordEncodeLock.Unlock()
	return nil
}

func (c *Client) listen() {
	reader := msgp.NewReader(net.Conn(c.brokerConn))
	for {
		if c.brokerDead {
			return
		}
		msg, err := common.MessageFromDecoderMsgp(reader)
		if err == io.EOF {
			log.Warn("connection closed. Do failover")
			c.doFailover()
			break
		}
		if err != nil {
			log.Warn(errors.Wrap(err, "Could not decode message"))
		}

		switch m := msg.(type) {
		case *common.PublishMessage:
			if c.hasPublishHandler {
				c.publishHandler(m)
			} else {
				log.Infof("Got publish message %v", m)
			}
		default:
			log.Infof("Got %T message %v", m, m)
		}
	}
}

func (c *Client) listenCoordinator() {
	if c.coordConn == nil {
		return
	}
	reader := msgp.NewReader(net.Conn(c.coordConn))
	for {
		msg, err := common.MessageFromDecoderMsgp(reader)
		if err == io.EOF {
			log.Warn("connection closed. Do failover")
			c.doFailover()
			break
		}
		if err != nil {
			log.Warn(errors.Wrap(err, "Could not decode message"))
		}

		log.Infof("Got %T message %v from coordinator", msg, msg)
		switch m := msg.(type) {
		case *common.BrokerAssignmentMessage:
			c.configureNewBroker(m)
		default:
			log.Infof("Got %T message %v from coordinator", m, m)
		}
	}
}

// after the duration expires, stop the client by signalling on c.Stop
func (c *Client) StopIn(d time.Duration) {
	go func(c *Client) {
		time.Sleep(d)
		c.Stop <- true
	}(c)
}

// subscribes the client to the given query via the broker specified in the
// client's configuration (or whatever next broker if the client has experienced
// a failover). Use AttachPublishHandler to do special handling of the received
// published messages
func (c *Client) Subscribe(query string) {
	// cache the query we are subscribing to
	c.query = query
	msg := &common.QueryMessage{
		UUID:  c.ID,
		Query: c.query,
	}
	if err := c.sendBroker(msg); err != nil {
		log.Errorf("Error? %v", errors.Cause(err))
	}
}

// call to resend subscription
func (c *Client) resubscribe() {
	c.Subscribe(c.query)
}

// Called externally, this adds new metadata to this client if it is acting
// as a publisher. If the client is not a publisher, this function runs but does
// not affect any part of the subscription operation.
// To DELETE a metadata key, use a value of nil for the key you want to delete. It
// will get folded into the next publish message, and then the keys will be removed
// from the local metadata map
func (c *Client) AddMetadata(pubid common.UUID, newm map[string]interface{}) {
	if len(newm) == 0 {
		return
	}
	c.publishersLock.Lock()
	if pub, found := c.publishers[pubid]; found {
		pub.AddMetadata(newm)
	}
	c.publishersLock.Unlock()
}

func (c *Client) AddPublisher(id common.UUID) *Publisher {
	pub := NewPublisher(id, c.BrokerAddress, c.CoordinatorAddress)
	c.publishersLock.Lock()
	c.publishers[id] = pub
	c.publishersLock.Unlock()
	return pub
}
<commit_msg>add handler for subscriptiondiff<commit_after>package client

import (
	"github.com/ccding/go-logging/logging"
	"github.com/gtfierro/cs262-project/common"
	"github.com/pkg/errors"
	uuidlib "github.com/satori/go.uuid"
	"github.com/tinylib/msgp/msgp"
	"io"
	"net"
	"os"
	"sync"
	"time"
)

var log *logging.Logger
var NamespaceUUID = uuidlib.FromStringOrNil("85ce106e-0ccf-11e6-81fc-0cc47a0f7eea")

func init() {
	log, _ = logging.WriterLogger("main", logging.DEBUG, logging.BasicFormat, logging.DefaultTimeFormat, os.Stderr, true)
}

// Creates a deterministic UUID from a given name. Names are easier to remember
// than UUIDs, so this should make writing scripts easier
func UUIDFromName(name string) common.UUID {
	return common.UUID(uuidlib.NewV5(NamespaceUUID, name).String())
}

// configuration for a client
type Config struct {
	// ip:port of the initial, local broker
	BrokerAddress string
	// ip:port of the coordinator
	CoordinatorAddress string
	// the client identifier. Must be Unique!
	ID common.UUID
}

type Client struct {
	// unique client identifier
	ID common.UUID

	// Handling the connection to the local broker
	// the IP:Port of the local broker we talk to
	BrokerAddress    *net.TCPAddr
	brokerConn       *net.TCPConn
	brokerEncoder    *msgp.Writer
	brokerEncodeLock sync.Mutex

	// the IP:Port of the coordinator that we fall back to
	CoordinatorAddress *net.TCPAddr
	coordConn          *net.TCPConn
	coordEncoder       *msgp.Writer
	coordEncodeLock    sync.Mutex

	// if true, then publishHandler is non-null
	hasPublishHandler bool
	// attach a publish handler using AttachPublishHandler
	publishHandler func(m *common.PublishMessage)
	publishersLock sync.RWMutex
	publishers     map[common.UUID]*Publisher

	hasDiffHandler bool
	diffHandler    func(m *common.SubscriptionDiffMessage)

	brokerDead bool

	// client signals on this channel when it is done
	Stop chan bool

	// the query this client is subscribed to
	query string
}

// Creates a new client with the given configuration
func NewClient(cfg *Config) (*Client, error) {
	var err error
	c := &Client{
		ID:                cfg.ID,
		Stop:              make(chan bool),
		hasPublishHandler: false,
		publishers:        make(map[common.UUID]*Publisher),
		hasDiffHandler:    false,
		brokerDead:        true,
	}
	c.BrokerAddress, err = net.ResolveTCPAddr("tcp", cfg.BrokerAddress)
	if err != nil {
		return c, errors.Wrap(err, "Could not resolve local broker address")
	}
	if err = c.connectBroker(c.BrokerAddress); err != nil {
		return c, errors.Wrap(err, "Could not connect to local broker")
	}

	c.CoordinatorAddress, err = net.ResolveTCPAddr("tcp", cfg.CoordinatorAddress)
	if err != nil {
		return c, errors.Wrap(err, "Could not resolve coordinator address")
	}

	// start listening for messages from the broker
	go c.listen()

	return c, nil
}

// This function is called whenever the client receives a published message
func (c *Client) AttachPublishHandler(f func(m *common.PublishMessage)) {
	c.hasPublishHandler = true
	c.publishHandler = f
}

func (c *Client) AttachDiffHandler(f func(m *common.SubscriptionDiffMessage)) {
	c.hasDiffHandler = true
	c.diffHandler = f
}

// This should be triggered when we can no longer contact our local broker. In this
// case, we sent a BrokerRequestMessage to the coordinator
func (c *Client) doFailover() {
	c.brokerDead = true
	// establish the coordinator connection
	c.connectCoordinator()
	// prepare the BrokerRequestMessage
	brm := &common.BrokerRequestMessage{
		LocalBrokerAddr: c.BrokerAddress.String(),
		IsPublisher:     false,
		UUID:            "392c1b18-0c37-11e6-b352-1002b58053c7",
	}
	// loop until we can contact the coordinator
	err := c.sendCoordinator(brm)
	for err != nil {
		time.Sleep(1)
		err = c.sendCoordinator(brm)
	}
}

func (c *Client) connectBroker(address *net.TCPAddr) error {
	var err error
	if c.brokerConn, err = net.DialTCP("tcp", nil, address); err != nil {
		return errors.Wrap(err, "Could not dial broker")
	}
	c.brokerEncodeLock.Lock()
	c.brokerEncoder = msgp.NewWriter(c.brokerConn)
	c.brokerEncodeLock.Unlock()
	c.brokerDead = false
	return nil
}

// Loop until we can finally connect to the coordinator.
// This blocks indefinitely until it is successful
func (c *Client) connectCoordinator() {
	var (
		err      error
		waitTime = 1 * time.Second
		maxWait  = 30 * time.Second
	)
	c.coordConn, err = net.DialTCP("tcp", nil, c.CoordinatorAddress)
	for err != nil {
		log.Warningf("Retrying coordinator connection to %v with delay %v", c.CoordinatorAddress, waitTime)
		time.Sleep(waitTime)
		waitTime *= 2
		if waitTime > maxWait {
			waitTime = maxWait
		}
		c.coordConn, err = net.DialTCP("tcp", nil, c.CoordinatorAddress)
	}
	log.Debug("Connected to coordinator")
	go c.listenCoordinator()
	c.coordEncoder = msgp.NewWriter(c.coordConn)
}

//TODO: implement
// This function should contact the coordinator to get the new broker
func (c *Client) configureNewBroker(m *common.BrokerAssignmentMessage) {
	var err error
	c.BrokerAddress, err = net.ResolveTCPAddr("tcp", m.ClientBrokerAddr)
	if err != nil {
		log.Critical(errors.Wrap(err, "Could not resolve local broker address"))
	}
	if err = c.connectBroker(c.BrokerAddress); err != nil {
		log.Critical(errors.Wrap(err, "Could not connect to local broker"))
	}
	// send our subscription once we're back
	c.resubscribe()
	go c.listen()
}

// Sends message to the currently configured broker
func (c *Client) sendBroker(m common.Sendable) error {
	c.brokerEncodeLock.Lock()
	defer c.brokerEncodeLock.Unlock()
	if err := m.Encode(c.brokerEncoder); err != nil {
		return errors.Wrap(err, "Could not encode message")
	}
	if err := c.brokerEncoder.Flush(); err != nil {
		// do failover if we fail to send
		go c.doFailover()
		return errors.Wrap(err, "Could not send message to broker")
	}
	return nil
}

func (c *Client) sendCoordinator(m common.Sendable) error {
	c.coordEncodeLock.Lock()
	if err := m.Encode(c.coordEncoder); err != nil {
		c.coordEncodeLock.Unlock()
		return errors.Wrap(err, "Could not encode message")
	}
	if err := c.coordEncoder.Flush(); err != nil {
		c.coordEncodeLock.Unlock()
		c.connectCoordinator()
		return errors.Wrap(err, "Could not send message to coordinator")
	}
	c.coordEncodeLock.Unlock()
	return nil
}

func (c *Client) listen() {
	reader := msgp.NewReader(net.Conn(c.brokerConn))
	for {
		if c.brokerDead {
			return
		}
		msg, err := common.MessageFromDecoderMsgp(reader)
		if err == io.EOF {
			log.Warn("connection closed. Do failover")
			c.doFailover()
			break
		}
		if err != nil {
			log.Warn(errors.Wrap(err, "Could not decode message"))
		}

		switch m := msg.(type) {
		case *common.PublishMessage:
			if c.hasPublishHandler {
				c.publishHandler(m)
			} else {
				log.Infof("Got publish message %v", m)
			}
		case *common.SubscriptionDiffMessage:
			if c.hasDiffHandler {
				c.diffHandler(m)
			} else {
				log.Infof("Got diff message %v", m)
			}
		default:
			log.Infof("Got %T message %v", m, m)
		}
	}
}

func (c *Client) listenCoordinator() {
	if c.coordConn == nil {
		return
	}
	reader := msgp.NewReader(net.Conn(c.coordConn))
	for {
		msg, err := common.MessageFromDecoderMsgp(reader)
		if err == io.EOF {
			log.Warn("connection closed. Do failover")
			c.doFailover()
			break
		}
		if err != nil {
			log.Warn(errors.Wrap(err, "Could not decode message"))
		}

		log.Infof("Got %T message %v from coordinator", msg, msg)
		switch m := msg.(type) {
		case *common.BrokerAssignmentMessage:
			c.configureNewBroker(m)
		default:
			log.Infof("Got %T message %v from coordinator", m, m)
		}
	}
}

// after the duration expires, stop the client by signalling on c.Stop
func (c *Client) StopIn(d time.Duration) {
	go func(c *Client) {
		time.Sleep(d)
		c.Stop <- true
	}(c)
}

// subscribes the client to the given query via the broker specified in the
// client's configuration (or whatever next broker if the client has experienced
// a failover). Use AttachPublishHandler to do special handling of the received
// published messages
func (c *Client) Subscribe(query string) {
	// cache the query we are subscribing to
	c.query = query
	msg := &common.QueryMessage{
		UUID:  c.ID,
		Query: c.query,
	}
	if err := c.sendBroker(msg); err != nil {
		log.Errorf("Error? %v", errors.Cause(err))
	}
}

// call to resend subscription
func (c *Client) resubscribe() {
	c.Subscribe(c.query)
}

// Called externally, this adds new metadata to this client if it is acting
// as a publisher. If the client is not a publisher, this function runs but does
// not affect any part of the subscription operation.
// To DELETE a metadata key, use a value of nil for the key you want to delete. It
// will get folded into the next publish message, and then the keys will be removed
// from the local metadata map
func (c *Client) AddMetadata(pubid common.UUID, newm map[string]interface{}) {
	if len(newm) == 0 {
		return
	}
	c.publishersLock.Lock()
	if pub, found := c.publishers[pubid]; found {
		pub.AddMetadata(newm)
	}
	c.publishersLock.Unlock()
}

func (c *Client) AddPublisher(id common.UUID) *Publisher {
	pub := NewPublisher(id, c.BrokerAddress, c.CoordinatorAddress)
	c.publishersLock.Lock()
	c.publishers[id] = pub
	c.publishersLock.Unlock()
	return pub
}
<|endoftext|>


<commit_before>package deje

import (
	"github.com/campadrenalin/go-deje/document"
	"github.com/campadrenalin/go-deje/util"
)

// Wraps the low-level capabilities of the basic Client to provide
// an easier, more useful API to downstream code.
type SimpleClient struct {
	client *Client
	tip    string
}

func NewSimpleClient(topic string) *SimpleClient {
	raw_client := NewClient(topic)
	simple_client := &SimpleClient{&raw_client, ""}
	raw_client.SetEventCallback(func(event interface{}) {
		map_ev, ok := event.(map[string]interface{})
		if !ok {
			return
		}
		evtype, ok := map_ev["type"].(string)
		if !ok {
			return
		}

		doc := simple_client.GetDoc()
		switch evtype {
		case "01-request-tip":
			simple_client.PublishTip()
		case "01-publish-tip":
			hash, ok := map_ev["tip_hash"].(string)
			if ok && simple_client.tip != hash {
				simple_client.RequestHistory()
			}
		case "01-request-history":
			simple_client.PublishHistory()
		case "01-publish-history":
			// This is intentionally structured so that the
			// coverage tests will be helpful for catching all
			// possible circumstances.
			history, ok := map_ev["history"].([]interface{})
			if !ok {
				return
			}
			for _, serial_event := range history {
				doc_ev := doc.NewEvent("")
				err := util.CloneMarshal(serial_event, &doc_ev)
				if err != nil {
					return
				}
				doc_ev.Register()
			}
			hash, ok := map_ev["tip_hash"].(string)
			if !ok {
				return
			}
			tip_event, ok := doc.Events[hash]
			if !ok {
				return
			}
			err := tip_event.Goto()
			if err != nil {
				return
			}
			simple_client.tip = hash
		}
	})
	return simple_client
}

// Connect and immediately request the tip event hash.
func (sc *SimpleClient) Connect(url string) error {
	err := sc.client.Connect(url)
	if err != nil {
		return err
	}
	return sc.RequestTip()
}

func (sc *SimpleClient) RequestTip() error {
	return sc.client.Publish(map[string]interface{}{
		"type": "01-request-tip",
	})
}

func (sc *SimpleClient) PublishTip() error {
	return sc.client.Publish(map[string]interface{}{
		"type":     "01-publish-tip",
		"tip_hash": sc.tip,
	})
}

func (sc *SimpleClient) RequestHistory() error {
	return sc.client.Publish(map[string]interface{}{
		"type": "01-request-history",
	})
}

func (sc *SimpleClient) PublishHistory() error {
	response := map[string]interface{}{
		"type":     "01-publish-history",
		"tip_hash": sc.tip,
	}
	doc := sc.GetDoc()
	ev, ok := doc.Events[sc.tip]
	if !ok {
		response["error"] = "not-found"
		return sc.client.Publish(response)
	}

	history, ok := ev.GetHistory()
	if !ok {
		response["error"] = "root-not-found"
		return sc.client.Publish(response)
	}
	response["history"] = history
	return sc.client.Publish(response)
}

// Navigate the Document to an Event, and promote it as the tip.
func (sc *SimpleClient) Promote(ev document.Event) error {
	if err := ev.Goto(); err != nil {
		return err
	}
	sc.tip = ev.Hash()
	return sc.PublishTip()
}

// Get the Document object owned by this Client.
func (sc *SimpleClient) GetDoc() *document.Document {
	return sc.client.Doc
}

// Return the current contents of the document.
func (sc *SimpleClient) Export() interface{} {
	return sc.client.Doc.State.Export()
}
<commit_msg>Move onRcv stuff into its own function<commit_after>package deje

import (
	"errors"

	"github.com/campadrenalin/go-deje/document"
	"github.com/campadrenalin/go-deje/util"
)

// Wraps the low-level capabilities of the basic Client to provide
// an easier, more useful API to downstream code.
type SimpleClient struct {
	client *Client
	tip    string
}

func NewSimpleClient(topic string) *SimpleClient {
	raw_client := NewClient(topic)
	simple_client := &SimpleClient{&raw_client, ""}
	raw_client.SetEventCallback(func(event interface{}) {
		simple_client.onRcv(event)
	})
	return simple_client
}

func (sc *SimpleClient) onRcv(event interface{}) error {
	map_ev, ok := event.(map[string]interface{})
	if !ok {
		return errors.New("Non-{} message")
	}
	evtype, ok := map_ev["type"].(string)
	if !ok {
		return errors.New("Message with no 'type' param")
	}

	doc := sc.GetDoc()
	switch evtype {
	case "01-request-tip":
		sc.PublishTip()
	case "01-publish-tip":
		hash, ok := map_ev["tip_hash"].(string)
		if !ok {
			return errors.New("Message with bad tip_hash param")
		}
		if sc.tip != hash {
			sc.RequestHistory()
		}
	case "01-request-history":
		sc.PublishHistory()
	case "01-publish-history":
		// This is intentionally structured so that the
		// coverage tests will be helpful for catching all
		// possible circumstances.
		history, ok := map_ev["history"].([]interface{})
		if !ok {
			return errors.New("History message with bad history param")
		}
		for _, serial_event := range history {
			doc_ev := doc.NewEvent("")
			err := util.CloneMarshal(serial_event, &doc_ev)
			if err != nil {
				return err
			}
			doc_ev.Register()
		}
		hash, ok := map_ev["tip_hash"].(string)
		if !ok {
			return errors.New("Message with bad tip_hash param")
		}
		tip_event, ok := doc.Events[hash]
		if !ok {
			return errors.New("Unknown event " + hash)
		}
		err := tip_event.Goto()
		if err != nil {
			return err
		}
		sc.tip = hash
	}
	return errors.New("Unfamiliar message type")
}

// Connect and immediately request the tip event hash.
func (sc *SimpleClient) Connect(url string) error {
	err := sc.client.Connect(url)
	if err != nil {
		return err
	}
	return sc.RequestTip()
}

func (sc *SimpleClient) RequestTip() error {
	return sc.client.Publish(map[string]interface{}{
		"type": "01-request-tip",
	})
}

func (sc *SimpleClient) PublishTip() error {
	return sc.client.Publish(map[string]interface{}{
		"type":     "01-publish-tip",
		"tip_hash": sc.tip,
	})
}

func (sc *SimpleClient) RequestHistory() error {
	return sc.client.Publish(map[string]interface{}{
		"type": "01-request-history",
	})
}

func (sc *SimpleClient) PublishHistory() error {
	response := map[string]interface{}{
		"type":     "01-publish-history",
		"tip_hash": sc.tip,
	}
	doc := sc.GetDoc()
	ev, ok := doc.Events[sc.tip]
	if !ok {
		response["error"] = "not-found"
		return sc.client.Publish(response)
	}

	history, ok := ev.GetHistory()
	if !ok {
		response["error"] = "root-not-found"
		return sc.client.Publish(response)
	}
	response["history"] = history
	return sc.client.Publish(response)
}

// Navigate the Document to an Event, and promote it as the tip.
func (sc *SimpleClient) Promote(ev document.Event) error {
	if err := ev.Goto(); err != nil {
		return err
	}
	sc.tip = ev.Hash()
	return sc.PublishTip()
}

// Get the Document object owned by this Client.
func (sc *SimpleClient) GetDoc() *document.Document {
	return sc.client.Doc
}

// Return the current contents of the document.
func (sc *SimpleClient) Export() interface{} {
	return sc.client.Doc.State.Export()
}
<|endoftext|>


<commit_before>// This is free and unencumbered software released into the public domain.
//
// Anyone is free to copy, modify, publish, use, compile, sell, or
// distribute this software, either in source code form or as a compiled
// binary, for any purpose, commercial or non-commercial, and by any
// means.
//
// In jurisdictions that recognize copyright laws, the author or authors
// of this software dedicate any and all copyright interest in the
// software to the public domain. We make this dedication for the benefit
// of the public at large and to the detriment of our heirs and
// successors. We intend this dedication to be an overt act of
// relinquishment in perpetuity of all present and future rights to this
// software under copyright law.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
// OTHER DEALINGS IN THE SOFTWARE.
//
// For more information, please refer to <http://unlicense.org>

// dtex is a tex compiler wrapper tool.
package main

import (
	"fmt"
	"hash/fnv"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
)

func Usage() {
	fmt.Fprintln(os.Stderr, "Usage: dtex [tex options] file.tex")
	fmt.Fprintln(os.Stderr, "")
	fmt.Fprintln(os.Stderr, "will compile file.tex as many times as necessary: until all the generated")
	fmt.Fprintln(os.Stderr, "temporary files don't change anymore, with a maximum of 5 compilations.")
	fmt.Fprintln(os.Stderr, "")
	fmt.Fprintln(os.Stderr, "Usage: dtex -clean")
	fmt.Fprintln(os.Stderr, "")
	fmt.Fprintln(os.Stderr, "will remove all temporary files used by this program.")
	os.Exit(1)
}

// root directory where the TEX engine writes temporary files (.aux, ...)
var tmp = filepath.Join(os.TempDir(), "dtex")

func main() {
	SetLogOutput()
	log.Println("Using temporary root:", tmp)
	args := os.Args[1:]
	file := ParseArgs(args)
	tmpbase := GetTmp(file)
	args = append([]string{"-output-directory", filepath.Dir(tmpbase)}, args...)
	tex := GetTexEngine()

	log.Println("Computing initial hashes of", tmpbase)
	hashes := NewHashes(tmpbase)
	for try := 0; hashes.Changed() && try < 5; try++ {
		log.Println("Compile iteration", try)
		Compile(tex, args)
		log.Println("Updating hashes")
		hashes.Update()
	}
	if hashes.Changed() {
		fmt.Fprintln(os.Stderr, "Warning: 5 compilations were maybe insufficient")
	}
	if err := os.Rename(tmpbase+".pdf", file+".pdf"); err != nil {
		Err("Move resulting pdf into place: %v\n", err)
	}
}

func IsVerbose() bool { return os.Getenv("VERBOSE") != "" }

func Err(msg string, a ...interface{}) {
	fmt.Fprintf(os.Stderr, msg, a...)
	os.Exit(1)
}

func SetLogOutput() {
	log.SetOutput(ioutil.Discard)
	if IsVerbose() {
		log.SetOutput(os.Stderr)
	}
}

func ParseArgs(args []string) string {
	if len(args) == 1 && args[0] == "-clean" {
		log.Printf("rm -r %q\n", tmp)
		if err := os.RemoveAll(tmp); err != nil {
			Err("clean temporary files: %v\n", err)
		}
		os.Exit(0)
	}
	if len(args) < 1 {
		Usage()
	}
	for _, a := range args {
		if a == "-output-directory" {
			Err("\"-output-directory\" flag not allowed\n")
		}
	}
	file := args[len(args)-1]
	if filepath.Ext(file) == ".tex" {
		file = file[:len(file)-len(".tex")]
	}
	return file
}

func GetTmp(file string) string {
	base, err := filepath.Abs(file)
	if err != nil {
		log.Printf("absolute path(%q): %v\n", file, err)
		base = filepath.Base(file)
	}
	volume := filepath.VolumeName(base)
	base = base[len(volume):]
	tmpbase := filepath.Join(tmp, base)
	tmpdir := filepath.Dir(tmpbase)
	if err := os.MkdirAll(tmpdir, 0755); err != nil {
		Err("Create temporary directory (%v): %v\n", tmpdir, err)
	}
	return tmpbase
}

func GetTexEngine() string {
	tex := "pdflatex"
	if t := os.Getenv("TEX"); t != "" {
		tex = t
	}
	return tex
}

type Hashes struct {
	base string
	h    map[string]uint64
	mod  bool
}

func NewHashes(base string) *Hashes {
	h := &Hashes{base: base, h: map[string]uint64{}}
	h.Update()
	h.mod = true
	return h
}

func (h *Hashes) Update() {
	pat := filepath.Join(filepath.Dir(h.base), "*.*")
	files, err := filepath.Glob(pat)
	if err != nil {
		Err("bad filepath.Glob(%q): %v\n", pat, err)
	}
	h.mod = false
	for _, file := range files {
		ext := filepath.Ext(file)
		if ext == ".pdf" || ext == ".log" {
			continue
		}
		id := HashFile(file)
		log.Println("Hashing", file, "→", id)
		if id != h.h[file] {
			log.Println("  file changed")
			h.mod = true
		}
		h.h[file] = id
	}
}

func (h *Hashes) Changed() bool { return h.mod }

func HashFile(file string) uint64 {
	h := fnv.New64a()
	f, err := os.Open(file)
	if err != nil {
		Err("Open file (%v): %v\n", file, err)
	}
	defer f.Close()
	if _, err := io.Copy(h, f); err != nil {
		Err("Read file (%v): %v\n", file, err)
	}
	return h.Sum64()
}

func Compile(tex string, args []string) {
	log.Printf("Running %v %q\n", tex, args)
	out, err := exec.Command(tex, args...).CombinedOutput()
	if err != nil {
		os.Stdout.Write(out)
		Err("Compilation error: %v\n", err)
	}
}
<commit_msg>use PDFTEX env variable instead of TEX<commit_after>// This is free and unencumbered software released into the public domain.
//
// Anyone is free to copy, modify, publish, use, compile, sell, or
// distribute this software, either in source code form or as a compiled
// binary, for any purpose, commercial or non-commercial, and by any
// means.
//
// In jurisdictions that recognize copyright laws, the author or authors
// of this software dedicate any and all copyright interest in the
// software to the public domain. We make this dedication for the benefit
// of the public at large and to the detriment of our heirs and
// successors. We intend this dedication to be an overt act of
// relinquishment in perpetuity of all present and future rights to this
// software under copyright law.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
// OTHER DEALINGS IN THE SOFTWARE.
//
// For more information, please refer to <http://unlicense.org>

// dtex is a tex compiler wrapper tool.
package main

import (
	"fmt"
	"hash/fnv"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
)

func Usage() {
	fmt.Fprintln(os.Stderr, "Usage: dtex [tex options] file.tex")
	fmt.Fprintln(os.Stderr, "")
	fmt.Fprintln(os.Stderr, "will compile file.tex as many times as necessary: until all the generated")
	fmt.Fprintln(os.Stderr, "temporary files don't change anymore, with a maximum of 5 compilations.")
	fmt.Fprintln(os.Stderr, "")
	fmt.Fprintln(os.Stderr, "Usage: dtex -clean")
	fmt.Fprintln(os.Stderr, "")
	fmt.Fprintln(os.Stderr, "will remove all temporary files used by this program.")
	os.Exit(1)
}

// root directory where the TEX engine writes temporary files (.aux, ...)
var tmp = filepath.Join(os.TempDir(), "dtex")

func main() {
	SetLogOutput()
	log.Println("Using temporary root:", tmp)
	args := os.Args[1:]
	file := ParseArgs(args)
	tmpbase := GetTmp(file)
	args = append([]string{"-output-directory", filepath.Dir(tmpbase)}, args...)
	tex := GetTexEngine()

	log.Println("Computing initial hashes of", tmpbase)
	hashes := NewHashes(tmpbase)
	for try := 0; hashes.Changed() && try < 5; try++ {
		log.Println("Compile iteration", try)
		Compile(tex, args)
		log.Println("Updating hashes")
		hashes.Update()
	}
	if hashes.Changed() {
		fmt.Fprintln(os.Stderr, "Warning: 5 compilations were maybe insufficient")
	}
	if err := os.Rename(tmpbase+".pdf", file+".pdf"); err != nil {
		Err("Move resulting pdf into place: %v\n", err)
	}
}

func IsVerbose() bool { return os.Getenv("VERBOSE") != "" }

func Err(msg string, a ...interface{}) {
	fmt.Fprintf(os.Stderr, msg, a...)
	os.Exit(1)
}

func SetLogOutput() {
	log.SetOutput(ioutil.Discard)
	if IsVerbose() {
		log.SetOutput(os.Stderr)
	}
}

func ParseArgs(args []string) string {
	if len(args) == 1 && args[0] == "-clean" {
		log.Printf("rm -r %q\n", tmp)
		if err := os.RemoveAll(tmp); err != nil {
			Err("clean temporary files: %v\n", err)
		}
		os.Exit(0)
	}
	if len(args) < 1 {
		Usage()
	}
	for _, a := range args {
		if a == "-output-directory" {
			Err("\"-output-directory\" flag not allowed\n")
		}
	}
	file := args[len(args)-1]
	if filepath.Ext(file) == ".tex" {
		file = file[:len(file)-len(".tex")]
	}
	return file
}

func GetTmp(file string) string {
	base, err := filepath.Abs(file)
	if err != nil {
		log.Printf("absolute path(%q): %v\n", file, err)
		base = filepath.Base(file)
	}
	volume := filepath.VolumeName(base)
	base = base[len(volume):]
	tmpbase := filepath.Join(tmp, base)
	tmpdir := filepath.Dir(tmpbase)
	if err := os.MkdirAll(tmpdir, 0755); err != nil {
		Err("Create temporary directory (%v): %v\n", tmpdir, err)
	}
	return tmpbase
}

func GetTexEngine() string {
	tex := "pdflatex"
	if t := os.Getenv("PDFTEX"); t != "" {
		tex = t
	}
	return tex
}

type Hashes struct {
	base string
	h    map[string]uint64
	mod  bool
}

func NewHashes(base string) *Hashes {
	h := &Hashes{base: base, h: map[string]uint64{}}
	h.Update()
	h.mod = true
	return h
}

func (h *Hashes) Update() {
	pat := filepath.Join(filepath.Dir(h.base), "*.*")
	files, err := filepath.Glob(pat)
	if err != nil {
		Err("bad filepath.Glob(%q): %v\n", pat, err)
	}
	h.mod = false
	for _, file := range files {
		ext := filepath.Ext(file)
		if ext == ".pdf" || ext == ".log" {
			continue
		}
		id := HashFile(file)
		log.Println("Hashing", file, "→", id)
		if id != h.h[file] {
			log.Println("  file changed")
			h.mod = true
		}
		h.h[file] = id
	}
}

func (h *Hashes) Changed() bool { return h.mod }

func HashFile(file string) uint64 {
	h := fnv.New64a()
	f, err := os.Open(file)
	if err != nil {
		Err("Open file (%v): %v\n", file, err)
	}
	defer f.Close()
	if _, err := io.Copy(h, f); err != nil {
		Err("Read file (%v): %v\n", file, err)
	}
	return h.Sum64()
}

func Compile(tex string, args []string) {
	log.Printf("Running %v %q\n", tex, args)
	out, err := exec.Command(tex, args...).CombinedOutput()
	if err != nil {
		os.Stdout.Write(out)
		Err("Compilation error: %v\n", err)
	}
}
<|endoftext|>


<commit_before>package addrs

import (
	"fmt"
)

// Resource is an address for a resource block within configuration, which
// contains potentially-multiple resource instances if that configuration
// block uses "count" or "for_each".
type Resource struct {
	referenceable
	Mode ResourceMode
	Type string
	Name string
}

func (r Resource) String() string {
	switch r.Mode {
	case ManagedResourceMode:
		return fmt.Sprintf("%s.%s", r.Type, r.Name)
	case DataResourceMode:
		return fmt.Sprintf("data.%s.%s", r.Type, r.Name)
	default:
		panic(fmt.Errorf("resource address with invalid mode %s", r.Mode))
	}
}

// Instance produces the address for a specific instance of the receiver
// that is idenfied by the given key.
func (r Resource) Instance(key InstanceKey) ResourceInstance {
	return ResourceInstance{
		Resource: r,
		Key:      key,
	}
}

// Absolute returns an AbsResource from the receiver and the given module
// instance address.
func (r Resource) Absolute(module ModuleInstance) AbsResource {
	return AbsResource{
		Module:   module,
		Resource: r,
	}
}

// ResourceInstance is an address for a specific instance of a resource.
// When a resource is defined in configuration with "count" or "for_each" it
// produces zero or more instances, which can be addressed using this type.
type ResourceInstance struct {
	referenceable
	Resource Resource
	Key      InstanceKey
}

func (r ResourceInstance) String() string {
	if r.Key == NoKey {
		return r.Resource.String()
	}
	return r.Resource.String() + r.Key.String()
}

// Absolute returns an AbsResourceInstance from the receiver and the given module
// instance address.
func (r ResourceInstance) Absolute(module ModuleInstance) AbsResourceInstance {
	return AbsResourceInstance{
		Module:   module,
		Resource: r,
	}
}

// Resource returns the address of a particular resource within the receiver.
func (m ModuleInstance) Resource(mode ResourceMode, typeName string, name string) AbsResource {
	return AbsResource{
		Module: m,
		Resource: Resource{
			Mode: mode,
			Type: typeName,
			Name: name,
		},
	}
}

// AbsResource is an absolute address for a resource under a given module path.
type AbsResource struct {
	Module   ModuleInstance
	Resource Resource
}

// AbsResourceInstance is an absolute address for a resource instance under a
// given module path.
type AbsResourceInstance struct {
	Module   ModuleInstance
	Resource ResourceInstance
}

// ResourceInstance returns the address of a particular resource instance within the receiver.
func (m ModuleInstance) ResourceInstance(mode ResourceMode, typeName string, name string, key InstanceKey) AbsResourceInstance {
	return AbsResourceInstance{
		Module: m,
		Resource: ResourceInstance{
			Resource: Resource{
				Mode: mode,
				Type: typeName,
				Name: name,
			},
			Key: key,
		},
	}
}

// ResourceMode defines which lifecycle applies to a given resource. Each
// resource lifecycle has a slightly different address format.
type ResourceMode rune

//go:generate stringer -type ResourceMode

const (
	// InvalidResourceMode is the zero value of ResourceMode and is not
	// a valid resource mode.
	InvalidResourceMode ResourceMode = 0

	// ManagedResourceMode indicates a managed resource, as defined by
	// "resource" blocks in configuration.
	ManagedResourceMode ResourceMode = 'M'

	// DataResourceMode indicates a data resource, as defined by
	// "data" blocks in configuration.
	DataResourceMode ResourceMode = 'D'
)
<commit_msg>addrs: implement String for AbsResource and AbsResourceInstance<commit_after>package addrs

import (
	"fmt"
)

// Resource is an address for a resource block within configuration, which
// contains potentially-multiple resource instances if that configuration
// block uses "count" or "for_each".
type Resource struct {
	referenceable
	Mode ResourceMode
	Type string
	Name string
}

func (r Resource) String() string {
	switch r.Mode {
	case ManagedResourceMode:
		return fmt.Sprintf("%s.%s", r.Type, r.Name)
	case DataResourceMode:
		return fmt.Sprintf("data.%s.%s", r.Type, r.Name)
	default:
		panic(fmt.Errorf("resource address with invalid mode %s", r.Mode))
	}
}

// Instance produces the address for a specific instance of the receiver
// that is idenfied by the given key.
func (r Resource) Instance(key InstanceKey) ResourceInstance {
	return ResourceInstance{
		Resource: r,
		Key:      key,
	}
}

// Absolute returns an AbsResource from the receiver and the given module
// instance address.
func (r Resource) Absolute(module ModuleInstance) AbsResource {
	return AbsResource{
		Module:   module,
		Resource: r,
	}
}

// ResourceInstance is an address for a specific instance of a resource.
// When a resource is defined in configuration with "count" or "for_each" it
// produces zero or more instances, which can be addressed using this type.
type ResourceInstance struct {
	referenceable
	Resource Resource
	Key      InstanceKey
}

func (r ResourceInstance) String() string {
	if r.Key == NoKey {
		return r.Resource.String()
	}
	return r.Resource.String() + r.Key.String()
}

// Absolute returns an AbsResourceInstance from the receiver and the given module
// instance address.
func (r ResourceInstance) Absolute(module ModuleInstance) AbsResourceInstance {
	return AbsResourceInstance{
		Module:   module,
		Resource: r,
	}
}

// AbsResource is an absolute address for a resource under a given module path.
type AbsResource struct {
	Module   ModuleInstance
	Resource Resource
}

// Resource returns the address of a particular resource within the receiver.
func (m ModuleInstance) Resource(mode ResourceMode, typeName string, name string) AbsResource {
	return AbsResource{
		Module: m,
		Resource: Resource{
			Mode: mode,
			Type: typeName,
			Name: name,
		},
	}
}

func (r AbsResource) String() string {
	if len(r.Module) == 0 {
		return r.Resource.String()
	}
	return fmt.Sprintf("%s.%s", r.Module.String(), r.Resource.String())
}

// AbsResourceInstance is an absolute address for a resource instance under a
// given module path.
type AbsResourceInstance struct {
	Module   ModuleInstance
	Resource ResourceInstance
}

// ResourceInstance returns the address of a particular resource instance within the receiver.
func (m ModuleInstance) ResourceInstance(mode ResourceMode, typeName string, name string, key InstanceKey) AbsResourceInstance {
	return AbsResourceInstance{
		Module: m,
		Resource: ResourceInstance{
			Resource: Resource{
				Mode: mode,
				Type: typeName,
				Name: name,
			},
			Key: key,
		},
	}
}

func (r AbsResourceInstance) String() string {
	if len(r.Module) == 0 {
		return r.Resource.String()
	}
	return fmt.Sprintf("%s.%s", r.Module.String(), r.Resource.String())
}

// ResourceMode defines which lifecycle applies to a given resource. Each
// resource lifecycle has a slightly different address format.
type ResourceMode rune

//go:generate stringer -type ResourceMode

const (
	// InvalidResourceMode is the zero value of ResourceMode and is not
	// a valid resource mode.
	InvalidResourceMode ResourceMode = 0

	// ManagedResourceMode indicates a managed resource, as defined by
	// "resource" blocks in configuration.
	ManagedResourceMode ResourceMode = 'M'

	// DataResourceMode indicates a data resource, as defined by
	// "data" blocks in configuration.
	DataResourceMode ResourceMode = 'D'
)
<|endoftext|>


<commit_before>// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
// is governed by an MIT-style license that can be found in the LICENSE file.

package logger

import (
	"log"
	"testing"
)

func TestLevels(t *testing.T) {
	l := NewStdlibLogger()
	l.SetFlags(log.Lshortfile | log.Ldate | log.Ltime)
	l.Debugln("hello debug")
	l.Verboseln("hello verbose")
	l.Infoln("hello info")
	l.Okln("hello OK")
	l.Warnln("hello warn")
}

// All benchmarks should be ran with LOGGER_DISCARD="<non-empty>", to redirect logging to ioutil.Discard

func BenchmarkLshortfile(b *testing.B) {
	l := NewStdlibLogger()
	l.SetFlags(log.Lshortfile)
	l.SetLevel(LevelOK)
	s := "hi"
	for i := 0; i < b.N; i++ {
		l.Okln(s)
	}
}

func BenchmarkBare(b *testing.B) {
	l := NewStdlibLogger()
	l.SetFlags(0)
	l.SetLevel(LevelOK)
	s := "hi"
	for i := 0; i < b.N; i++ {
		l.Okln(s)
	}
}

func BenchmarkLevelTooLowBare(b *testing.B) {
	l := NewStdlibLogger()
	l.SetFlags(0)
	l.SetLevel(LevelOK)
	s := "hi"
	for i := 0; i < b.N; i++ {
		l.Debugln(s)
	}
}

func BenchmarkEmptyFormat(b *testing.B) {
	l := NewStdlibLogger()
	l.SetFlags(0)
	l.SetLevel(LevelOK)
	s := ""
	s1 := "hi"
	for i := 0; i < b.N; i++ {
		l.Okf(s, s1)
	}
}
<commit_msg>renamed benchmarks<commit_after>// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
// is governed by an MIT-style license that can be found in the LICENSE file.

package logger

import (
	"log"
	"testing"
)

func TestLevels(t *testing.T) {
	l := NewStdlibLogger()
	l.SetFlags(log.Lshortfile | log.Ldate | log.Ltime)
	l.Debugln("hello debug")
	l.Verboseln("hello verbose")
	l.Infoln("hello info")
	l.Okln("hello OK")
	l.Warnln("hello warn")
}

// All benchmarks should be ran with LOGGER_DISCARD="<non-empty>", to redirect logging to ioutil.Discard

func BenchmarkLshortfileLn(b *testing.B) {
	l := NewStdlibLogger()
	l.SetFlags(log.Lshortfile)
	l.SetLevel(LevelOK)
	s := "hi"
	for i := 0; i < b.N; i++ {
		l.Okln(s)
	}
}

func BenchmarkNoFlagLn(b *testing.B) {
	l := NewStdlibLogger()
	l.SetFlags(0)
	l.SetLevel(LevelOK)
	s := "hi"
	for i := 0; i < b.N; i++ {
		l.Okln(s)
	}
}

func BenchmarkNoFlagF(b *testing.B) {
	l := NewStdlibLogger()
	l.SetFlags(0)
	l.SetLevel(LevelOK)
	s := ""
	s1 := "hi"
	for i := 0; i < b.N; i++ {
		l.Okf(s, s1)
	}
}

func BenchmarkLevelTooLow(b *testing.B) {
	l := NewStdlibLogger()
	l.SetFlags(0)
	l.SetLevel(LevelOK)
	s := "hi"
	for i := 0; i < b.N; i++ {
		l.Debugln(s)
	}
}
<|endoftext|>