<commit_before>package bot

import (
	"github.com/bwmarrin/discordgo"
	"go.uber.org/zap"
	"fmt"
	"github.com/FederationOfFathers/dashboard/messaging"
)

type DiscordAPI struct {
	Config DiscordCfg
	discord *discordgo.Session
	assignmentMsgs map[string]map[string]string
}

type EmojiRole struct {
	EmojiId string `yaml:"emojiId"`
	RoleId string `yaml:"roleId"`
}

type DiscordEmojiRoleGroup struct {
	MessageTitle string `yaml:"messageTitle"`
	MessageBody string `yaml:"messageBody"`
	Roles []EmojiRole `yaml:"roles"`

}

type DiscordRoleCfg struct {
	ChannelId string `yaml:"channelId"`
	EmojiRoleGroups []DiscordEmojiRoleGroup `yaml:"emojiRoles"`

}

type DiscordCfg struct {
	ClientId string `yaml:"appClientId"`
	Token string `yaml:"botToken"`
	StreamChannelId string `yaml:"streamChannelId"`
	GuildId string `yaml:"guildId"`
	RoleCfg DiscordRoleCfg `yaml:"roleConfig"`
}

func NewDiscordAPI(cfg DiscordCfg) DiscordAPI {
	return DiscordAPI{
		Config: cfg,
	}
}

func (d DiscordAPI) roleAssignmentHandler(s *discordgo.Session, event *discordgo.MessageReactionAdd) {

	// skip if the event was from the bot/app
	if event.UserID == d.Config.ClientId {
		return
	}

	// only handle if the message is one we have configured
	if roles, ok := d.assignmentMsgs[event.MessageID]; ok {

		// Unicode emojis use the unicode character (name) as the id. Others use the name and integer as the id.
		emojiId := event.Emoji.Name
		if event.Emoji.ID != "" {
			emojiId = fmt.Sprintf(":%s:%s", event.Emoji.Name, event.Emoji.ID)
		}

		if roleId, ok := roles[emojiId]; ok{
			// get the user from the server/guild
			member, err := d.discord.GuildMember(d.Config.GuildId, event.UserID)
			if err != nil {
				Logger.Error("Unable to get member", zap.Error(err))
				return
			}

			userHadRole := false
			// if the member has the mapped role, remove it
			for _, id := range member.Roles {
				if id == roleId {
					d.removeRoleFromUser(event.UserID, roleId)
					userHadRole = true // true, even if an attempt was made and failed
				}
			}

			// if the member does not have the mapped role, add it
			if !userHadRole {
				d.addRoleToUser(event.UserID, roleId)
			}
		}

		// remove the users reaction. If the add/remove failed, they can click it again to re-trigger
		err := d.discord.MessageReactionRemove(
			event.ChannelID,
			event.MessageID,
			emojiId,
			event.UserID,
		)

		if err != nil {
		Logger.Error("could not remove reaction from message",
			zap.String("user", event.UserID),
			zap.String("emoji", emojiId))
		}
	}

}

// Needs to be called before any other API function work
func (d *DiscordAPI) Connect() {
	dg, err := discordgo.New("Bot " + d.Config.Token)
	if err != nil {
		Logger.Error("Unable to create discord connection", zap.Error(err))
		return
	}

	d.discord = dg
	dg.Open()
}

func (d *DiscordAPI) StartRoleHandlers(){
	d.listRoles()
	d.clearRoleChannel()
	d.createRoleMessages()
	d.discord.AddHandler(d.roleAssignmentHandler)
	Logger.Info("Role assignment handler started")
}

// Needs to be called to disconnect from discord
func (d *DiscordAPI) Shutdown() {
	d.discord.Close()
}

func (d DiscordAPI) PostStreamMessage(sm messaging.StreamMessage) error {
	if d.discord == nil {
		return fmt.Errorf("discord API not connected")
	}
	if d.Config.StreamChannelId == "" {
		return fmt.Errorf("stream channel id not configured")
	}
	author := discordgo.MessageEmbedAuthor{
		Name: fmt.Sprintf("%s is live!", sm.Username),
	}
	thumbnail := discordgo.MessageEmbedThumbnail{
		URL: sm.UserLogo,
	}
	footer := discordgo.MessageEmbedFooter{
		Text: fmt.Sprintf("%s | %s", sm.Platform, sm.Timestamp),
		IconURL: sm.PlatformLogo,
	}
	messageEmbed := discordgo.MessageEmbed{
		Description: sm.URL,
		Color: sm.PlatformColorInt,
		URL: sm.URL,
		Author: &author,
		Thumbnail: &thumbnail,
		Footer: &footer,
		Fields: []*discordgo.MessageEmbedField{
			{
				Name: "Game",
				Value: fmt.Sprintf("%s - %s", sm.Game, sm.Description),
				Inline: false,
			},
		},
	}
	_, err := d.discord.ChannelMessageSendEmbed(d.Config.StreamChannelId, &messageEmbed)
	return err
}

// removes all messages entered by this bot in the channel. Uses the ClientID of the bot/app
func (d *DiscordAPI) clearRoleChannel() {
	channelId := d.Config.RoleCfg.ChannelId

	messages, err := d.discord.ChannelMessages(channelId,50,"","","")
	if err != nil {
		Logger.Error("Unable to access channel messages", zap.String("channelId", channelId), zap.Error(err))
		return
	}
	for _, message := range messages {
		if message.Author.ID == d.Config.ClientId {
			err := d.discord.ChannelMessageDelete(channelId, message.ID)
			if err == nil {
				Logger.Info("Removed bot message", zap.String("messageId", message.ID))
			} else {
				Logger.Error("Unable to remove message", zap.String("messageId", message.ID), zap.Error(err))
			}

		}
	}
}

// creates messages in the channel and adds the emojis
func (d *DiscordAPI) createRoleMessages() {
	d.assignmentMsgs = make(map[string]map[string]string)
	for _, group := range d.Config.RoleCfg.EmojiRoleGroups {
		messageEmbed := discordgo.MessageEmbed{
			Title: group.MessageTitle,
			Description: group.MessageBody,
			Color: 15581239,
		}
		// create the message
		message, err := d.discord.ChannelMessageSendEmbed(d.Config.RoleCfg.ChannelId, &messageEmbed)
		if err != nil {
			Logger.Error("Unable to create message", zap.Error(err))
			continue
		}
		Logger.Info("Added role message", zap.String("message", group.MessageTitle), zap.String("messageId", message.ID))

		// add the emojis
		emojiRoles := make(map[string]string)
		for _, role := range group.Roles {
			emojiRoles[role.EmojiId] = role.RoleId
			err := d.discord.MessageReactionAdd(d.Config.RoleCfg.ChannelId, message.ID, role.EmojiId)
			if err != nil {
				Logger.Error("Unable to add emoji to message",
					zap.String("emojiId", role.EmojiId),
					zap.String("messageId", message.ID),
					zap.Error(err),
				)
			}
		}

		d.assignmentMsgs[message.ID] = emojiRoles
	}
}

func (d DiscordAPI) addRoleToUser(userId string, roleId string) {
	err := d.discord.GuildMemberRoleAdd(d.Config.GuildId, userId, roleId)
	if err != nil {
		Logger.Error("could not add role",
			zap.String("userId", userId),
			zap.String("roleId", roleId),
			zap.Error(err))
	} else {
		Logger.Info("added role to user",
			zap.String("userId", userId),
			zap.String("roleId", roleId),
		)
	}
}

func (d DiscordAPI) removeRoleFromUser(userId string, roleId string) {
	err := d.discord.GuildMemberRoleRemove(d.Config.GuildId, userId, roleId)
	if err != nil {
		Logger.Error("could not remove role", zap.Error(err))
	} else {
		Logger.Info("removed role from user",
			zap.String("userId", userId),
			zap.String("roleId", roleId),
		)
	}
}
func (d *DiscordAPI) listRoles() {
	roles, err := d.discord.GuildRoles(d.Config.GuildId)

	if err != nil {
		Logger.Error("Could not get roles", zap.Error(err))
		return
	}

	for i, role := range roles {
		Logger.Info(fmt.Sprintf("Role %d", i), zap.Any("role",role))
	}
}<commit_msg>Adding more log<commit_after>package bot

import (
	"github.com/bwmarrin/discordgo"
	"go.uber.org/zap"
	"fmt"
	"github.com/FederationOfFathers/dashboard/messaging"
)

type DiscordAPI struct {
	Config DiscordCfg
	discord *discordgo.Session
	assignmentMsgs map[string]map[string]string
}

type EmojiRole struct {
	EmojiId string `yaml:"emojiId"`
	RoleId string `yaml:"roleId"`
}

type DiscordEmojiRoleGroup struct {
	MessageTitle string `yaml:"messageTitle"`
	MessageBody string `yaml:"messageBody"`
	Roles []EmojiRole `yaml:"roles"`

}

type DiscordRoleCfg struct {
	ChannelId string `yaml:"channelId"`
	EmojiRoleGroups []DiscordEmojiRoleGroup `yaml:"emojiRoles"`

}

type DiscordCfg struct {
	ClientId string `yaml:"appClientId"`
	Token string `yaml:"botToken"`
	StreamChannelId string `yaml:"streamChannelId"`
	GuildId string `yaml:"guildId"`
	RoleCfg DiscordRoleCfg `yaml:"roleConfig"`
}

func NewDiscordAPI(cfg DiscordCfg) DiscordAPI {
	return DiscordAPI{
		Config: cfg,
	}
}

func (d DiscordAPI) roleAssignmentHandler(s *discordgo.Session, event *discordgo.MessageReactionAdd) {

	// skip if the event was from the bot/app
	if event.UserID == d.Config.ClientId {
		return
	}

	// only handle if the message is one we have configured
	if roles, ok := d.assignmentMsgs[event.MessageID]; ok {

		// Unicode emojis use the unicode character (name) as the id. Others use the name and integer as the id.
		emojiId := event.Emoji.Name
		if event.Emoji.ID != "" {
			emojiId = fmt.Sprintf(":%s:%s", event.Emoji.Name, event.Emoji.ID)
		}

		if roleId, ok := roles[emojiId]; ok{
			// get the user from the server/guild
			member, err := d.discord.GuildMember(d.Config.GuildId, event.UserID)
			if err != nil {
				Logger.Error("Unable to get member", zap.Error(err))
				return
			}

			userHadRole := false
			// if the member has the mapped role, remove it
			for _, id := range member.Roles {
				if id == roleId {
					d.removeRoleFromUser(event.UserID, roleId)
					userHadRole = true // true, even if an attempt was made and failed
				}
			}

			// if the member does not have the mapped role, add it
			if !userHadRole {
				d.addRoleToUser(event.UserID, roleId)
			}
		}

		// remove the users reaction. If the add/remove failed, they can click it again to re-trigger
		err := d.discord.MessageReactionRemove(
			event.ChannelID,
			event.MessageID,
			emojiId,
			event.UserID,
		)

		if err != nil {
		Logger.Error("could not remove reaction from message",
			zap.String("user", event.UserID),
			zap.String("emoji", emojiId),
			zap.Error(err))
		}
	}

}

// Needs to be called before any other API function work
func (d *DiscordAPI) Connect() {
	dg, err := discordgo.New("Bot " + d.Config.Token)
	if err != nil {
		Logger.Error("Unable to create discord connection", zap.Error(err))
		return
	}

	d.discord = dg
	dg.Open()
}

func (d *DiscordAPI) StartRoleHandlers(){
	d.listRoles()
	d.clearRoleChannel()
	d.createRoleMessages()
	d.discord.AddHandler(d.roleAssignmentHandler)
	Logger.Info("Role assignment handler started")
}

// Needs to be called to disconnect from discord
func (d *DiscordAPI) Shutdown() {
	d.discord.Close()
}

func (d DiscordAPI) PostStreamMessage(sm messaging.StreamMessage) error {
	if d.discord == nil {
		return fmt.Errorf("discord API not connected")
	}
	if d.Config.StreamChannelId == "" {
		return fmt.Errorf("stream channel id not configured")
	}
	author := discordgo.MessageEmbedAuthor{
		Name: fmt.Sprintf("%s is live!", sm.Username),
	}
	thumbnail := discordgo.MessageEmbedThumbnail{
		URL: sm.UserLogo,
	}
	footer := discordgo.MessageEmbedFooter{
		Text: fmt.Sprintf("%s | %s", sm.Platform, sm.Timestamp),
		IconURL: sm.PlatformLogo,
	}
	messageEmbed := discordgo.MessageEmbed{
		Description: sm.URL,
		Color: sm.PlatformColorInt,
		URL: sm.URL,
		Author: &author,
		Thumbnail: &thumbnail,
		Footer: &footer,
		Fields: []*discordgo.MessageEmbedField{
			{
				Name: "Game",
				Value: fmt.Sprintf("%s - %s", sm.Game, sm.Description),
				Inline: false,
			},
		},
	}
	_, err := d.discord.ChannelMessageSendEmbed(d.Config.StreamChannelId, &messageEmbed)
	return err
}

// removes all messages entered by this bot in the channel. Uses the ClientID of the bot/app
func (d *DiscordAPI) clearRoleChannel() {
	channelId := d.Config.RoleCfg.ChannelId

	messages, err := d.discord.ChannelMessages(channelId,50,"","","")
	if err != nil {
		Logger.Error("Unable to access channel messages", zap.String("channelId", channelId), zap.Error(err))
		return
	}
	for _, message := range messages {
		if message.Author.ID == d.Config.ClientId {
			err := d.discord.ChannelMessageDelete(channelId, message.ID)
			if err == nil {
				Logger.Info("Removed bot message", zap.String("messageId", message.ID))
			} else {
				Logger.Error("Unable to remove message", zap.String("messageId", message.ID), zap.Error(err))
			}

		}
	}
}

// creates messages in the channel and adds the emojis
func (d *DiscordAPI) createRoleMessages() {
	d.assignmentMsgs = make(map[string]map[string]string)
	for _, group := range d.Config.RoleCfg.EmojiRoleGroups {
		messageEmbed := discordgo.MessageEmbed{
			Title: group.MessageTitle,
			Description: group.MessageBody,
			Color: 15581239,
		}
		// create the message
		message, err := d.discord.ChannelMessageSendEmbed(d.Config.RoleCfg.ChannelId, &messageEmbed)
		if err != nil {
			Logger.Error("Unable to create message", zap.Error(err))
			continue
		}
		Logger.Info("Added role message", zap.String("message", group.MessageTitle), zap.String("messageId", message.ID))

		// add the emojis
		emojiRoles := make(map[string]string)
		for _, role := range group.Roles {
			emojiRoles[role.EmojiId] = role.RoleId
			err := d.discord.MessageReactionAdd(d.Config.RoleCfg.ChannelId, message.ID, role.EmojiId)
			if err != nil {
				Logger.Error("Unable to add emoji to message",
					zap.String("emojiId", role.EmojiId),
					zap.String("messageId", message.ID),
					zap.Error(err),
				)
			}
		}

		d.assignmentMsgs[message.ID] = emojiRoles
	}
}

func (d DiscordAPI) addRoleToUser(userId string, roleId string) {
	err := d.discord.GuildMemberRoleAdd(d.Config.GuildId, userId, roleId)
	if err != nil {
		Logger.Error("could not add role",
			zap.String("userId", userId),
			zap.String("roleId", roleId),
			zap.Error(err))
	} else {
		Logger.Info("added role to user",
			zap.String("userId", userId),
			zap.String("roleId", roleId),
		)
	}
}

func (d DiscordAPI) removeRoleFromUser(userId string, roleId string) {
	err := d.discord.GuildMemberRoleRemove(d.Config.GuildId, userId, roleId)
	if err != nil {
		Logger.Error("could not remove role", zap.Error(err))
	} else {
		Logger.Info("removed role from user",
			zap.String("userId", userId),
			zap.String("roleId", roleId),
		)
	}
}
func (d *DiscordAPI) listRoles() {
	roles, err := d.discord.GuildRoles(d.Config.GuildId)

	if err != nil {
		Logger.Error("Could not get roles", zap.Error(err))
		return
	}

	for i, role := range roles {
		Logger.Info(fmt.Sprintf("Role %d", i), zap.Any("role",role))
	}
}<|endoftext|>


<commit_before>package citadel

import (
	"crypto/tls"
	"fmt"
	"strings"
	"time"

	"github.com/samalba/dockerclient"
)

type Engine struct {
	ID     string   `json:"id,omitempty"`
	Addr   string   `json:"addr,omitempty"`
	Cpus   float64  `json:"cpus,omitempty"`
	Memory float64  `json:"memory,omitempty"`
	Labels []string `json:"labels,omitempty"`

	client       *dockerclient.DockerClient
	eventHandler EventHandler
}

func (e *Engine) Connect(config *tls.Config) error {
	c, err := dockerclient.NewDockerClient(e.Addr, config)
	if err != nil {
		return err
	}

	e.client = c

	return nil
}

func (e *Engine) SetClient(c *dockerclient.DockerClient) {
	e.client = c
}

// IsConnected returns true if the engine is connected to a remote docker API
func (e *Engine) IsConnected() bool {
	return e.client != nil
}

func (e *Engine) Start(c *Container, pullImage bool) error {
	var (
		err    error
		env    = []string{}
		client = e.client
		i      = c.Image
	)
	c.Engine = e

	for k, v := range i.Environment {
		env = append(env, fmt.Sprintf("%s=%s", k, v))
	}

	env = append(env,
		fmt.Sprintf("_citadel_type=%s", i.Type),
		fmt.Sprintf("_citadel_labels=%s", strings.Join(i.Labels, ",")),
	)

	vols := make(map[string]struct{})
	binds := []string{}
	for _, v := range i.Volumes {
		if strings.Index(v, ":") > -1 {
			cv := strings.Split(v, ":")
			binds = append(binds, v)
			v = cv[1]
		}
		vols[v] = struct{}{}
	}
	config := &dockerclient.ContainerConfig{
		Hostname:     i.Hostname,
		Domainname:   i.Domainname,
		Image:        i.Name,
		Cmd:          i.Args,
		Memory:       int(i.Memory) * 1024 * 1024,
		Env:          env,
		CpuShares:    int(i.Cpus * 100.0 / e.Cpus),
		ExposedPorts: make(map[string]struct{}),
		Volumes:      vols,
	}

	links := []string{}
	for k, v := range i.Links {
		links = append(links, fmt.Sprintf("%s:%s", k, v))
	}
	hostConfig := &dockerclient.HostConfig{
		PublishAllPorts: i.Publish,
		PortBindings:    make(map[string][]dockerclient.PortBinding),
		Links:           links,
		VolumesFrom:     i.VolumesFrom,
		Binds:           binds,
		RestartPolicy: dockerclient.RestartPolicy{
			Name:              i.RestartPolicy.Name,
			MaximumRetryCount: i.RestartPolicy.MaximumRetryCount,
		},
		NetworkMode: i.NetworkMode,
	}

	for _, b := range i.BindPorts {
		key := fmt.Sprintf("%d/%s", b.ContainerPort, b.Proto)
		config.ExposedPorts[key] = struct{}{}

		hostConfig.PortBindings[key] = []dockerclient.PortBinding{
			{
				HostPort: fmt.Sprint(b.Port),
			},
		}
	}

	imageInfo := parseImageName(i.Name)
	if pullImage {
		if err := client.PullImage(imageInfo.Name, imageInfo.Tag); err != nil {
			return err
		}
	}

	if c.ID, err = client.CreateContainer(config, c.Name); err != nil {
		return err
	}

	if err := client.StartContainer(c.ID, hostConfig); err != nil {
		return err
	}

	return e.updatePortInformation(c)
}

func (e *Engine) ListImages() ([]string, error) {
	images, err := e.client.ListImages()
	if err != nil {
		return nil, err
	}

	out := []string{}

	for _, i := range images {
		for _, t := range i.RepoTags {
			out = append(out, t)
		}
	}

	return out, nil
}

func (e *Engine) updatePortInformation(c *Container) error {
	info, err := e.client.InspectContainer(c.ID)
	if err != nil {
		return err
	}

	return parsePortInformation(info, c)
}

func (e *Engine) ListContainers(all bool) ([]*Container, error) {
	out := []*Container{}

	c, err := e.client.ListContainers(all)
	if err != nil {
		return nil, err
	}

	for _, ci := range c {
		cc, err := FromDockerContainer(ci.Id, ci.Image, e)
		if err != nil {
			return nil, err
		}

		out = append(out, cc)
	}

	return out, nil
}

func (e *Engine) Kill(container *Container, sig int) error {
	return e.client.KillContainer(container.ID)
}

func (e *Engine) Stop(container *Container) error {
	return e.client.StopContainer(container.ID, 8)
}

func (e *Engine) Restart(container *Container, timeout int) error {
	return e.client.RestartContainer(container.ID, timeout)
}

func (e *Engine) Remove(container *Container) error {
	return e.client.RemoveContainer(container.ID)
}

func (e *Engine) Events(h EventHandler) error {
	if e.eventHandler != nil {
		return fmt.Errorf("event handler already set")
	}
	e.eventHandler = h

	e.client.StartMonitorEvents(e.handler)

	return nil
}

func (e *Engine) String() string {
	return fmt.Sprintf("engine %s addr %s", e.ID, e.Addr)
}

func (e *Engine) handler(ev *dockerclient.Event, args ...interface{}) {
	event := &Event{
		Engine: e,
		Type:   ev.Status,
		Time:   time.Unix(int64(ev.Time), 0),
	}

	container, err := FromDockerContainer(ev.Id, ev.From, e)
	if err != nil {
		// TODO: un fuck this shit, fuckin handler
		return
	}

	event.Container = container

	e.eventHandler.Handle(event)
}
<commit_msg>remove VolumesFrom<commit_after>package citadel

import (
	"crypto/tls"
	"fmt"
	"strings"
	"time"

	"github.com/samalba/dockerclient"
)

type Engine struct {
	ID     string   `json:"id,omitempty"`
	Addr   string   `json:"addr,omitempty"`
	Cpus   float64  `json:"cpus,omitempty"`
	Memory float64  `json:"memory,omitempty"`
	Labels []string `json:"labels,omitempty"`

	client       *dockerclient.DockerClient
	eventHandler EventHandler
}

func (e *Engine) Connect(config *tls.Config) error {
	c, err := dockerclient.NewDockerClient(e.Addr, config)
	if err != nil {
		return err
	}

	e.client = c

	return nil
}

func (e *Engine) SetClient(c *dockerclient.DockerClient) {
	e.client = c
}

// IsConnected returns true if the engine is connected to a remote docker API
func (e *Engine) IsConnected() bool {
	return e.client != nil
}

func (e *Engine) Start(c *Container, pullImage bool) error {
	var (
		err    error
		env    = []string{}
		client = e.client
		i      = c.Image
	)
	c.Engine = e

	for k, v := range i.Environment {
		env = append(env, fmt.Sprintf("%s=%s", k, v))
	}

	env = append(env,
		fmt.Sprintf("_citadel_type=%s", i.Type),
		fmt.Sprintf("_citadel_labels=%s", strings.Join(i.Labels, ",")),
	)

	vols := make(map[string]struct{})
	binds := []string{}
	for _, v := range i.Volumes {
		if strings.Index(v, ":") > -1 {
			cv := strings.Split(v, ":")
			binds = append(binds, v)
			v = cv[1]
		}
		vols[v] = struct{}{}
	}
	config := &dockerclient.ContainerConfig{
		Hostname:     i.Hostname,
		Domainname:   i.Domainname,
		Image:        i.Name,
		Cmd:          i.Args,
		Memory:       int(i.Memory) * 1024 * 1024,
		Env:          env,
		CpuShares:    int(i.Cpus * 100.0 / e.Cpus),
		ExposedPorts: make(map[string]struct{}),
		Volumes:      vols,
	}

	links := []string{}
	for k, v := range i.Links {
		links = append(links, fmt.Sprintf("%s:%s", k, v))
	}
	hostConfig := &dockerclient.HostConfig{
		PublishAllPorts: i.Publish,
		PortBindings:    make(map[string][]dockerclient.PortBinding),
		Links:           links,
		Binds:           binds,
		RestartPolicy: dockerclient.RestartPolicy{
			Name:              i.RestartPolicy.Name,
			MaximumRetryCount: i.RestartPolicy.MaximumRetryCount,
		},
		NetworkMode: i.NetworkMode,
	}

	for _, b := range i.BindPorts {
		key := fmt.Sprintf("%d/%s", b.ContainerPort, b.Proto)
		config.ExposedPorts[key] = struct{}{}

		hostConfig.PortBindings[key] = []dockerclient.PortBinding{
			{
				HostPort: fmt.Sprint(b.Port),
			},
		}
	}

	imageInfo := parseImageName(i.Name)
	if pullImage {
		if err := client.PullImage(imageInfo.Name, imageInfo.Tag); err != nil {
			return err
		}
	}

	if c.ID, err = client.CreateContainer(config, c.Name); err != nil {
		return err
	}

	if err := client.StartContainer(c.ID, hostConfig); err != nil {
		return err
	}

	return e.updatePortInformation(c)
}

func (e *Engine) ListImages() ([]string, error) {
	images, err := e.client.ListImages()
	if err != nil {
		return nil, err
	}

	out := []string{}

	for _, i := range images {
		for _, t := range i.RepoTags {
			out = append(out, t)
		}
	}

	return out, nil
}

func (e *Engine) updatePortInformation(c *Container) error {
	info, err := e.client.InspectContainer(c.ID)
	if err != nil {
		return err
	}

	return parsePortInformation(info, c)
}

func (e *Engine) ListContainers(all bool) ([]*Container, error) {
	out := []*Container{}

	c, err := e.client.ListContainers(all)
	if err != nil {
		return nil, err
	}

	for _, ci := range c {
		cc, err := FromDockerContainer(ci.Id, ci.Image, e)
		if err != nil {
			return nil, err
		}

		out = append(out, cc)
	}

	return out, nil
}

func (e *Engine) Kill(container *Container, sig int) error {
	return e.client.KillContainer(container.ID)
}

func (e *Engine) Stop(container *Container) error {
	return e.client.StopContainer(container.ID, 8)
}

func (e *Engine) Restart(container *Container, timeout int) error {
	return e.client.RestartContainer(container.ID, timeout)
}

func (e *Engine) Remove(container *Container) error {
	return e.client.RemoveContainer(container.ID)
}

func (e *Engine) Events(h EventHandler) error {
	if e.eventHandler != nil {
		return fmt.Errorf("event handler already set")
	}
	e.eventHandler = h

	e.client.StartMonitorEvents(e.handler)

	return nil
}

func (e *Engine) String() string {
	return fmt.Sprintf("engine %s addr %s", e.ID, e.Addr)
}

func (e *Engine) handler(ev *dockerclient.Event, args ...interface{}) {
	event := &Event{
		Engine: e,
		Type:   ev.Status,
		Time:   time.Unix(int64(ev.Time), 0),
	}

	container, err := FromDockerContainer(ev.Id, ev.From, e)
	if err != nil {
		// TODO: un fuck this shit, fuckin handler
		return
	}

	event.Container = container

	e.eventHandler.Handle(event)
}
<|endoftext|>


<commit_before>/*
   bountyforcode - A bounty system for bug and feature fixes
   Copyright (C) 2013 Abhi Yerra <abhi@berkeley.edu>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as
   published by the Free Software Foundation, either version 3 of the
   License, or (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/
package main

import (
	"flag"
	"fmt"
	. "github.com/abhiyerra/bountyforcode/bounty/app/controllers"
	. "github.com/abhiyerra/bountyforcode/bounty/app/models"
	"github.com/abhiyerra/coinbase"
	"github.com/gorilla/mux"
	"log"
	"net/http"
)

var (
	domain string
)

func InitConfig() {
	flag.StringVar(&PostgresHost, "pghost", "", "the host for postgres")
	flag.StringVar(&PostgresDb, "dbname", "", "the db for postgres")
	flag.StringVar(&GithubClientId, "github_client_id", "", "github client id")
	flag.StringVar(&GithubClientSecret, "github_client_secret", "", "github client secret")
	flag.StringVar(&GithubRedirectUrl, "github_redirect_url", "", "github redirect url")
	flag.StringVar(&domain, "domain", "", "domain this is running on")
	flag.StringVar(&SecretStoreKey, "secret_store_key", "", "Secret session store key")
	flag.StringVar(&HtmlDir, "html_dir", "", "Location of the html files")
	flag.StringVar(&coinbase.CoinbaseApiKey, "coinbase_api_key", "", "Coinbase API Key")
	flag.StringVar(&coinbase.CoinbaseCallbackSecret, "coinbase_callback_secret", "", "Coinbase Callback Secret")

	flag.Parse()

	if coinbase.CoinbaseCallbackSecret == "" {
		log.Fatal("CoinbaseCallbackSecret can't be empty")
	}
}

func main() {
	InitConfig()
	InitDb()
	InitSessionStore()
	InitGithub()

	defer CloseDb()

	log.Printf("Server running on %s", domain)

	m := mux.NewRouter()
	m.HandleFunc("/v1/register/activate", RegisterAuthorizeHandler).Methods("GET") // TODO Should be authorize
	m.HandleFunc("/v1/register", RegisterHandler).Methods("GET")
	m.HandleFunc("/v1/user", UserSessionHandler).Methods("GET")
	m.HandleFunc("/v1/users/{id}", UserHandler).Methods("GET")
	m.HandleFunc("/v1/projects/{subdomain:[a-z]+}/issues", ProjectIssuesHandler).Methods("GET")
	m.HandleFunc("/v1/issues", IssuesHandler).Methods("GET")
	m.HandleFunc("/v1/issues", CreateIssueHandler).Methods("POST")
	m.HandleFunc("/v1/issues/{id}", ShowIssueHandler).Methods("GET")
	m.HandleFunc("/v1/issues/{id}/bounty", NewBountyHandler).Methods("GET")
	m.HandleFunc("/v1/issues/{id}/bounties", BountiesHandler).Methods("GET")
	m.HandleFunc(fmt.Sprintf("/v1/coinbase/%s", coinbase.CoinbaseCallbackSecret), CoinbaseCallbackHandler).Methods("POST")

	m.HandleFunc("/admin", AdminHandler).Methods("GET")

	http.Handle("/", m)
	http.ListenAndServe(":3000", nil)
}
<commit_msg>init is run by default<commit_after>/*
   bountyforcode - A bounty system for bug and feature fixes
   Copyright (C) 2013 Abhi Yerra <abhi@berkeley.edu>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as
   published by the Free Software Foundation, either version 3 of the
   License, or (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/
package main

import (
	"flag"
	"fmt"
	. "github.com/abhiyerra/bountyforcode/bounty/app/controllers"
	. "github.com/abhiyerra/bountyforcode/bounty/app/models"
	"github.com/abhiyerra/coinbase"
	"github.com/gorilla/mux"
	"log"
	"net/http"
)

var (
	domain string
)

func init() {
	flag.StringVar(&PostgresHost, "pghost", "", "the host for postgres")
	flag.StringVar(&PostgresDb, "dbname", "", "the db for postgres")
	flag.StringVar(&GithubClientId, "github_client_id", "", "github client id")
	flag.StringVar(&GithubClientSecret, "github_client_secret", "", "github client secret")
	flag.StringVar(&GithubRedirectUrl, "github_redirect_url", "", "github redirect url")
	flag.StringVar(&domain, "domain", "", "domain this is running on")
	flag.StringVar(&SecretStoreKey, "secret_store_key", "", "Secret session store key")
	flag.StringVar(&HtmlDir, "html_dir", "", "Location of the html files")
	flag.StringVar(&coinbase.CoinbaseApiKey, "coinbase_api_key", "", "Coinbase API Key")
	flag.StringVar(&coinbase.CoinbaseCallbackSecret, "coinbase_callback_secret", "", "Coinbase Callback Secret")

	flag.Parse()

	if coinbase.CoinbaseCallbackSecret == "" {
		log.Fatal("CoinbaseCallbackSecret can't be empty")
	}
}

func main() {
	InitDb()
	InitSessionStore()
	InitGithub()

	defer CloseDb()

	log.Printf("Server running on %s", domain)

	m := mux.NewRouter()
	m.HandleFunc("/v1/register/activate", RegisterAuthorizeHandler).Methods("GET") // TODO Should be authorize
	m.HandleFunc("/v1/register", RegisterHandler).Methods("GET")
	m.HandleFunc("/v1/user", UserSessionHandler).Methods("GET")
	m.HandleFunc("/v1/users/{id}", UserHandler).Methods("GET")
	m.HandleFunc("/v1/projects/{subdomain:[a-z]+}/issues", ProjectIssuesHandler).Methods("GET")
	m.HandleFunc("/v1/issues", IssuesHandler).Methods("GET")
	m.HandleFunc("/v1/issues", CreateIssueHandler).Methods("POST")
	m.HandleFunc("/v1/issues/{id}", ShowIssueHandler).Methods("GET")
	m.HandleFunc("/v1/issues/{id}/bounty", NewBountyHandler).Methods("GET")
	m.HandleFunc("/v1/issues/{id}/bounties", BountiesHandler).Methods("GET")
	m.HandleFunc(fmt.Sprintf("/v1/coinbase/%s", coinbase.CoinbaseCallbackSecret), CoinbaseCallbackHandler).Methods("POST")

	m.HandleFunc("/admin", AdminHandler).Methods("GET")

	http.Handle("/", m)
	http.ListenAndServe(":3000", nil)
}
<|endoftext|>


<commit_before>package atlas

import "errors"

// ErrInvalidMeasurementType is a new error
var ErrInvalidMeasurementType = errors.New("invalid measurement type")

// ErrInvalidAPIKey is returned when the key is invalid
var ErrInvalidAPIKey = errors.New("invalid API key")

// ErrAPIKeyIsMandatory is returned when a call need one
var ErrAPIKeyIsMandatory = errors.New("API call requires an API key")
<commit_msg>APIError is an error.<commit_after>package atlas

import (
	"github.com/pkg/errors"
)

// ErrInvalidMeasurementType is a new error
var ErrInvalidMeasurementType = errors.New("invalid measurement type")

// ErrInvalidAPIKey is returned when the key is invalid
var ErrInvalidAPIKey = errors.New("invalid API key")

// ErrAPIKeyIsMandatory is returned when a call need one
var ErrAPIKeyIsMandatory = errors.New("API call requires an API key")

// ErrInvalidAPIAnswer
var ErrInvalidError = errors.New("Invalid request")

var lastError APIError

func (e APIError) Error() string {
	return e.Err.Detail
}
<|endoftext|>


<commit_before>package gads

import (
	"encoding/xml"
	"fmt"
	"strings"
)

type OperationError struct {
	Code      int64  `xml:"OperationError>Code"`
	Details   string `xml:"OperationError>Details"`
	ErrorCode string `xml:"OperationError>ErrorCode"`
	Message   string `xml:"OperationError>Message"`
}

type BudgetError struct {
	Path    string `xml:"fieldPath"`
	String  string `xml:"errorString"`
	Trigger string `xml:"trigger"`
	Reason  string `xml:"reason"`
}

type CriterionError struct {
	FieldPath   string `xml:"fieldPath"`
	Trigger     string `xml:"trigger"`
	ErrorString string `xml:"errorString"`
	Reason      string `xml:"reason"`
}

type TargetError struct {
	FieldPath   string `xml:"fieldPath"`
	Trigger     string `xml:"trigger"`
	ErrorString string `xml:"errorString"`
	Reason      string `xml:"reason"`
}

type AdGroupServiceError struct {
	FieldPath   string `xml:"fieldPath"`
	Trigger     string `xml:"trigger"`
	ErrorString string `xml:"errorString"`
	Reason      string `xml:"reason"`
}

type NotEmptyError struct {
	FieldPath   string `xml:"fieldPath"`
	Trigger     string `xml:"trigger"`
	ErrorString string `xml:"errorString"`
	Reason      string `xml:"reason"`
}

type AdError struct {
	FieldPath   string `xml:"fieldPath"`
	Trigger     string `xml:"trigger"`
	ErrorString string `xml:"errorString"`
	Reason      string `xml:"reason"`
}

type ApiExceptionFault struct {
	Message string        `xml:"message"`
	Type    string        `xml:"ApplicationException.Type"`
	Errors  []interface{} `xml:"errors"`
}

func (aes *ApiExceptionFault) UnmarshalXML(dec *xml.Decoder, start xml.StartElement) (err error) {
	for token, err := dec.Token(); err == nil; token, err = dec.Token() {
		switch start := token.(type) {
		case xml.StartElement:
			switch start.Name.Local {
			case "message":
				if err := dec.DecodeElement(&aes.Message, &start); err != nil {
					return err
				}
			case "ApplicationException.Type":
				if err := dec.DecodeElement(&aes.Type, &start); err != nil {
					return err
				}
			case "errors":
				errorType, _ := findAttr(start.Attr, xml.Name{Space: "http://www.w3.org/2001/XMLSchema-instance", Local: "type"})
				switch errorType {
				case "CriterionError":
					e := CriterionError{}
					dec.DecodeElement(&e, &start)
					aes.Errors = append(aes.Errors, e)
				case "TargetError":
					e := TargetError{}
					dec.DecodeElement(&e, &start)
					aes.Errors = append(aes.Errors, e)
				case "BudgetError":
					e := BudgetError{}
					dec.DecodeElement(&e, &start)
					aes.Errors = append(aes.Errors, e)
				case "AdGroupServiceError":
					e := AdGroupServiceError{}
					dec.DecodeElement(&e, &start)
					aes.Errors = append(aes.Errors, e)
				case "NotEmptyError":
					e := NotEmptyError{}
					dec.DecodeElement(&e, &start)
					aes.Errors = append(aes.Errors, e)
				case "AdError":
					e := AdError{}
					dec.DecodeElement(&e, &start)
					aes.Errors = append(aes.Errors, e)
				default:
					return fmt.Errorf("Unknown error type -> %s", start)
				}
			case "reason":
				break
			default:
				return fmt.Errorf("Unknown error field -> %s", start)
			}
		}
	}
	return err
}

type ErrorsType struct {
	ApiExceptionFaults []ApiExceptionFault `xml:"ApiExceptionFault"`
}

type Fault struct {
	XMLName     xml.Name   `xml:"Fault"`
	FaultCode   string     `xml:"faultcode"`
	FaultString string     `xml:"faultstring"`
	Errors      ErrorsType `xml:"detail"`
}

func (f *ErrorsType) Error() string {
	errors := []string{}
	for _, e := range f.ApiExceptionFaults {
		errors = append(errors, fmt.Sprintf("%s", e.Message))
	}
	return strings.Join(errors, "\n")
}
<commit_msg>Adding RateExceeded Error<commit_after>package gads

import (
	"encoding/xml"
	"fmt"
	"strings"
)

type OperationError struct {
	Code      int64  `xml:"OperationError>Code"`
	Details   string `xml:"OperationError>Details"`
	ErrorCode string `xml:"OperationError>ErrorCode"`
	Message   string `xml:"OperationError>Message"`
}

type BudgetError struct {
	Path    string `xml:"fieldPath"`
	String  string `xml:"errorString"`
	Trigger string `xml:"trigger"`
	Reason  string `xml:"reason"`
}

type CriterionError struct {
	FieldPath   string `xml:"fieldPath"`
	Trigger     string `xml:"trigger"`
	ErrorString string `xml:"errorString"`
	Reason      string `xml:"reason"`
}

type TargetError struct {
	FieldPath   string `xml:"fieldPath"`
	Trigger     string `xml:"trigger"`
	ErrorString string `xml:"errorString"`
	Reason      string `xml:"reason"`
}

type AdGroupServiceError struct {
	FieldPath   string `xml:"fieldPath"`
	Trigger     string `xml:"trigger"`
	ErrorString string `xml:"errorString"`
	Reason      string `xml:"reason"`
}

type NotEmptyError struct {
	FieldPath   string `xml:"fieldPath"`
	Trigger     string `xml:"trigger"`
	ErrorString string `xml:"errorString"`
	Reason      string `xml:"reason"`
}

type AdError struct {
	FieldPath   string `xml:"fieldPath"`
	Trigger     string `xml:"trigger"`
	ErrorString string `xml:"errorString"`
	Reason      string `xml:"reason"`
}

// if you exceed the quota given by google
type RateExceededError struct {
	RateName          string `xml:"rateName"`  // For example OperationsByMinute
	RateScope         string `xml:"rateScope"` // ACCOUNT or DEVELOPER
	ErrorString       string `xml:"errorString"`
	Reason            string `xml:"reason"`
	RetryAfterSeconds uint   `xml:"retryAfterSeconds"` // Try again in...
}

type ApiExceptionFault struct {
	Message string        `xml:"message"`
	Type    string        `xml:"ApplicationException.Type"`
	Errors  []interface{} `xml:"errors"`
}

func (aes *ApiExceptionFault) UnmarshalXML(dec *xml.Decoder, start xml.StartElement) (err error) {
	for token, err := dec.Token(); err == nil; token, err = dec.Token() {
		switch start := token.(type) {
		case xml.StartElement:
			switch start.Name.Local {
			case "message":
				if err := dec.DecodeElement(&aes.Message, &start); err != nil {
					return err
				}
			case "ApplicationException.Type":
				if err := dec.DecodeElement(&aes.Type, &start); err != nil {
					return err
				}
			case "errors":
				errorType, _ := findAttr(start.Attr, xml.Name{Space: "http://www.w3.org/2001/XMLSchema-instance", Local: "type"})
				switch errorType {
				case "CriterionError":
					e := CriterionError{}
					dec.DecodeElement(&e, &start)
					aes.Errors = append(aes.Errors, e)
				case "TargetError":
					e := TargetError{}
					dec.DecodeElement(&e, &start)
					aes.Errors = append(aes.Errors, e)
				case "BudgetError":
					e := BudgetError{}
					dec.DecodeElement(&e, &start)
					aes.Errors = append(aes.Errors, e)
				case "AdGroupServiceError":
					e := AdGroupServiceError{}
					dec.DecodeElement(&e, &start)
					aes.Errors = append(aes.Errors, e)
				case "NotEmptyError":
					e := NotEmptyError{}
					dec.DecodeElement(&e, &start)
					aes.Errors = append(aes.Errors, e)
				case "AdError":
					e := AdError{}
					dec.DecodeElement(&e, &start)
					aes.Errors = append(aes.Errors, e)
				case "RateExceededError":
					e := RateExceededError{}
					dec.DecodeElement(&e, &start)
					aes.Errors = append(aes.Errors, e)
				default:
					return fmt.Errorf("Unknown error type -> %s", start)
				}
			case "reason":
				break
			default:
				return fmt.Errorf("Unknown error field -> %s", start)
			}
		}
	}
	return err
}

type ErrorsType struct {
	ApiExceptionFaults []ApiExceptionFault `xml:"ApiExceptionFault"`
}

func (f *ErrorsType) Error() string {
	errors := []string{}
	for _, e := range f.ApiExceptionFaults {
		errors = append(errors, fmt.Sprintf("%s", e.Message))
	}
	return strings.Join(errors, "\n")
}

type Fault struct {
	XMLName     xml.Name   `xml:"Fault"`
	FaultCode   string     `xml:"faultcode"`
	FaultString string     `xml:"faultstring"`
	Errors      ErrorsType `xml:"detail"`
}

func (f Fault) Error() string {
	return f.FaultString + " - " + f.Errors.Error()
}
<|endoftext|>


<commit_before>package merr

import (
	"fmt"
)

type Errors interface {
	Errors() []error
}

type multiError struct {
	errs []error
}

func New(errs ...error) error {
	if len(errs) == 0 {
		return nil
	}

	return &multiError{errs}
}

func (e *multiError) Error() string {
	errs := e.errs
	tail := "..."

	switch len(errs) {
	case 0:
		return ""
	case 1:
		tail = ""
	}

	return fmt.Sprint("errors: ", len(errs), errs[0], tail)
}

func (e *multiError) Errors() []error {
	return e.errs
}
<commit_msg>package comment<commit_after>// multiple error
package merr

import (
	"fmt"
)

type Errors interface {
	Errors() []error
}

type multiError struct {
	errs []error
}

func New(errs ...error) error {
	if len(errs) == 0 {
		return nil
	}

	return &multiError{errs}
}

func (e *multiError) Error() string {
	errs := e.errs
	tail := "..."

	switch len(errs) {
	case 0:
		return ""
	case 1:
		tail = ""
	}

	return fmt.Sprint("errors: ", len(errs), errs[0], tail)
}

func (e *multiError) Errors() []error {
	return e.errs
}
<|endoftext|>


<commit_before>package synapse

import (
	"log"
	"net"
	"syscall"
	"time"
)

// This file contains the error handlers
// for dial errors and network i/o errors.
// The goal here is to determine which
// errors warrant permanently dropping
// a connection vs. continually re-dialing.

const (
	// starting redial wait, in seconds
	startWait = 3 * time.Second

	// maximum redial wait, in seconds;
	// after backoff reaches this point,
	// we drop the connection permanently
	maxWait = 600 * time.Second
)

// handleDialError determines whether or not
// it should continue to dial the remote address or
// drop it permanently. doing nothing ignores the error.
func (c *clusterClient) handleDialError(addr string, err error) {
	// simplest case: temporary net error
	if neterr, ok := err.(net.Error); ok {
		if neterr.Temporary() {
			go c.redialPauseLoop(addr)
			return
		}
	}

	if operr, ok := err.(*net.OpError); ok {
		if _, ok := operr.Err.(syscall.Errno); ok {
			err = operr.Err
			goto errno
		}
	}

	// TODO: validate that these are
	// actually the responses we want
	// to have under these conditions.
errno:
	if errno, ok := err.(syscall.Errno); ok {
		if !errno.Temporary() {
			switch errno {
			// we don't need to handle
			// the "drop permanently" case,
			// because that is the default behavior.

			// redial w/ backoff
			case syscall.EPIPE, syscall.ESHUTDOWN, syscall.ESTALE, syscall.ETIMEDOUT, syscall.ECONNREFUSED:
				go c.redialPauseLoop(addr)
				return

			}
		}
	}

	log.Printf("synapse cluster: permanently dropping remote @ %s %s: %s", c.nwk, addr, err)
}

// redialPauseLoop is called to wait/loop on dialing
// a problematic address
func (c *clusterClient) redialPauseLoop(addr string) {
	// TODO: smarter backoff
	//
	// right now we start at 3s and double until 600s, then break

	wait := startWait
	for {
		if wait > maxWait {
			log.Printf("synapse cluster: permanently dropping remote @ %s %s", c.nwk, addr)
			return
		}
		log.Printf("synapse cluster: redialing %s in %s", addr, wait)
		time.Sleep(wait)
		err := c.dial(c.nwk, addr, false)
		if err == nil {
			break
		}
		log.Printf("synapse cluster: error dialing %s: %s", addr, err)
		wait *= 2
	}
}

// handleErr is responsible for figuring out
// if an error warrants re-dialing a connection
func (c *clusterClient) handleErr(v *client, err error) {

	// ignore timeouts and user errors
	// from this package; they don't
	// indicate connection failure
	switch err {
	case nil, ErrTimeout, ErrTooLarge:
		return
	case ErrClosed:
		go c.redial(v)
		return
	}

	// ignore protocol-level errors
	if _, ok := err.(Status); ok {
		return
	}

	// most syscall-level errors are
	// returned as OpErrors by the net pkg
	if operr, ok := err.(*net.OpError); ok {
		if _, ok = operr.Err.(syscall.Errno); ok {
			err = operr.Err
			goto errno
		}
		if !operr.Temporary() {
			go c.redial(v)
			return
		}
		return
	}

	// this is the logic for handling
	// syscall-level errors; it is not
	// complete or exhaustive
errno:
	if errno, ok := err.(syscall.Errno); ok {

		// errors that are "temporary"
		// - EINTR (interrupted)
		// - EMFILE (out of file descriptors)
		// - ECONNRESET (connection reset)
		// - ECONNABORTED (connection aborted...?)
		// - EAGAIN (virtually never seen by clients of "net")
		// - EWOULDBLOCK (see above; POSIX says it may be identical to EAGAIN)
		// - ETIMEOUT (timeout)
		if !errno.Temporary() {
			switch errno {

			// errors for which we permanently drop the remote
			// TODO: validate
			case syscall.EACCES, syscall.EADDRINUSE, syscall.EADDRNOTAVAIL,
				syscall.EAFNOSUPPORT, syscall.EBADMSG, syscall.EDQUOT, syscall.EFAULT,
				syscall.EOPNOTSUPP, syscall.ESOCKTNOSUPPORT:
				if c.remove(v) {
					log.Printf("synapse cluster: permanently dropping remote @ %s: %s", v.conn.RemoteAddr(), err)
				}
				v.Close()
				return

			// redial loop
			case syscall.EHOSTDOWN, syscall.EHOSTUNREACH, syscall.EPIPE, syscall.ESHUTDOWN:

				// ensure idempotency
				if c.remove(v) {
					v.Close()
					go c.redialPauseLoop(v.conn.RemoteAddr().String())
				}

			}
		}
	}
}
<commit_msg>don't redial on intentional closes...<commit_after>package synapse

import (
	"log"
	"net"
	"syscall"
	"time"
)

// This file contains the error handlers
// for dial errors and network i/o errors.
// The goal here is to determine which
// errors warrant permanently dropping
// a connection vs. continually re-dialing.

const (
	// starting redial wait, in seconds
	startWait = 3 * time.Second

	// maximum redial wait, in seconds;
	// after backoff reaches this point,
	// we drop the connection permanently
	maxWait = 600 * time.Second
)

// handleDialError determines whether or not
// it should continue to dial the remote address or
// drop it permanently. doing nothing ignores the error.
func (c *clusterClient) handleDialError(addr string, err error) {
	// simplest case: temporary net error
	if neterr, ok := err.(net.Error); ok {
		if neterr.Temporary() {
			go c.redialPauseLoop(addr)
			return
		}
	}

	if operr, ok := err.(*net.OpError); ok {
		if _, ok := operr.Err.(syscall.Errno); ok {
			err = operr.Err
			goto errno
		}
	}

	// TODO: validate that these are
	// actually the responses we want
	// to have under these conditions.
errno:
	if errno, ok := err.(syscall.Errno); ok {

		// the Temporary() case
		// is already taken care of, so
		// we only need to handle
		// the non-temporary cases
		switch errno {

		// redial w/ backoff
		case syscall.EPIPE, syscall.ESHUTDOWN, syscall.ESTALE, syscall.ETIMEDOUT, syscall.ECONNREFUSED:
			go c.redialPauseLoop(addr)
			return

		}

	}

	// if we got here, we're dropping the connection
	log.Printf("synapse cluster: permanently dropping remote @ %s %s: %s", c.nwk, addr, err)
}

// redialPauseLoop is called to wait/loop on dialing
// a problematic address
func (c *clusterClient) redialPauseLoop(addr string) {
	// TODO: smarter backoff
	//
	// right now we start at 3s and double until 600s, then break

	wait := startWait
	for {
		if wait > maxWait {
			log.Printf("synapse cluster: permanently dropping remote @ %s %s", c.nwk, addr)
			return
		}
		log.Printf("synapse cluster: redialing %s in %s", addr, wait)
		time.Sleep(wait)
		err := c.dial(c.nwk, addr, false)
		if err == nil {
			break
		}
		log.Printf("synapse cluster: error dialing %s: %s", addr, err)
		wait *= 2
	}
}

// handleErr is responsible for figuring out
// if an error warrants re-dialing a connection
func (c *clusterClient) handleErr(v *client, err error) {

	// ignore timeouts and user errors
	// from this package; they don't
	// indicate connection failure
	switch err {
	case nil, ErrTimeout, ErrTooLarge, ErrClosed:
		return
	}

	// ignore protocol-level errors
	if _, ok := err.(Status); ok {
		return
	}

	// ignore "temporary" errors
	if neterr, ok := err.(net.Error); ok {
		if neterr.Temporary() {
			return
		}
	}

	// most syscall-level errors are
	// returned as OpErrors by the net pkg
	if operr, ok := err.(*net.OpError); ok {
		if _, ok = operr.Err.(syscall.Errno); ok {
			err = operr.Err
			goto errno
		}
	}

	// this is the logic for handling
	// syscall-level errors; it is not
	// complete or exhaustive
errno:
	if errno, ok := err.(syscall.Errno); ok {

		// errors that are "temporary" according to the stdlib:
		// - EINTR (interrupted)
		// - EMFILE (out of file descriptors)
		// - ECONNRESET (connection reset)
		// - ECONNABORTED (connection aborted...?)
		// - EAGAIN (virtually never seen by clients of "net")
		// - EWOULDBLOCK (see above; POSIX says it may be identical to EAGAIN)
		// - ETIMEOUT (timeout)

		switch errno {

		// errors for which we permanently drop the remote
		// TODO: validate
		case syscall.EACCES, syscall.EADDRNOTAVAIL, syscall.EPROTOTYPE,
			syscall.EAFNOSUPPORT, syscall.EBADMSG, syscall.EDQUOT, syscall.EFAULT,
			syscall.EOPNOTSUPP, syscall.ENOSR:
			if c.remove(v) {
				log.Printf("synapse cluster: permanently dropping remote @ %s: %s", v.conn.RemoteAddr(), err)
			}
			v.Close()
			return

		// redial loop
		case syscall.EHOSTDOWN, syscall.EHOSTUNREACH, syscall.EPIPE, syscall.ESHUTDOWN, syscall.EIO, syscall.ENETRESET:

			// ensure idempotency
			if c.remove(v) {
				v.Close()
				go c.redialPauseLoop(v.conn.RemoteAddr().String())
			}

		}
	}
}
<|endoftext|>


<commit_before>package records

import (
	"math/rand"
	"strconv"
	"testing"
)

// BenchmarkInsertRR *only* tests insertRR, not the taskRecord funcs.
func BenchmarkInsertRR(b *testing.B) {
	const (
		clusterSize = 1000
		appCount    = 5
	)
	var (
		slaves = make([]string, clusterSize)
		apps   = make([]string, appCount)
		rg     = &RecordGenerator{
			As:   rrs{},
			SRVs: rrs{},
		}
	)
	for i := 0; i < clusterSize; i++ {
		slaves[i] = "slave" + strconv.Itoa(i)
	}
	for i := 0; i < appCount; i++ {
		apps[i] = "app" + strconv.Itoa(i)
	}

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		var (
			si = rand.Int31n(clusterSize)
			ai = rand.Int31n(appCount)
		)
		rg.insertRR(apps[ai], slaves[si], "A")
	}
}
<commit_msg>added BenchmarkTaskRecord_withoutDiscoveryInfo benchmark<commit_after>package records

import (
	"math/rand"
	"strconv"
	"testing"

	"github.com/mesosphere/mesos-dns/records/labels"
	"github.com/mesosphere/mesos-dns/records/state"
)

// BenchmarkInsertRR *only* tests insertRR, not the taskRecord funcs.
func BenchmarkInsertRR(b *testing.B) {
	const (
		clusterSize = 1000
		appCount    = 5
	)
	var (
		slaves = make([]string, clusterSize)
		apps   = make([]string, appCount)
		rg     = &RecordGenerator{
			As:   rrs{},
			SRVs: rrs{},
		}
	)
	for i := 0; i < clusterSize; i++ {
		slaves[i] = "slave" + strconv.Itoa(i)
	}
	for i := 0; i < appCount; i++ {
		apps[i] = "app" + strconv.Itoa(i)
	}

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		var (
			si = rand.Int31n(clusterSize)
			ai = rand.Int31n(appCount)
		)
		rg.insertRR(apps[ai], slaves[si], "A")
	}
}

func BenchmarkTaskRecord_withoutDiscoveryInfo(b *testing.B) {
	const (
		clusterSize = 1000
		taskCount   = 1000
	)
	type params struct {
		task      state.Task
		f         state.Framework
		domain    string
		spec      labels.Func
		ipSources []string
		enumFW    EnumerableFramework
		rg        RecordGenerator
	}
	var (
		initialState = params{
			task: state.Task{
				State: "TASK_RUNNING",
			},
			f:         state.Framework{Name: "foo"},
			domain:    "mesos",
			spec:      labels.RFC1123,
			ipSources: []string{"host"},
			rg: RecordGenerator{
				As:   rrs{},
				SRVs: rrs{},
			},
		}
		slaves = make([]string, clusterSize)
		tasks  = make([]string, taskCount)
	)
	for i := 0; i < clusterSize; i++ {
		slaves[i] = "slave" + strconv.Itoa(i)
	}
	for i := 0; i < taskCount; i++ {
		tasks[i] = "task" + strconv.Itoa(i)
	}
	tt := initialState
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		var (
			si = rand.Int31n(clusterSize)
			ti = rand.Int31n(taskCount)
		)
		tt.task.Name = tasks[ti]
		tt.task.SlaveIP = slaves[si]
		tt.task.SlaveID = "ID-" + slaves[si]
		tt.rg.taskRecord(tt.task, tt.f, tt.domain, tt.spec, tt.ipSources, &tt.enumFW)
	}
}
<|endoftext|>


<commit_before>package sigserver

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"
)

import (
	"github.com/hashicorp/errwrap"
	"github.com/robfig/cron"
)

import (
	"github.com/dekobon/clamav-mirror/sigupdate"
	"github.com/dekobon/clamav-mirror/utils"
	"io"
	"path/filepath"
)

var githash = "unknown"
var buildstamp = "unknown"
var appversion = "unknown"

var logger *log.Logger
var logFatal *log.Logger
var dataDirectory string

func init() {
	logger = log.New(os.Stdout, "", log.LstdFlags)
	logFatal = log.New(os.Stderr, "", log.LstdFlags|log.Lshortfile)
}

// RunUpdaterAndServer is the functional entry point to the application. This
// function starts the HTTP server and the periodic task executor.
func RunUpdaterAndServer(verboseMode bool, dataFilePath string, downloadMirrorURL string,
	diffCountThreshold uint16, port uint16, refreshHourInterval uint16) error {

	dataDirectory = dataFilePath

	{
		err := scheduleUpdates(verboseMode, dataFilePath, downloadMirrorURL,
			diffCountThreshold, refreshHourInterval)

		if err != nil {
			return errwrap.Wrapf("Error scheduling periodic updates. {{err}}", err)
		}
	}

	{
		err := runServer(port)

		if err != nil {
			return errwrap.Wrapf("Error running HTTP server. {{err}}", err)
		}
	}

	return nil
}

func runServer(port uint16) error {
	listenAddr := ":" + strconv.Itoa(int(port))
	logger.Printf("Starting ClamAV signature mirror HTTP server on port [%v]",
		listenAddr)
	http.HandleFunc("/", handler)
	http.ListenAndServe(listenAddr, nil)

	return nil
}

func scheduleUpdates(verboseMode bool, dataFilePath string, downloadMirrorURL string,
	diffCountThreshold uint16, refreshHourInterval uint16) error {

	cronSchedule := fmt.Sprintf("0 */%v * * *", refreshHourInterval)

	run := func() {
		err := sigupdate.RunSignatureUpdate(verboseMode, dataFilePath,
			downloadMirrorURL, diffCountThreshold)

		if err != nil {
			logger.Println(err)
		}
	}

	// Update once before scheduling
	run()

	c := cron.New()
	c.AddFunc(cronSchedule, run)
	c.Start()

	return nil
}

func validFileRequested(path string, file string) bool {
	dir := filepath.Dir(path)
	validFileExtension := (strings.HasSuffix(file, ".cvd") || strings.HasSuffix(file, ".cdiff")) &&
		!strings.Contains(file, "..")
	validDir := dir == "/"
	// CVD or cdiff filenames should never be very big
	validFilename := len(file) < 48 && len(file) > 4

	return validDir && validFileExtension && validFilename
}

func handler(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Server", "ClamAV Mirror")
	w.Header().Add("X-Robots-Tag", "noindex, nofollow, noarchive")

	// Outright reject large paths before doing any processing
	if len(r.URL.Path) > 128 {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	path := filepath.Clean(r.URL.Path)
	file := filepath.Base(path)

	// If a non-signature file extension was requested, we just 404
	if !validFileRequested(path, file) {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	dataFilePath := dataDirectory + string(filepath.Separator) + file
	fileExists := utils.Exists(dataFilePath)

	if !fileExists {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	stat, err := os.Stat(dataFilePath)

	if err != nil {
		logger.Printf("Error running stat on file [%v]. %v", dataFilePath, err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.Header().Set("Last-Modified", stat.ModTime().UTC().Format(http.TimeFormat))
	w.Header().Set("Content-Type", "application/octet-stream")

	modifiedSince, err := http.ParseTime(r.Header.Get("If-Modified-Since"))

	if err != nil {
		logger.Printf("Couldn't parse time value [%v]. %v", r.Header.Get("If-Modified-Since"), err)
	}

	if !(r.Method == "GET" || r.Method == "HEAD") {
		logger.Printf("[%v] {%v} %v DENIED", r.Method, r.RemoteAddr, r.URL)
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}

	if modifiedSince.After(stat.ModTime()) || modifiedSince.Equal(stat.ModTime()) {
		w.WriteHeader(http.StatusNotModified)
	}

	logger.Printf("[%v] {%v} %v --> %v", r.Method, r.RemoteAddr, r.URL, dataFilePath)

	if r.Method == "GET" {
		dataFileReader, err := os.Open(dataFilePath)
		defer dataFileReader.Close()

		if err != nil {
			logger.Printf("Error reading [%v] from disk. %v",
				dataFilePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			return
		}

		io.Copy(w, dataFileReader)
	}

	w.WriteHeader(http.StatusOK)
}
<commit_msg>Fixed hourly periodic task definition.<commit_after>package sigserver

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"
)

import (
	"github.com/hashicorp/errwrap"
	"github.com/robfig/cron"
)

import (
	"github.com/dekobon/clamav-mirror/sigupdate"
	"github.com/dekobon/clamav-mirror/utils"
	"io"
	"path/filepath"
)

var githash = "unknown"
var buildstamp = "unknown"
var appversion = "unknown"

var logger *log.Logger
var logFatal *log.Logger
var dataDirectory string

func init() {
	logger = log.New(os.Stdout, "", log.LstdFlags)
	logFatal = log.New(os.Stderr, "", log.LstdFlags|log.Lshortfile)
}

// RunUpdaterAndServer is the functional entry point to the application. This
// function starts the HTTP server and the periodic task executor.
func RunUpdaterAndServer(verboseMode bool, dataFilePath string, downloadMirrorURL string,
	diffCountThreshold uint16, port uint16, refreshHourInterval uint16) error {

	dataDirectory = dataFilePath

	{
		err := scheduleUpdates(verboseMode, dataFilePath, downloadMirrorURL,
			diffCountThreshold, refreshHourInterval)

		if err != nil {
			return errwrap.Wrapf("Error scheduling periodic updates. {{err}}", err)
		}
	}

	{
		err := runServer(port)

		if err != nil {
			return errwrap.Wrapf("Error running HTTP server. {{err}}", err)
		}
	}

	return nil
}

func runServer(port uint16) error {
	listenAddr := ":" + strconv.Itoa(int(port))
	logger.Printf("Starting ClamAV signature mirror HTTP server on port [%v]",
		listenAddr)
	http.HandleFunc("/", handler)
	http.ListenAndServe(listenAddr, nil)

	return nil
}

func scheduleUpdates(verboseMode bool, dataFilePath string, downloadMirrorURL string,
	diffCountThreshold uint16, refreshHourInterval uint16) error {

	cronSchedule := fmt.Sprintf("@every %dh", refreshHourInterval)

	run := func() {
		err := sigupdate.RunSignatureUpdate(verboseMode, dataFilePath,
			downloadMirrorURL, diffCountThreshold)

		if err != nil {
			logger.Println(err)
		}
	}

	// Update once before scheduling
	run()

	c := cron.New()
	c.AddFunc(cronSchedule, run)
	c.Start()

	return nil
}

func validFileRequested(path string, file string) bool {
	dir := filepath.Dir(path)
	validFileExtension := (strings.HasSuffix(file, ".cvd") || strings.HasSuffix(file, ".cdiff")) &&
		!strings.Contains(file, "..")
	validDir := dir == "/"
	// CVD or cdiff filenames should never be very big
	validFilename := len(file) < 48 && len(file) > 4

	return validDir && validFileExtension && validFilename
}

func handler(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Server", "ClamAV Mirror")
	w.Header().Add("X-Robots-Tag", "noindex, nofollow, noarchive")

	// Outright reject large paths before doing any processing
	if len(r.URL.Path) > 128 {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	path := filepath.Clean(r.URL.Path)
	file := filepath.Base(path)

	// If a non-signature file extension was requested, we just 404
	if !validFileRequested(path, file) {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	dataFilePath := dataDirectory + string(filepath.Separator) + file
	fileExists := utils.Exists(dataFilePath)

	if !fileExists {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	stat, err := os.Stat(dataFilePath)

	if err != nil {
		logger.Printf("Error running stat on file [%v]. %v", dataFilePath, err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.Header().Set("Last-Modified", stat.ModTime().UTC().Format(http.TimeFormat))
	w.Header().Set("Content-Type", "application/octet-stream")

	modifiedSince, err := http.ParseTime(r.Header.Get("If-Modified-Since"))

	if err != nil {
		logger.Printf("Couldn't parse time value [%v]. %v", r.Header.Get("If-Modified-Since"), err)
	}

	if !(r.Method == "GET" || r.Method == "HEAD") {
		logger.Printf("[%v] {%v} %v DENIED", r.Method, r.RemoteAddr, r.URL)
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}

	if modifiedSince.After(stat.ModTime()) || modifiedSince.Equal(stat.ModTime()) {
		w.WriteHeader(http.StatusNotModified)
	}

	logger.Printf("[%v] {%v} %v --> %v", r.Method, r.RemoteAddr, r.URL, dataFilePath)

	if r.Method == "GET" {
		dataFileReader, err := os.Open(dataFilePath)
		defer dataFileReader.Close()

		if err != nil {
			logger.Printf("Error reading [%v] from disk. %v",
				dataFilePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			return
		}

		io.Copy(w, dataFileReader)
	}

	w.WriteHeader(http.StatusOK)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"

	"github.com/Sirupsen/logrus"
	_ "github.com/eliothedeman/bangarang/alarm/console"
	_ "github.com/eliothedeman/bangarang/alarm/email"
	_ "github.com/eliothedeman/bangarang/alarm/influxdb"
	_ "github.com/eliothedeman/bangarang/alarm/pd"
	"github.com/eliothedeman/bangarang/api"
	"github.com/eliothedeman/bangarang/config"
	"github.com/eliothedeman/bangarang/pipeline"
	_ "github.com/eliothedeman/bangarang/provider/http"
	_ "github.com/eliothedeman/bangarang/provider/tcp"
)

var (
	confFile = flag.String("conf", "/etc/bangarang/conf.json", "path main config file")
	dev      = flag.Bool("dev", false, "puts bangarang in a dev testing mode")
	version  = flag.Bool("version", false, "display the version of this binary")
	confType = flag.String("conf-type", "db", `type of configuration used ["db", "json"]`)
)

const (
	versionNumber = "0.8.2"
)

func init() {
	logrus.SetLevel(logrus.InfoLevel)
	tf := &logrus.TextFormatter{}
	tf.FullTimestamp = true
	tf.ForceColors = true
	logrus.SetFormatter(tf)
}

func handleSigs() {
	stop := make(chan os.Signal)
	signal.Notify(stop, os.Kill, os.Interrupt)

	done := <-stop
	logrus.Fatal(done.String())
}

func main() {
	flag.Parse()

	// display the current version and exit
	if *version {
		fmt.Print(versionNumber)
		os.Exit(0)
	}

	// load configuration
	cp := config.GetProvider(*confType, *confFile)
	if cp == nil {
		logrus.Fatalf("Unable to load config of type %s at location %s", *confType, *confFile)
	}
	ac, err := cp.GetCurrent()
	if err != nil {
		logrus.Fatal(err)
	}

	if ac.LogLevel == "" {
		ac.LogLevel = "info"
	}

	ll, err := logrus.ParseLevel(ac.LogLevel)
	if err != nil {
		logrus.Error(err)
	} else {
		logrus.SetLevel(ll)
	}

	logrus.Infof("Starting processing pipeline with %d policie(s)", len(ac.Policies))
	// create and start up a new pipeline
	p := pipeline.NewPipeline(ac)
	p.Start()

	logrus.Infof("Serving the http api on port %d", 8081)
	// create and start a new api server
	apiServer := api.NewServer(ac.APIPort, p)
	go apiServer.Serve()

	handleSigs()
}
<commit_msg>updated version number<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"

	"github.com/Sirupsen/logrus"
	_ "github.com/eliothedeman/bangarang/alarm/console"
	_ "github.com/eliothedeman/bangarang/alarm/email"
	_ "github.com/eliothedeman/bangarang/alarm/influxdb"
	_ "github.com/eliothedeman/bangarang/alarm/pd"
	"github.com/eliothedeman/bangarang/api"
	"github.com/eliothedeman/bangarang/config"
	"github.com/eliothedeman/bangarang/pipeline"
	_ "github.com/eliothedeman/bangarang/provider/http"
	_ "github.com/eliothedeman/bangarang/provider/tcp"
)

var (
	confFile = flag.String("conf", "/etc/bangarang/conf.json", "path main config file")
	dev      = flag.Bool("dev", false, "puts bangarang in a dev testing mode")
	version  = flag.Bool("version", false, "display the version of this binary")
	confType = flag.String("conf-type", "db", `type of configuration used ["db", "json"]`)
)

const (
	versionNumber = "0.8.3"
)

func init() {
	logrus.SetLevel(logrus.InfoLevel)
	tf := &logrus.TextFormatter{}
	tf.FullTimestamp = true
	tf.ForceColors = true
	logrus.SetFormatter(tf)
}

func handleSigs() {
	stop := make(chan os.Signal)
	signal.Notify(stop, os.Kill, os.Interrupt)

	done := <-stop
	logrus.Fatal(done.String())
}

func main() {
	flag.Parse()

	// display the current version and exit
	if *version {
		fmt.Print(versionNumber)
		os.Exit(0)
	}

	// load configuration
	cp := config.GetProvider(*confType, *confFile)
	if cp == nil {
		logrus.Fatalf("Unable to load config of type %s at location %s", *confType, *confFile)
	}
	ac, err := cp.GetCurrent()
	if err != nil {
		logrus.Fatal(err)
	}

	if ac.LogLevel == "" {
		ac.LogLevel = "info"
	}

	ll, err := logrus.ParseLevel(ac.LogLevel)
	if err != nil {
		logrus.Error(err)
	} else {
		logrus.SetLevel(ll)
	}

	logrus.Infof("Starting processing pipeline with %d policie(s)", len(ac.Policies))
	// create and start up a new pipeline
	p := pipeline.NewPipeline(ac)
	p.Start()

	logrus.Infof("Serving the http api on port %d", 8081)
	// create and start a new api server
	apiServer := api.NewServer(ac.APIPort, p)
	go apiServer.Serve()

	handleSigs()
}
<|endoftext|>


<commit_before>package design

import (
	. "github.com/goadesign/goa/design"
	. "github.com/goadesign/goa/design/apidsl"
)

var User = Type("User", func() {
	Attribute("id", String, "ID of logged user")
	Attribute("subscriber", Boolean, "Flag whether user is subscriber (has paid for access)")
	Attribute("browser_id", String, "Anonymized ID of user's browser")
	Attribute("url", String, "URL of the content/conversion point", func() {
		Format("uri")
	})
	Attribute("adblock", Boolean, "Flag whether user has adblock enabled")
	Attribute("window_height", Number, "Height of the users browser window")
	Attribute("window_width", Number, "Width of the users browser window")
	Attribute("cookies", Boolean, "Flag whether user has cookies enabled")
	Attribute("websockets", Boolean, "Flag whether user has websockets supported browser")
	Attribute("user_agent", String, "User agent of client")
	Attribute("ip_address", String, "IP address of client", func() {
		Format("ip")
	})
	Attribute("source", Source, "UTM and social source metadata")
	Attribute("remp_session_id", String, "ID of reader's session")
	Attribute("remp_pageview_id", String, "ID of pageview")
	Attribute("referer", String, "Value of HTTP referer header (if present)")
	Attribute("explicit_referer_medium", String, "Explicit referer medium value that overrides implicit referer medium based on Referer header")
})

var System = Type("System", func() {
	Attribute("property_token", UUID, "Property token")
	Attribute("time", DateTime, "Time of occurrence")
	Required("property_token", "time")
})

var Pageview = Type("Pageview", func() {
	Description("Pageview is the payload for tracking pageview event")

	Attribute("action", String, func() {
		Enum("load", "timespent", "progress")
	})
	Attribute("system", System)
	Attribute("user", User)
	Attribute("article", Article)
	Attribute("timespent", TimeSpent)
	Attribute("progress", Progress)

	Required("system", "user", "action")
})

var Commerce = Type("Commerce", func() {
	Description("Commerce is the payload for tracking commerce event")

	Attribute("system", System)
	Attribute("user", User)
	Attribute("article", Article)
	Attribute("step", String, func() {
		Enum("checkout", "payment", "purchase", "refund")
	})
	Attribute("remp_commerce_id", String, "ID of event")
	Attribute("checkout", CommerceCheckout, "Used when user enters the checkout process (reviews the cart)")
	Attribute("payment", CommercePayment, "Used when user confirmed checkout and was redirected to the payment processor")
	Attribute("purchase", CommercePayment, "Used when payment processor confirms the payment")
	Attribute("refund", CommercePayment, "Used when refund is issued. Revenue should contain refunded amount of money.")

	Required("system", "user", "step")
})

var Event = Type("Event", func() {
	Description("TrackEvent is the payload for tracking generic event")

	Attribute("system", System)
	Attribute("user", User)

	Attribute("category", String, "Category of event (time, video, comment)")
	Attribute("action", String, "Specific action (read, pause, reply)")
	Attribute("value", Number, "Numeric value of event (read 60 seconds, paused after 200 seconds, 3rd comment")
	Attribute("tags", HashOf(String, String), "Custom filtering tags")
	Attribute("fields", HashOf(String, Any), "Additinal key-value data")
	Attribute("remp_event_id", String, "ID of event")
	Attribute("article_id", String, "Article ID")

	Required("system", "category", "action")
})

var Article = Type("Article", func() {
	Description("Article is the payload for tracking article-related data")

	Attribute("id", String, "ID of article")
	Attribute("category", String, "Page category (homepage, world news...")
	Attribute("tags", ArrayOf(String), "List of tags (breaking news, trump...")
	Attribute("author_id", String, "ID of author")
	Attribute("locked", Boolean, "Flag whether content was locked for the visitor")
	Attribute("variants", HashOf(String, String), "Hash of key-value pairs bearing A/B test variant information (what's A/B-tested / variant label)")

	Required("id")
})

var TimeSpent = Type("TimeSpent", func() {
	Description("Update time spent on page")

	Attribute("seconds", Integer, "Number of seconds from previous update of time spent on page")
	Attribute("unload", Boolean, "Flag to indicate last update of time spent on page before unload event")

	Required("seconds")
})

var Progress = Type("Progress", func() {
	Description("Update reading progress on pageview")

	Attribute("page_ratio", Number, "Whole page reading ratio", func() {
		Minimum(0)
		Maximum(1)
	})
	Attribute("article_ratio", Number, "Article reading ratio", func() {
		Minimum(0)
		Maximum(1)
	})
	Attribute("unload", Boolean, "Flag to indicate last update of progress on page before unload event")

	Required("page_ratio")
})

var Source = Type("source", func() {
	Description("User-source related data")

	Attribute("utm_source", String, "Origin of user (e.g. remp_campaign)")
	Attribute("utm_medium", String, "Medium through which the came (e.g. overlay, inline)")
	Attribute("utm_campaign", String, "Reference to specific campaign (e.g. campaign ID")
	Attribute("utm_content", String, "Reference to specific campaign mean (e.g. banner ID)")
	Attribute("banner_variant", String, "Reference to specific banner variant (e.g. variant ID)")
})

var CommerceCheckout = Type("CommerceCheckout", func() {
	Attribute("funnel_id", String, "ID of funnel user is being routed trough")

	Required("funnel_id")
})

var CommercePayment = Type("CommercePayment", func() {
	Attribute("funnel_id", String, "ID of funnel user is being routed trough")
	Attribute("transaction_id", String, "Public ID of transaction (variable symbol)")
	Attribute("product_ids", ArrayOf(String), "Public IDs of selected products")
	Attribute("revenue", Revenue, "Amount of money for given payment")

	Required("revenue", "transaction_id", "product_ids")
})

var Revenue = Type("Revenue", func() {
	Attribute("amount", Number, "Numeric amount of money")
	Attribute("currency", String, "ISO 4217 representation of currency")

	Required("amount", "currency")
})

var Entity = Type("Entity", func() {
	Attribute("entity_def", func() {
		Attribute("id", String)
		Attribute("name", String)
		Attribute("data", HashOf(String, Any))

		Required("id", "name", "data")
	})
	Attribute("system", System)
	Required("entity_def", "system")
})
<commit_msg>Documentation in Beam Tracker API improved<commit_after>package design

import (
	. "github.com/goadesign/goa/design"
	. "github.com/goadesign/goa/design/apidsl"
)

var User = Type("User", func() {
	Attribute("id", String, "ID of logged user")
	Attribute("subscriber", Boolean, "Flag whether user is subscriber (has paid for access)")
	Attribute("browser_id", String, "Anonymized ID of user's browser")
	Attribute("url", String, "URL of the content/conversion point", func() {
		Format("uri")
	})
	Attribute("adblock", Boolean, "Flag whether user has adblock enabled")
	Attribute("window_height", Number, "Height of the users browser window")
	Attribute("window_width", Number, "Width of the users browser window")
	Attribute("cookies", Boolean, "Flag whether user has cookies enabled")
	Attribute("websockets", Boolean, "Flag whether user has websockets supported browser")
	Attribute("user_agent", String, "User agent of client")
	Attribute("ip_address", String, "IP address of client", func() {
		Format("ip")
	})
	Attribute("source", Source, "UTM and social source metadata")
	Attribute("remp_session_id", String, "ID of reader's session")
	Attribute("remp_pageview_id", String, "ID of pageview")
	Attribute("referer", String, "Value of HTTP referer header (if present)")
	Attribute("explicit_referer_medium", String, "Explicit referer medium value that overrides implicit referer medium based on Referer header")
})

var System = Type("System", func() {
	Attribute("property_token", UUID, "Property token")
	Attribute("time", DateTime, "Time of occurrence")
	Required("property_token", "time")
})

var Pageview = Type("Pageview", func() {
	Description("Pageview is the payload for tracking pageview event")

	Attribute("action", String, func() {
		Enum("load", "timespent", "progress")
	})
	Attribute("system", System)
	Attribute("user", User)
	Attribute("article", Article)
	Attribute("timespent", TimeSpent)
	Attribute("progress", Progress)

	Required("system", "user", "action")
})

var Commerce = Type("Commerce", func() {
	Description("Commerce is the payload for tracking commerce event")

	Attribute("system", System)
	Attribute("user", User)
	Attribute("article", Article)
	Attribute("step", String, func() {
		Enum("checkout", "payment", "purchase", "refund")
	})
	Attribute("remp_commerce_id", String, "ID of event")
	Attribute("checkout", CommerceCheckout, "Used when user enters the checkout process (reviews the cart)")
	Attribute("payment", CommercePayment, "Used when user confirmed checkout and was redirected to the payment processor")
	Attribute("purchase", CommercePayment, "Used when payment processor confirms the payment")
	Attribute("refund", CommercePayment, "Used when refund is issued. Revenue should contain refunded amount of money.")

	Required("system", "user", "step")
})

var Event = Type("Event", func() {
	Description("TrackEvent is the payload for tracking generic event")

	Attribute("system", System)
	Attribute("user", User)

	Attribute("category", String, "Category of event (time, video, comment)")
	Attribute("action", String, "Specific action (read, pause, reply)")
	Attribute("value", Number, "Numeric value of event (read 60 seconds, paused after 200 seconds, 3rd comment")
	Attribute("tags", HashOf(String, String), "Custom filtering tags")
	Attribute("fields", HashOf(String, Any), "Additinal key-value data")
	Attribute("remp_event_id", String, "ID of event")
	Attribute("article_id", String, "ID of article this event is linked to. Events linked to the specific article can be later displayed at Beam dashboard timeline and at article detail timeline.")

	Required("system", "category", "action")
})

var Article = Type("Article", func() {
	Description("Article is the payload for tracking article-related data")

	Attribute("id", String, "ID of article")
	Attribute("category", String, "Page category (homepage, world news...")
	Attribute("tags", ArrayOf(String), "List of tags (breaking news, trump...")
	Attribute("author_id", String, "ID of author")
	Attribute("locked", Boolean, "Flag whether content was locked for the visitor")
	Attribute("variants", HashOf(String, String), "Hash of key-value pairs bearing A/B test variant information (what's A/B-tested / variant label)")

	Required("id")
})

var TimeSpent = Type("TimeSpent", func() {
	Description("Update time spent on page")

	Attribute("seconds", Integer, "Number of seconds from previous update of time spent on page")
	Attribute("unload", Boolean, "Flag to indicate last update of time spent on page before unload event")

	Required("seconds")
})

var Progress = Type("Progress", func() {
	Description("Update reading progress on pageview")

	Attribute("page_ratio", Number, "Whole page reading ratio", func() {
		Minimum(0)
		Maximum(1)
	})
	Attribute("article_ratio", Number, "Article reading ratio", func() {
		Minimum(0)
		Maximum(1)
	})
	Attribute("unload", Boolean, "Flag to indicate last update of progress on page before unload event")

	Required("page_ratio")
})

var Source = Type("source", func() {
	Description("User-source related data")

	Attribute("utm_source", String, "Origin of user (e.g. remp_campaign)")
	Attribute("utm_medium", String, "Medium through which the came (e.g. overlay, inline)")
	Attribute("utm_campaign", String, "Reference to specific campaign (e.g. campaign ID")
	Attribute("utm_content", String, "Reference to specific campaign mean (e.g. banner ID)")
	Attribute("banner_variant", String, "Reference to specific banner variant (e.g. variant ID)")
})

var CommerceCheckout = Type("CommerceCheckout", func() {
	Attribute("funnel_id", String, "ID of funnel user is being routed trough")

	Required("funnel_id")
})

var CommercePayment = Type("CommercePayment", func() {
	Attribute("funnel_id", String, "ID of funnel user is being routed trough")
	Attribute("transaction_id", String, "Public ID of transaction (variable symbol)")
	Attribute("product_ids", ArrayOf(String), "Public IDs of selected products")
	Attribute("revenue", Revenue, "Amount of money for given payment")

	Required("revenue", "transaction_id", "product_ids")
})

var Revenue = Type("Revenue", func() {
	Attribute("amount", Number, "Numeric amount of money")
	Attribute("currency", String, "ISO 4217 representation of currency")

	Required("amount", "currency")
})

var Entity = Type("Entity", func() {
	Attribute("entity_def", func() {
		Attribute("id", String)
		Attribute("name", String)
		Attribute("data", HashOf(String, Any))

		Required("id", "name", "data")
	})
	Attribute("system", System)
	Required("entity_def", "system")
})
<|endoftext|>


<commit_before>package vitals

import (
	boshassert "bosh/assert"
	boshstats "bosh/platform/stats"
	fakestats "bosh/platform/stats/fakes"
	boshdirs "bosh/settings/directories"
	"github.com/stretchr/testify/assert"
	"testing"
)

func TestVitalsConstruction(t *testing.T) {
	_, service := buildVitalsService()
	vitals, err := service.Get()

	expectedVitals := map[string]interface{}{
		"cpu": map[string]string{
			"sys":  "10.0",
			"user": "56.0",
			"wait": "1.0",
		},
		"disk": map[string]interface{}{
			"system": map[string]string{
				"percent":       "50",
				"inode_percent": "10",
			},
			"ephemeral": map[string]string{
				"percent":       "75",
				"inode_percent": "20",
			},
			"persistent": map[string]string{
				"percent":       "100",
				"inode_percent": "75",
			},
		},
		"load": []string{"0.20", "4.55", "1.12"},
		"mem": map[string]string{
			"kb":      "700",
			"percent": "70",
		},
		"swap": map[string]string{
			"kb":      "600",
			"percent": "60",
		},
	}

	assert.NoError(t, err)

	boshassert.MatchesJsonMap(t, vitals, expectedVitals)
}

func TestGettingVitalsWhenMissingDisks(t *testing.T) {
	statsCollector, service := buildVitalsService()
	statsCollector.DiskStats = map[string]boshstats.DiskStats{
		"/": boshstats.DiskStats{
			DiskUsage:  boshstats.Usage{Used: 100, Total: 200},
			InodeUsage: boshstats.Usage{Used: 50, Total: 500},
		},
	}

	vitals, err := service.Get()
	assert.NoError(t, err)

	boshassert.LacksJsonKey(t, vitals.Disk, "ephemeral")
	boshassert.LacksJsonKey(t, vitals.Disk, "persistent")
}

func TestGetGettingVitalsOnSystemDiskError(t *testing.T) {
	statsCollector, service := buildVitalsService()
	statsCollector.DiskStats = map[string]boshstats.DiskStats{}

	_, err := service.Get()
	assert.Error(t, err)
}

func buildVitalsService() (statsCollector *fakestats.FakeStatsCollector, service Service) {
	dirProvider := boshdirs.NewDirectoriesProvider("/fake/base/dir")
	statsCollector = &fakestats.FakeStatsCollector{
		CpuLoad: boshstats.CpuLoad{
			One:     0.2,
			Five:    4.55,
			Fifteen: 1.123,
		},
		CpuStats: boshstats.CpuStats{
			User:  56,
			Sys:   10,
			Wait:  1,
			Total: 100,
		},
		MemStats: boshstats.Usage{
			Used:  700 * 1024,
			Total: 1000 * 1024,
		},
		SwapStats: boshstats.Usage{
			Used:  600 * 1024,
			Total: 1000 * 1024,
		},
		DiskStats: map[string]boshstats.DiskStats{
			"/": boshstats.DiskStats{
				DiskUsage:  boshstats.Usage{Used: 100, Total: 200},
				InodeUsage: boshstats.Usage{Used: 50, Total: 500},
			},
			dirProvider.DataDir(): boshstats.DiskStats{
				DiskUsage:  boshstats.Usage{Used: 15, Total: 20},
				InodeUsage: boshstats.Usage{Used: 10, Total: 50},
			},
			dirProvider.StoreDir(): boshstats.DiskStats{
				DiskUsage:  boshstats.Usage{Used: 2, Total: 2},
				InodeUsage: boshstats.Usage{Used: 3, Total: 4},
			},
		},
	}

	service = NewService(statsCollector, dirProvider)
	return
}
<commit_msg>moved tests for platform/vitals into their own package<commit_after>package vitals_test

import (
	. "bosh/platform/vitals"
	boshassert "bosh/assert"
	boshstats "bosh/platform/stats"
	fakestats "bosh/platform/stats/fakes"
	boshdirs "bosh/settings/directories"
	"github.com/stretchr/testify/assert"
	"testing"
)

func TestVitalsConstruction(t *testing.T) {
	_, service := buildVitalsService()
	vitals, err := service.Get()

	expectedVitals := map[string]interface{}{
		"cpu": map[string]string{
			"sys":  "10.0",
			"user": "56.0",
			"wait": "1.0",
		},
		"disk": map[string]interface{}{
			"system": map[string]string{
				"percent":       "50",
				"inode_percent": "10",
			},
			"ephemeral": map[string]string{
				"percent":       "75",
				"inode_percent": "20",
			},
			"persistent": map[string]string{
				"percent":       "100",
				"inode_percent": "75",
			},
		},
		"load": []string{"0.20", "4.55", "1.12"},
		"mem": map[string]string{
			"kb":      "700",
			"percent": "70",
		},
		"swap": map[string]string{
			"kb":      "600",
			"percent": "60",
		},
	}

	assert.NoError(t, err)

	boshassert.MatchesJsonMap(t, vitals, expectedVitals)
}

func TestGettingVitalsWhenMissingDisks(t *testing.T) {
	statsCollector, service := buildVitalsService()
	statsCollector.DiskStats = map[string]boshstats.DiskStats{
		"/": boshstats.DiskStats{
			DiskUsage:  boshstats.Usage{Used: 100, Total: 200},
			InodeUsage: boshstats.Usage{Used: 50, Total: 500},
		},
	}

	vitals, err := service.Get()
	assert.NoError(t, err)

	boshassert.LacksJsonKey(t, vitals.Disk, "ephemeral")
	boshassert.LacksJsonKey(t, vitals.Disk, "persistent")
}

func TestGetGettingVitalsOnSystemDiskError(t *testing.T) {
	statsCollector, service := buildVitalsService()
	statsCollector.DiskStats = map[string]boshstats.DiskStats{}

	_, err := service.Get()
	assert.Error(t, err)
}

func buildVitalsService() (statsCollector *fakestats.FakeStatsCollector, service Service) {
	dirProvider := boshdirs.NewDirectoriesProvider("/fake/base/dir")
	statsCollector = &fakestats.FakeStatsCollector{
		CpuLoad: boshstats.CpuLoad{
			One:     0.2,
			Five:    4.55,
			Fifteen: 1.123,
		},
		CpuStats: boshstats.CpuStats{
			User:  56,
			Sys:   10,
			Wait:  1,
			Total: 100,
		},
		MemStats: boshstats.Usage{
			Used:  700 * 1024,
			Total: 1000 * 1024,
		},
		SwapStats: boshstats.Usage{
			Used:  600 * 1024,
			Total: 1000 * 1024,
		},
		DiskStats: map[string]boshstats.DiskStats{
			"/": boshstats.DiskStats{
				DiskUsage:  boshstats.Usage{Used: 100, Total: 200},
				InodeUsage: boshstats.Usage{Used: 50, Total: 500},
			},
			dirProvider.DataDir(): boshstats.DiskStats{
				DiskUsage:  boshstats.Usage{Used: 15, Total: 20},
				InodeUsage: boshstats.Usage{Used: 10, Total: 50},
			},
			dirProvider.StoreDir(): boshstats.DiskStats{
				DiskUsage:  boshstats.Usage{Used: 2, Total: 2},
				InodeUsage: boshstats.Usage{Used: 3, Total: 4},
			},
		},
	}

	service = NewService(statsCollector, dirProvider)
	return
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"encoding/binary"
	"hash"
	"hash/crc64"
	"io"
	"os"
	"sync"
)

// An io.Reader implementation that also keeps a crc64 as it reads.  Fancy!
type hashingReader struct {
	innerReader io.Reader
	hasher      hash.Hash64
}

func (r hashingReader) Read(buf []byte) (int, error) {
	n, err := r.innerReader.Read(buf)
	if err == nil {
		r.hasher.Write(buf[:n])
	}
	return n, err
}

func archiveReader(file io.Reader) {
	var workInProgress sync.WaitGroup
	fileOutputChan := make(map[string]chan block)

	hashReader := hashingReader{file, crc64.New(crc64.MakeTable(crc64.ECMA))}
	file = hashReader

	fileHeader := make([]byte, 8)
	_, err := io.ReadFull(file, fileHeader)
	if err != nil {
		logger.Fatalln("Archive read error:", err.Error())
	} else if bytes.Equal(fileHeader, fastArchiverHeader) {
		logger.Fatalln("Archive header not recognized")
	}

	for {
		var pathSize uint16
		err = binary.Read(file, binary.BigEndian, &pathSize)
		if err == io.EOF {
			break
		} else if err != nil {
			logger.Fatalln("Archive read error:", err.Error())
		}

		buf := make([]byte, pathSize)
		_, err = io.ReadFull(file, buf)
		if err != nil {
			logger.Fatalln("Archive read error:", err.Error())
		}
		filePath := string(buf)

		blockType := make([]byte, 1)
		_, err = io.ReadFull(file, blockType)
		if err != nil {
			logger.Fatalln("Archive read error:", err.Error())
		}

		if blockType[0] == byte(blockTypeStartOfFile) {
			var uid uint32
			var gid uint32
			var mode os.FileMode

			err = binary.Read(file, binary.BigEndian, &uid)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			err = binary.Read(file, binary.BigEndian, &gid)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			err = binary.Read(file, binary.BigEndian, &mode)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			c := make(chan block, 1)
			fileOutputChan[filePath] = c
			workInProgress.Add(1)
			go writeFile(c, &workInProgress)
			c <- block{filePath, 0, nil, blockTypeStartOfFile, int(uid), int(gid), mode}
		} else if blockType[0] == byte(blockTypeEndOfFile) {
			c := fileOutputChan[filePath]
			c <- block{filePath, 0, nil, blockTypeEndOfFile, 0, 0, 0}
			close(c)
			delete(fileOutputChan, filePath)
		} else if blockType[0] == byte(blockTypeData) {
			var blockSize uint16
			err = binary.Read(file, binary.BigEndian, &blockSize)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			blockData := make([]byte, blockSize)
			_, err = io.ReadFull(file, blockData)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			c := fileOutputChan[filePath]
			c <- block{filePath, blockSize, blockData, blockTypeData, 0, 0, 0}
		} else if blockType[0] == byte(blockTypeDirectory) {
			var uid uint32
			var gid uint32
			var mode os.FileMode

			err = binary.Read(file, binary.BigEndian, &uid)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}
			err = binary.Read(file, binary.BigEndian, &gid)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}
			err = binary.Read(file, binary.BigEndian, &mode)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			if ignorePerms {
				mode = os.ModeDir | 0755
			}
			err = os.Mkdir(filePath, mode)
			if err != nil && !os.IsExist(err) {
				logger.Fatalln("Directory create error:", err.Error())
			}
			if !ignoreOwners {
				err = os.Chown(filePath, int(uid), int(gid))
				if err != nil {
					logger.Println("Directory chown error:", err.Error())
				}
			}
		} else if blockType[0] == byte(blockTypeChecksum) {
			currentChecksum := hashReader.hasher.Sum64()

			var expectedChecksum uint64
			binary.Read(file, binary.BigEndian, &expectedChecksum)

			if expectedChecksum != currentChecksum {
				logger.Fatalln("crc64 mismatch, expected", expectedChecksum, "was", currentChecksum)
			}
		} else {
			logger.Fatalln("Archive error: unrecognized block type", blockType[0])
		}
	}

	workInProgress.Wait()
}

func writeFile(blockSource chan block, workInProgress *sync.WaitGroup) {
	var file *os.File = nil
	var bufferedFile *bufio.Writer
	for block := range blockSource {
		if block.blockType == blockTypeStartOfFile {
			if verbose {
				logger.Println(block.filePath)
			}

			tmp, err := os.Create(block.filePath)
			if err != nil {
				logger.Fatalln("File create error:", err.Error())
			}
			file = tmp
			bufferedFile = bufio.NewWriter(file)

			if !ignoreOwners {
				err = file.Chown(block.uid, block.gid)
				if err != nil {
					logger.Println("Unable to chown file to", block.uid, "/", block.gid, ":", err.Error())
				}
			}
			if !ignorePerms {
				err = file.Chmod(block.mode)
				if err != nil {
					logger.Println("Unable to chmod file to", block.mode, ":", err.Error())
				}
			}
		} else if block.blockType == blockTypeEndOfFile {
			bufferedFile.Flush()
			file.Close()
			file = nil
		} else {
			_, err := bufferedFile.Write(block.buffer[:block.numBytes])
			if err != nil {
				logger.Fatalln("File write error:", err.Error())
			}
		}
	}
	workInProgress.Done()
}
<commit_msg>fix inverted logic on header read check<commit_after>package main

import (
	"bufio"
	"bytes"
	"encoding/binary"
	"hash"
	"hash/crc64"
	"io"
	"os"
	"sync"
)

// An io.Reader implementation that also keeps a crc64 as it reads.  Fancy!
type hashingReader struct {
	innerReader io.Reader
	hasher      hash.Hash64
}

func (r hashingReader) Read(buf []byte) (int, error) {
	n, err := r.innerReader.Read(buf)
	if err == nil {
		r.hasher.Write(buf[:n])
	}
	return n, err
}

func archiveReader(file io.Reader) {
	var workInProgress sync.WaitGroup
	fileOutputChan := make(map[string]chan block)

	hashReader := hashingReader{file, crc64.New(crc64.MakeTable(crc64.ECMA))}
	file = hashReader

	fileHeader := make([]byte, 8)
	_, err := io.ReadFull(file, fileHeader)
	if err != nil {
		logger.Fatalln("Archive read error:", err.Error())
	} else if !bytes.Equal(fileHeader, fastArchiverHeader) {
		logger.Fatalln("Archive header not recognized")
	}

	for {
		var pathSize uint16
		err = binary.Read(file, binary.BigEndian, &pathSize)
		if err == io.EOF {
			break
		} else if err != nil {
			logger.Fatalln("Archive read error:", err.Error())
		}

		buf := make([]byte, pathSize)
		_, err = io.ReadFull(file, buf)
		if err != nil {
			logger.Fatalln("Archive read error:", err.Error())
		}
		filePath := string(buf)

		blockType := make([]byte, 1)
		_, err = io.ReadFull(file, blockType)
		if err != nil {
			logger.Fatalln("Archive read error:", err.Error())
		}

		if blockType[0] == byte(blockTypeStartOfFile) {
			var uid uint32
			var gid uint32
			var mode os.FileMode

			err = binary.Read(file, binary.BigEndian, &uid)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			err = binary.Read(file, binary.BigEndian, &gid)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			err = binary.Read(file, binary.BigEndian, &mode)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			c := make(chan block, 1)
			fileOutputChan[filePath] = c
			workInProgress.Add(1)
			go writeFile(c, &workInProgress)
			c <- block{filePath, 0, nil, blockTypeStartOfFile, int(uid), int(gid), mode}
		} else if blockType[0] == byte(blockTypeEndOfFile) {
			c := fileOutputChan[filePath]
			c <- block{filePath, 0, nil, blockTypeEndOfFile, 0, 0, 0}
			close(c)
			delete(fileOutputChan, filePath)
		} else if blockType[0] == byte(blockTypeData) {
			var blockSize uint16
			err = binary.Read(file, binary.BigEndian, &blockSize)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			blockData := make([]byte, blockSize)
			_, err = io.ReadFull(file, blockData)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			c := fileOutputChan[filePath]
			c <- block{filePath, blockSize, blockData, blockTypeData, 0, 0, 0}
		} else if blockType[0] == byte(blockTypeDirectory) {
			var uid uint32
			var gid uint32
			var mode os.FileMode

			err = binary.Read(file, binary.BigEndian, &uid)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}
			err = binary.Read(file, binary.BigEndian, &gid)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}
			err = binary.Read(file, binary.BigEndian, &mode)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			if ignorePerms {
				mode = os.ModeDir | 0755
			}
			err = os.Mkdir(filePath, mode)
			if err != nil && !os.IsExist(err) {
				logger.Fatalln("Directory create error:", err.Error())
			}
			if !ignoreOwners {
				err = os.Chown(filePath, int(uid), int(gid))
				if err != nil {
					logger.Println("Directory chown error:", err.Error())
				}
			}
		} else if blockType[0] == byte(blockTypeChecksum) {
			currentChecksum := hashReader.hasher.Sum64()

			var expectedChecksum uint64
			binary.Read(file, binary.BigEndian, &expectedChecksum)

			if expectedChecksum != currentChecksum {
				logger.Fatalln("crc64 mismatch, expected", expectedChecksum, "was", currentChecksum)
			}
		} else {
			logger.Fatalln("Archive error: unrecognized block type", blockType[0])
		}
	}

	workInProgress.Wait()
}

func writeFile(blockSource chan block, workInProgress *sync.WaitGroup) {
	var file *os.File = nil
	var bufferedFile *bufio.Writer
	for block := range blockSource {
		if block.blockType == blockTypeStartOfFile {
			if verbose {
				logger.Println(block.filePath)
			}

			tmp, err := os.Create(block.filePath)
			if err != nil {
				logger.Fatalln("File create error:", err.Error())
			}
			file = tmp
			bufferedFile = bufio.NewWriter(file)

			if !ignoreOwners {
				err = file.Chown(block.uid, block.gid)
				if err != nil {
					logger.Println("Unable to chown file to", block.uid, "/", block.gid, ":", err.Error())
				}
			}
			if !ignorePerms {
				err = file.Chmod(block.mode)
				if err != nil {
					logger.Println("Unable to chmod file to", block.mode, ":", err.Error())
				}
			}
		} else if block.blockType == blockTypeEndOfFile {
			bufferedFile.Flush()
			file.Close()
			file = nil
		} else {
			_, err := bufferedFile.Write(block.buffer[:block.numBytes])
			if err != nil {
				logger.Fatalln("File write error:", err.Error())
			}
		}
	}
	workInProgress.Done()
}
<|endoftext|>