<commit_before>package main

import (
	"encoding/json"
	"goim/libs/define"
	inet "goim/libs/net"
	"goim/libs/proto"
	"net/rpc"
	"time"

	log "github.com/thinkboy/log4go"
)

var (
	cometServiceMap = make(map[int32]**rpc.Client)
)

const (
	CometService              = "PushRPC"
	CometServicePing          = "PushRPC.Ping"
	CometServiceRooms         = "PushRPC.Rooms"
	CometServicePushMsg       = "PushRPC.PushMsg"
	CometServiceMPushMsg      = "PushRPC.MPushMsg"
	CometServiceBroadcast     = "PushRPC.Broadcast"
	CometServiceBroadcastRoom = "PushRPC.BroadcastRoom"
)

func InitComet(addrs map[int32]string) (err error) {
	for serverID, addrsTmp := range addrs {
		var (
			rpcClient     *rpc.Client
			quit          chan struct{}
			network, addr string
		)
		if network, addr, err = inet.ParseNetwork(addrsTmp); err != nil {
			log.Error("inet.ParseNetwork() error(%v)", err)
			return
		}
		if rpcClient, err = rpc.Dial(network, addr); err != nil {
			log.Error("rpc.Dial(\"%s\") error(%s)", addr, err)
		}
		go Reconnect(&rpcClient, quit, network, addr)
		log.Info("init comet rpc addr:%s connection", addr)
		cometServiceMap[serverID] = &rpcClient
	}
	return
}

// Reconnect for ping rpc server and reconnect with it when it's crash.
func Reconnect(dst **rpc.Client, quit chan struct{}, network, address string) {
	var (
		tmp    *rpc.Client
		err    error
		call   *rpc.Call
		ch     = make(chan *rpc.Call, 1)
		client = *dst
		args   = proto.NoArg{}
		reply  = proto.NoReply{}
	)
	for {
		select {
		case <-quit:
			return
		default:
			if client != nil {
				call = <-client.Go(CometServicePing, &args, &reply, ch).Done
				if call.Error != nil {
					log.Error("rpc ping %s error(%v)", address, call.Error)
				}
			}
			if client == nil || call.Error != nil {
				if tmp, err = rpc.Dial(network, address); err == nil {
					*dst = tmp
					client = tmp
				}
			}
		}
		time.Sleep(1 * time.Second)
	}
}

// get comet server client by server id
func getCometByServerId(serverId int32) (*rpc.Client, error) {
	if client, ok := cometServiceMap[serverId]; !ok || *client == nil {
		return nil, ErrComet
	} else {
		return *client, nil
	}
}

func mPushComet(serverId int32, subkeys []string, body json.RawMessage) {
	var (
		p     = &proto.Proto{Ver: 0, Operation: define.OP_SEND_SMS_REPLY, Body: body}
		args  = &proto.MPushMsgArg{Keys: subkeys, P: p}
		reply = &proto.MPushMsgReply{}
		c     *rpc.Client
		err   error
	)
	c, err = getCometByServerId(serverId)
	if err != nil {
		log.Error("getCometByServerId(\"%d\") error(%v)", serverId, err)
		return
	}
	if err = c.Call(CometServiceMPushMsg, args, reply); err != nil {
		log.Error("c.Call(\"%s\", %v, reply) error(%v)", CometServiceMPushMsg, *args, err)
	}
}

func broadcast(msg []byte) {
	var (
		p    = &proto.Proto{Ver: 0, Operation: define.OP_SEND_SMS_REPLY, Body: msg}
		args = &proto.BoardcastArg{P: p}
	)
	for serverId, c := range cometServiceMap {
		if *c != nil {
			go broadcastComet(*c, args)
		} else {
			log.Error("doesn`t push message to serverId:%d", serverId)
		}
	}
}

func broadcastComet(c *rpc.Client, args *proto.BoardcastArg) (err error) {
	var reply = proto.NoReply{}
	if err = c.Call(CometServiceBroadcast, args, &reply); err != nil {
		log.Error("c.Call(\"%s\", %v, reply) error(%v)", CometServiceBroadcast, *args, err)
	}
	return
}

func broadcastRoomBytes(roomId int32, body []byte) {
	var (
		p        = &proto.Proto{Ver: 0, Operation: define.OP_RAW, Body: body}
		args     = proto.BoardcastRoomArg{P: p, RoomId: roomId}
		reply    = proto.NoReply{}
		c        *rpc.Client
		serverId int32
		servers  map[int32]struct{}
		err      error
		ok       bool
	)

	//TODO concurrent push to per server?
	if servers, ok = RoomServersMap[roomId]; ok {
		for serverId, _ = range servers {
			if c, err = getCometByServerId(serverId); err != nil {
				log.Error("getCometByServerId(%d) error(%v)", serverId, err)
				continue
			}
			if err = c.Call(CometServiceBroadcastRoom, &args, &reply); err != nil {
				log.Error("c.Call(\"%s\", %v, reply) serverId:%d error(%v)", CometServiceBroadcastRoom, args, serverId, err)
			}
		}
	}
}

func roomsComet(c *rpc.Client) []int32 {
	var (
		args  = proto.NoArg{}
		reply = proto.RoomsReply{}
		err   error
	)
	if err = c.Call(CometServiceRooms, &args, &reply); err != nil {
		log.Error("c.Call(\"%s\", 0, reply) error(%v)", CometServiceRooms, err)
		return nil
	}
	return reply.RoomIds
}
<commit_msg>optimize code<commit_after>package main

import (
	"encoding/json"
	"goim/libs/define"
	inet "goim/libs/net"
	"goim/libs/proto"
	"net/rpc"
	"time"

	log "github.com/thinkboy/log4go"
)

var (
	cometServiceMap = make(map[int32]**rpc.Client)
)

const (
	CometService              = "PushRPC"
	CometServicePing          = "PushRPC.Ping"
	CometServiceRooms         = "PushRPC.Rooms"
	CometServicePushMsg       = "PushRPC.PushMsg"
	CometServiceMPushMsg      = "PushRPC.MPushMsg"
	CometServiceBroadcast     = "PushRPC.Broadcast"
	CometServiceBroadcastRoom = "PushRPC.BroadcastRoom"
)

func InitComet(addrs map[int32]string) (err error) {
	for serverID, addrsTmp := range addrs {
		var (
			rpcClient     *rpc.Client
			quit          chan struct{}
			network, addr string
		)
		if network, addr, err = inet.ParseNetwork(addrsTmp); err != nil {
			log.Error("inet.ParseNetwork() error(%v)", err)
			return
		}
		if rpcClient, err = rpc.Dial(network, addr); err != nil {
			log.Error("rpc.Dial(\"%s\") error(%s)", addr, err)
		}
		go Reconnect(&rpcClient, quit, network, addr)
		log.Info("init comet rpc addr:%s connection", addr)
		cometServiceMap[serverID] = &rpcClient
	}
	return
}

// Reconnect for ping rpc server and reconnect with it when it's crash.
func Reconnect(dst **rpc.Client, quit chan struct{}, network, address string) {
	var (
		tmp    *rpc.Client
		err    error
		call   *rpc.Call
		ch     = make(chan *rpc.Call, 1)
		client = *dst
		args   = proto.NoArg{}
		reply  = proto.NoReply{}
	)
	for {
		select {
		case <-quit:
			return
		default:
			if client != nil {
				call = <-client.Go(CometServicePing, &args, &reply, ch).Done
				if call.Error != nil {
					log.Error("rpc ping %s error(%v)", address, call.Error)
				}
			}
			if client == nil || call.Error != nil {
				if tmp, err = rpc.Dial(network, address); err == nil {
					*dst = tmp
					client = tmp
				}
			}
		}
		time.Sleep(1 * time.Second)
	}
}

// get comet server client by server id
func getCometByServerId(serverId int32) (*rpc.Client, error) {
	if client, ok := cometServiceMap[serverId]; !ok || *client == nil {
		return nil, ErrComet
	} else {
		return *client, nil
	}
}

// mPushComet push a message to a batch of subkeys
func mPushComet(serverId int32, subkeys []string, body json.RawMessage) {
	var (
		args = &proto.MPushMsgArg{Keys: subkeys,
			P: proto.Proto{Ver: 0, Operation: define.OP_SEND_SMS_REPLY, Body: body}}
		reply = &proto.MPushMsgReply{}
		c     *rpc.Client
		err   error
	)
	c, err = getCometByServerId(serverId)
	if err != nil {
		log.Error("getCometByServerId(\"%d\") error(%v)", serverId, err)
		return
	}
	if err = c.Call(CometServiceMPushMsg, args, reply); err != nil {
		log.Error("c.Call(\"%s\", %v, reply) error(%v)", CometServiceMPushMsg, *args, err)
	}
}

// broadcast broadcast a message to all
func broadcast(msg []byte) {
	var (
		args = &proto.BoardcastArg{P: proto.Proto{Ver: 0, Operation: define.OP_SEND_SMS_REPLY, Body: msg}}
	)
	for serverId, c := range cometServiceMap {
		if *c != nil {
			go broadcastComet(*c, args)
		} else {
			log.Error("doesn`t push message to serverId:%d", serverId)
		}
	}
}

// broadcastComet a message to specified comet
func broadcastComet(c *rpc.Client, args *proto.BoardcastArg) (err error) {
	var reply = proto.NoReply{}
	if err = c.Call(CometServiceBroadcast, args, &reply); err != nil {
		log.Error("c.Call(\"%s\", %v, reply) error(%v)", CometServiceBroadcast, *args, err)
	}
	return
}

// broadcastRoomBytes broadcast aggregation messages to room
func broadcastRoomBytes(roomId int32, body []byte) {
	var (
		args     = proto.BoardcastRoomArg{P: proto.Proto{Ver: 0, Operation: define.OP_RAW, Body: body}, RoomId: roomId}
		reply    = proto.NoReply{}
		c        *rpc.Client
		serverId int32
		servers  map[int32]struct{}
		err      error
		ok       bool
	)

	//TODO concurrent push to each server?
	if servers, ok = RoomServersMap[roomId]; ok {
		for serverId, _ = range servers {
			if c, err = getCometByServerId(serverId); err != nil {
				log.Error("getCometByServerId(%d) error(%v)", serverId, err)
				continue
			}
			if err = c.Call(CometServiceBroadcastRoom, &args, &reply); err != nil {
				log.Error("c.Call(\"%s\", %v, reply) serverId:%d error(%v)", CometServiceBroadcastRoom, args, serverId, err)
			}
		}
	}
}

func roomsComet(c *rpc.Client) []int32 {
	var (
		args  = proto.NoArg{}
		reply = proto.RoomsReply{}
		err   error
	)
	if err = c.Call(CometServiceRooms, &args, &reply); err != nil {
		log.Error("c.Call(\"%s\", 0, reply) error(%v)", CometServiceRooms, err)
		return nil
	}
	return reply.RoomIds
}
<|endoftext|>


<commit_before>//      __      ___        _______ _____  ______ ______
//     /\ \    / / |      |__   __|  __ \|  ____|  ____|
//    /  \ \  / /| |         | |  | |__) | |__  | |__
//   / /\ \ \/ / | |         | |  |  _  /|  __| |  __|
//  / ____ \  /  | |____     | |  | | \ \| |____| |____
// /_/    \_\/   |______|    |_|  |_|  \_\______|______|
//

package AVL_Tree

import (
	"bytes"
	"testing"
)

func TestAVLTREE(t *testing.T) {

	printErr := func(err error) {
		if err != nil {
			t.Errorf(err.Error())
		}
	}

	//First create the AVL tree to be tested with
	tree := NewAVLTree()

	//Test adding several values to the AVL Tree
	//  at the same time test how the heights of the tree react
	heightTest := func(expectedHeight int) {
		height := tree.trunk.Height
		if height != expectedHeight {
			t.Errorf("bad height, expected %v found %v ", expectedHeight, height)
		}
	}

	printErr(tree.Add([]byte("keyOne"), []byte("valueOne")))
	heightTest(0)
	printErr(tree.Add([]byte("keyTwo"), []byte("valueTwo")))
	heightTest(1)
	printErr(tree.Add([]byte("keyThree"), []byte("valueThree")))
	heightTest(1)
	printErr(tree.Add([]byte("keyFour"), []byte("valueFour")))
	heightTest(2)

	//Test retrieving saved values
	retrieveTest := func(key, expectedVal string) {
		recievedVal, err := tree.Get([]byte(key))
		printErr(err)
		if bytes.Compare(recievedVal, []byte(expectedVal)) != 0 {
			t.Errorf("bad expected %v recieved %v ", expectedVal, string(recievedVal[:]))
		}
	}

	retrieveTest("keyOne", "valueOne")
	retrieveTest("keyTwo", "valueTwo")
	retrieveTest("keyThree", "valueThree")
	retrieveTest("keyFour", "valueFour")

	//Test adding a duplicate value

	//Test updating an existing value

	//Test updating a non existent value

	//Test removing saved values from the tree

	//Test retrieval of saved values
}
<commit_msg>updating remainder of tests<commit_after>//      __      ___        _______ _____  ______ ______
//     /\ \    / / |      |__   __|  __ \|  ____|  ____|
//    /  \ \  / /| |         | |  | |__) | |__  | |__
//   / /\ \ \/ / | |         | |  |  _  /|  __| |  __|
//  / ____ \  /  | |____     | |  | | \ \| |____| |____
// /_/    \_\/   |______|    |_|  |_|  \_\______|______|
//

package AVL_Tree

import (
	"bytes"
	"testing"
)

func TestAVLTREE(t *testing.T) {

	printErr := func(err error) {
		if err != nil {
			t.Errorf(err.Error())
		}
	}

	//First create the AVL tree to be tested with
	tree := NewAVLTree()

	//Test adding several values to the AVL Tree
	//  at the same time test how the heights of the tree react
	heightTest := func(expectedHeight int) {
		height := tree.trunk.Height
		if height != expectedHeight {
			t.Errorf("bad height, expected %v found %v ", expectedHeight, height)
		}
	}

	//some expected tree forms:
	// a   a   b      b       b       d      d
	//    /   / \    / \     / \     / \    / \
	//   b   a   c  a   c   a   d   b   e  c   e
	//                   \     / \   \
	//                    d   c   e   c

	printErr(tree.Add([]byte("a"), []byte("vA")))
	heightTest(0)
	printErr(tree.Add([]byte("b"), []byte("vB")))
	heightTest(1)
	printErr(tree.Add([]byte("c"), []byte("vC")))
	heightTest(1)
	printErr(tree.Add([]byte("d"), []byte("vD")))
	heightTest(2)
	printErr(tree.Add([]byte("e"), []byte("vE")))
	heightTest(2)

	//Test retrieving saved values
	retrieveTest := func(key, expectedVal string, expectedExists bool) {
		recievedVal, err := tree.Get([]byte(key))
		if expectedExists {
			printErr(err)
			if bytes.Compare(recievedVal, []byte(expectedVal)) != 0 {
				t.Errorf("bad expected %v recieved %v ", expectedVal, string(recievedVal[:]))
			}
		} else {
			if err == nil {
				t.Errorf("expected to receive an error when attempting to retrieve non-existent value for key ", key)
			}
		}
	}

	retrieveTest("a", "vA", true)
	retrieveTest("b", "vB", true)
	retrieveTest("c", "vC", true)
	retrieveTest("d", "vD", true)
	retrieveTest("e", "vE", true)

	//Test adding a duplicate value
	expErr := tree.Add([]byte("a"), []byte("vA"))
	if expErr == nil {
		t.Errorf("expected to receive an error when attempting to add duplicate values")
	}

	//Test updating an existing value
	printErr(tree.Update([]byte("a"), []byte("vAA")))
	retrieveTest("a", "vAA", true)

	//Test updating a non existent value
	expErr = tree.Update([]byte("z"), []byte("vZ"))
	if expErr == nil {
		t.Errorf("expected to receive an error when attempting to update non-existent value")
	}

	//Test removing saved values from the tree
	printErr(tree.Remove([]byte("a")))
	heightTest(2)
	printErr(tree.Remove([]byte("b")))
	heightTest(1)

	//Test bad retrieval of old saved value
	retrieveTest("a", "", false)
	retrieveTest("b", "", false)
	retrieveTest("c", "vC", true)
	retrieveTest("d", "vD", true)
	retrieveTest("e", "vE", true)
}
<|endoftext|>


<commit_before>// +build !windows

package nodos

import (
	"errors"
	"io"
	"os"
)

func copyFile(src, dst string, isFailIfExists bool) error {
	srcFd, err := os.Open(src)
	if err != nil {
		return err
	}
	defer srcFd.Close()

	if isFailIfExists {
		_, err = os.Stat(dst)
		if err == nil {
			return os.ErrExist
		}
	}
	dstFd, err := os.Create(dst)
	if err != nil {
		return err
	}

	_, err = io.Copy(dstFd, srcFd)

	if err != nil && err != io.EOF {
		dstFd.Close()
		return err
	}
	if err = dstFd.Close(); err != nil {
		return err
	}
	if fi, err := srcFd.Stat(); err != nil {
		return err
	} else {
		modTime := fi.ModTime()
		if err := os.Chtimes(dst, modTime, modTime); err != nil {
			return err
		}
	}
	return nil
}

func moveFile(src, dst string) error {
	return os.Rename(src, dst)
}
<commit_msg>Fix: linux build error<commit_after>// +build !windows

package nodos

import (
	"io"
	"os"
)

func copyFile(src, dst string, isFailIfExists bool) error {
	srcFd, err := os.Open(src)
	if err != nil {
		return err
	}
	defer srcFd.Close()

	if isFailIfExists {
		_, err = os.Stat(dst)
		if err == nil {
			return os.ErrExist
		}
	}
	dstFd, err := os.Create(dst)
	if err != nil {
		return err
	}

	_, err = io.Copy(dstFd, srcFd)

	if err != nil && err != io.EOF {
		dstFd.Close()
		return err
	}
	if err = dstFd.Close(); err != nil {
		return err
	}
	if fi, err := srcFd.Stat(); err != nil {
		return err
	} else {
		modTime := fi.ModTime()
		if err := os.Chtimes(dst, modTime, modTime); err != nil {
			return err
		}
	}
	return nil
}

func moveFile(src, dst string) error {
	return os.Rename(src, dst)
}
<|endoftext|>


<commit_before>/*
 *    Copyright (C) 2014 Christian Muehlhaeuser
 *
 *    This program is free software: you can redistribute it and/or modify
 *    it under the terms of the GNU Affero General Public License as published
 *    by the Free Software Foundation, either version 3 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *    Authors:
 *      Christian Muehlhaeuser <muesli@gmail.com>
 */

// beehive's Html Extraction module.
package htmlextractbee

import (
	"github.com/muesli/beehive/bees"
	"github.com/advancedlogic/GoOse"
	"strings"
)

type HtmlExtractBee struct {
	bees.Bee

	url string

	evchan chan bees.Event
}

func (mod *HtmlExtractBee) Action(action bees.Action) []bees.Placeholder {
	outs := []bees.Placeholder{}

	switch action.Name {
	case "extract":
		var url string
		for _, opt := range action.Options {
			if opt.Name == "url" {
				url = opt.Value.(string)
				if start := strings.Index(url, "http://"); start >= 0 {
					url = url[start:]
					if end := strings.Index(url, " "); end >= 0 {
						url = url[:end]
					}
				}
			}
		}

		g := goose.New()
    	article := g.ExtractFromUrl(url)
    	if len(strings.TrimSpace(article.Title)) > 0 {
	    	ev := bees.Event{
				Bee:  mod.Name(),
				Name: "info_extracted",
				Options: []bees.Placeholder{
					bees.Placeholder{
						Name:  "title",
						Type:  "string",
						Value: article.Title,
					},
					bees.Placeholder{
						Name:  "domain",
						Type:  "string",
						Value: article.Domain,
					},
					bees.Placeholder{
						Name:  "topimage",
						Type:  "url",
						Value: article.TopImage,
					},
					bees.Placeholder{
						Name:  "finalurl",
						Type:  "url",
						Value: article.FinalUrl,
					},
					bees.Placeholder{
						Name:  "meta_description",
						Type:  "string",
						Value: article.MetaDescription,
					},
					bees.Placeholder{
						Name:  "meta_keywords",
						Type:  "string",
						Value: article.MetaKeywords,
					},
				},
			}
			mod.evchan <- ev
		}

	default:
		panic("Unknown action triggered in " +mod.Name()+": "+action.Name)
	}

	return outs
}

func (mod *HtmlExtractBee) Run(eventChan chan bees.Event) {
	mod.evchan = eventChan
}
<commit_msg>* Support http* urls in extractor.<commit_after>/*
 *    Copyright (C) 2014 Christian Muehlhaeuser
 *
 *    This program is free software: you can redistribute it and/or modify
 *    it under the terms of the GNU Affero General Public License as published
 *    by the Free Software Foundation, either version 3 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *    Authors:
 *      Christian Muehlhaeuser <muesli@gmail.com>
 */

// beehive's Html Extraction module.
package htmlextractbee

import (
	"github.com/muesli/beehive/bees"
	"github.com/advancedlogic/GoOse"
	"strings"
)

type HtmlExtractBee struct {
	bees.Bee

	url string

	evchan chan bees.Event
}

func (mod *HtmlExtractBee) Action(action bees.Action) []bees.Placeholder {
	outs := []bees.Placeholder{}

	switch action.Name {
	case "extract":
		var url string
		for _, opt := range action.Options {
			if opt.Name == "url" {
				url = opt.Value.(string)
				if start := strings.Index(url, "http"); start >= 0 {
					url = url[start:]
					if end := strings.Index(url, " "); end >= 0 {
						url = url[:end]
					}
				}
			}
		}

		g := goose.New()
    	article := g.ExtractFromUrl(url)
    	if len(strings.TrimSpace(article.Title)) > 0 {
	    	ev := bees.Event{
				Bee:  mod.Name(),
				Name: "info_extracted",
				Options: []bees.Placeholder{
					bees.Placeholder{
						Name:  "title",
						Type:  "string",
						Value: article.Title,
					},
					bees.Placeholder{
						Name:  "domain",
						Type:  "string",
						Value: article.Domain,
					},
					bees.Placeholder{
						Name:  "topimage",
						Type:  "url",
						Value: article.TopImage,
					},
					bees.Placeholder{
						Name:  "finalurl",
						Type:  "url",
						Value: article.FinalUrl,
					},
					bees.Placeholder{
						Name:  "meta_description",
						Type:  "string",
						Value: article.MetaDescription,
					},
					bees.Placeholder{
						Name:  "meta_keywords",
						Type:  "string",
						Value: article.MetaKeywords,
					},
				},
			}
			mod.evchan <- ev
		}

	default:
		panic("Unknown action triggered in " +mod.Name()+": "+action.Name)
	}

	return outs
}

func (mod *HtmlExtractBee) Run(eventChan chan bees.Event) {
	mod.evchan = eventChan
}
<|endoftext|>


<commit_before>package lzma

import (
	"io"
)

// Maximum and minimum values for individual parameters.
const (
	MinLC       = 0
	MaxLC       = 8
	MinLP       = 0
	MaxLP       = 4
	MinPB       = 0
	MaxPB       = 4
	MinDictSize = 1 << 12
	MaxDictSize = 1<<32 - 1
)

// Parameters contain all information required to decode or encode an LZMA
// stream.
//
// The DictSize will be limited by MaxInt32 on 32-bit platforms.
type Parameters struct {
	// number of literal context bits
	LC int
	// number of literal position bits
	LP int
	// number of position bits
	PB int
	// size of the dictionary in bytes
	DictSize uint32
	// size of uncompressed data in bytes
	Size int64
	// header includes unpacked size
	SizeInHeader bool
	// end-of-stream marker requested
	EOS bool
}

// verifyParameters checks parameters for errors.
func verifyParameters(p *Parameters) error {
	if p == nil {
		return newError("parameters must be non-nil")
	}
	if !(MinLC <= p.LC && p.LC <= MaxLC) {
		return newError("LC out of range")
	}
	if !(MinLP <= p.LP && p.LP <= MaxLP) {
		return newError("LP out of range")
	}
	if !(MinPB <= p.PB && p.PB <= MaxPB) {
		return newError("PB out ouf range")
	}
	if !(MinDictSize <= p.DictSize && p.DictSize <= MaxDictSize) {
		return newError("DictSize out of range")
	}
	hlen := int(p.DictSize)
	if hlen < 0 {
		return newError("DictSize cannot be converted into int")
	}
	if p.Size < 0 {
		return newError("length must not be negative")
	}
	return nil
}

// getUint32LE reads an uint32 integer from a byte slize
func getUint32LE(b []byte) uint32 {
	x := uint32(b[3]) << 24
	x |= uint32(b[2]) << 16
	x |= uint32(b[1]) << 8
	x |= uint32(b[0])
	return x
}

// getUint64LE converts the uint64 value stored as little endian to an uint64
// value.
func getUint64LE(b []byte) uint64 {
	x := uint64(b[7]) << 56
	x |= uint64(b[6]) << 48
	x |= uint64(b[5]) << 40
	x |= uint64(b[4]) << 32
	x |= uint64(b[3]) << 24
	x |= uint64(b[2]) << 16
	x |= uint64(b[1]) << 8
	x |= uint64(b[0])
	return x
}

// putUint32LE puts an uint32 integer into a byte slice that must have at least
// a lenght of 4 bytes.
func putUint32LE(b []byte, x uint32) {
	b[0] = byte(x)
	b[1] = byte(x >> 8)
	b[2] = byte(x >> 16)
	b[3] = byte(x >> 24)
}

// putUint64LE puts the uint64 value into the byte slice as little endian
// value. The byte slice b must have at least place for 8 bytes.
func putUint64LE(b []byte, x uint64) {
	b[0] = byte(x)
	b[1] = byte(x >> 8)
	b[2] = byte(x >> 16)
	b[3] = byte(x >> 24)
	b[4] = byte(x >> 32)
	b[5] = byte(x >> 40)
	b[6] = byte(x >> 48)
	b[7] = byte(x >> 56)
}

// readHeader reads the classic LZMA header.
func readHeader(r io.Reader) (p *Parameters, err error) {
	b := make([]byte, 13)
	_, err = io.ReadFull(r, b)
	if err != nil {
		return nil, err
	}
	p = new(Parameters)
	x := int(b[0])
	p.LC = x % 9
	x /= 9
	p.LP = x % 5
	p.PB = x / 5
	if !(MinPB <= p.PB && p.PB <= MaxPB) {
		return nil, newError("PB out of range")
	}
	p.DictSize = getUint32LE(b[1:])
	if p.DictSize < MinDictSize {
		// The LZMA specification makes the following recommendation.
		p.DictSize = MinDictSize
	}
	u := getUint64LE(b[5:])
	if u == noHeaderLen {
		p.Size = 0
		p.EOS = true
		p.SizeInHeader = false
		return p, nil
	}
	p.Size = int64(u)
	if p.Size < 0 {
		return nil, newError(
			"unpack length in header not supported by int64")
	}
	p.EOS = false
	p.SizeInHeader = true
	return p, nil
}

// writeHeader writes the header for classic LZMA files.
func writeHeader(w io.Writer, p *Parameters) error {
	var err error
	if err = verifyParameters(p); err != nil {
		return err
	}
	b := make([]byte, 13)
	b[0] = byte((p.PB*5+p.LP)*9 + p.LC)
	putUint32LE(b[1:5], p.DictSize)
	var l uint64
	if p.SizeInHeader {
		l = uint64(p.Size)
	} else {
		l = noHeaderLen
	}
	putUint64LE(b[5:], l)
	_, err = w.Write(b)
	return err
}
<commit_msg>lzma: introduced Properties type and methods and functions<commit_after>package lzma

import (
	"io"
)

// Maximum and minimum values for individual parameters.
const (
	MinLC       = 0
	MaxLC       = 8
	MinLP       = 0
	MaxLP       = 4
	MinPB       = 0
	MaxPB       = 4
	MinDictSize = 1 << 12
	MaxDictSize = 1<<32 - 1
)

// Properties contains the parametes lc, lp and pb.
type Properties struct {
	// number of literal context bits
	LC int
	// number of literal position bits
	LP int
	// number of position bits
	PB int
}

// verifyProperties checks the argument for any errors.
func verifyProperties(p *Properties) error {
	if !(MinLC <= p.LC && p.LC <= MaxLC) {
		return newError("lc out of range")
	}
	if !(MinLP <= p.LP && p.LP <= MaxLC) {
		return newError("lp out of range")
	}
	if !(MinPB <= p.PB && p.PB <= MaxPB) {
		return newError("pb out of range")
	}
	return nil
}

// props reads a single properties byte.
func (p *Properties) Decode(b byte) error {
	x := int(b)
	p.LC = x % 9
	x /= 9
	p.LP = x % 5
	p.PB = x / 5
	if !(MinPB <= p.PB && p.PB <= MaxPB) {
		return newError("PB out of range")
	}
	return nil
}

// Encodes the properties in a single byte.
func (p *Properties) Byte() byte {
	return byte((p.PB*5+p.LP)*9 + p.LC)
}

// Parameters contain all information required to decode or encode an LZMA
// stream.
//
// The DictSize will be limited by MaxInt32 on 32-bit platforms.
type Parameters struct {
	// number of literal context bits
	LC int
	// number of literal position bits
	LP int
	// number of position bits
	PB int
	// size of the dictionary in bytes
	DictSize uint32
	// size of uncompressed data in bytes
	Size int64
	// header includes unpacked size
	SizeInHeader bool
	// end-of-stream marker requested
	EOS bool
}

// Properties returns lc, lp and pb as Properties value.
func (p *Parameters) Properties() *Properties {
	return &Properties{LC: p.LC, LP: p.LP, PB: p.PB}
}

// verifyParameters checks parameters for errors.
func verifyParameters(p *Parameters) error {
	if p == nil {
		return newError("parameters must be non-nil")
	}
	if err := verifyProperties(p.Properties()); err != nil {
		return err
	}
	if !(MinDictSize <= p.DictSize && p.DictSize <= MaxDictSize) {
		return newError("DictSize out of range")
	}
	hlen := int(p.DictSize)
	if hlen < 0 {
		return newError("DictSize cannot be converted into int")
	}
	if p.Size < 0 {
		return newError("length must not be negative")
	}
	return nil
}

// getUint32LE reads an uint32 integer from a byte slize
func getUint32LE(b []byte) uint32 {
	x := uint32(b[3]) << 24
	x |= uint32(b[2]) << 16
	x |= uint32(b[1]) << 8
	x |= uint32(b[0])
	return x
}

// getUint64LE converts the uint64 value stored as little endian to an uint64
// value.
func getUint64LE(b []byte) uint64 {
	x := uint64(b[7]) << 56
	x |= uint64(b[6]) << 48
	x |= uint64(b[5]) << 40
	x |= uint64(b[4]) << 32
	x |= uint64(b[3]) << 24
	x |= uint64(b[2]) << 16
	x |= uint64(b[1]) << 8
	x |= uint64(b[0])
	return x
}

// putUint32LE puts an uint32 integer into a byte slice that must have at least
// a lenght of 4 bytes.
func putUint32LE(b []byte, x uint32) {
	b[0] = byte(x)
	b[1] = byte(x >> 8)
	b[2] = byte(x >> 16)
	b[3] = byte(x >> 24)
}

// putUint64LE puts the uint64 value into the byte slice as little endian
// value. The byte slice b must have at least place for 8 bytes.
func putUint64LE(b []byte, x uint64) {
	b[0] = byte(x)
	b[1] = byte(x >> 8)
	b[2] = byte(x >> 16)
	b[3] = byte(x >> 24)
	b[4] = byte(x >> 32)
	b[5] = byte(x >> 40)
	b[6] = byte(x >> 48)
	b[7] = byte(x >> 56)
}

// readHeader reads the classic LZMA header.
func readHeader(r io.Reader) (p *Parameters, err error) {
	b := make([]byte, 13)
	_, err = io.ReadFull(r, b)
	if err != nil {
		return nil, err
	}
	p = new(Parameters)
	var props Properties
	if err = props.Decode(b[0]); err != nil {
		return nil, err
	}
	p.LC, p.LP, p.PB = props.LC, props.LP, props.PB
	p.DictSize = getUint32LE(b[1:])
	if p.DictSize < MinDictSize {
		// The LZMA specification makes the following recommendation.
		p.DictSize = MinDictSize
	}
	u := getUint64LE(b[5:])
	if u == noHeaderLen {
		p.Size = 0
		p.EOS = true
		p.SizeInHeader = false
		return p, nil
	}
	p.Size = int64(u)
	if p.Size < 0 {
		return nil, newError(
			"unpack length in header not supported by int64")
	}
	p.EOS = false
	p.SizeInHeader = true
	return p, nil
}

// writeHeader writes the header for classic LZMA files.
func writeHeader(w io.Writer, p *Parameters) error {
	var err error
	if err = verifyParameters(p); err != nil {
		return err
	}
	b := make([]byte, 13)
	b[0] = p.Properties().Byte()
	putUint32LE(b[1:5], p.DictSize)
	var l uint64
	if p.SizeInHeader {
		l = uint64(p.Size)
	} else {
		l = noHeaderLen
	}
	putUint64LE(b[5:], l)
	_, err = w.Write(b)
	return err
}
<|endoftext|>


<commit_before>package main

// 4m in real	1m45.438s
//	 user	0m0.036s
//	 sys	0m0.017s

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"

	"../../"	// "github.com/mediawen/watson-go-sdk"
)

type Cfg struct {
	User string 			`json:"user"`
	Pass string 			`json:"pass"`
}

func loadCfg(name string) (*Cfg, error) {
	f, err := ioutil.ReadFile(name)
	if err != nil {
		return nil, err
	}

	cfg := &Cfg{}

	err = json.Unmarshal(f, &cfg)
	if err != nil {
		return nil, err
	}

	return cfg, nil
}

func main() {
	args := os.Args
	if len(args) < 2 {
		fmt.Printf(
			"usage: stt out.srt model in.[wav|flac]\n" +
			"       stt -l\n")
		return
	}

	cfg, err := loadCfg("./stt.cfg.json")
	if err != nil {
		log.Fatal(err)
	}

	w := watson.New(cfg.User, cfg.Pass)

	ml, err := w.GetModels()
	if err != nil {
		log.Fatal(err)
	}

	if args[1] == "-l" {
		if len(args) > 2 {
			log.Fatal("too many args")
		}

		for _, m := range ml.Models {
			fmt.Printf("%s %-8d=> %s\n", m.Lang, m.Rate, m.Name)
		}

		return
	}

	out := args[1]
	model := args[2]
	in := args[3]
	ext := ""
	switch path.Ext(in) {
	case ".wav":
		ext = "wav"
	case ".flac":
		ext = "flac"
	case ".json":
		ext = "json"
	default:
		log.Fatal("stt: unknown file format: ", in)
	}

	found := false
	for _, m := range ml.Models {
		if m.Name == model {
			found = true
		}
	}
	if !found {
		log.Fatal("model not found")
	}

	is, err := os.Open(in)
	if err != nil {
		log.Fatal(err)
	}
	defer is.Close()

	os, err := os.Create(out)
	if err != nil {
		log.Fatal(err)
	}
	defer os.Close()

	tt, err := w.Recognize(is, model, ext)
	if err != nil {
		log.Fatal(err)
	}

	for _, w := range tt.Words {
		fmt.Printf("%v\n", w)
	}
}
<commit_msg>remove comment<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"

	"../../"	// "github.com/mediawen/watson-go-sdk"
)

type Cfg struct {
	User string 			`json:"user"`
	Pass string 			`json:"pass"`
}

func loadCfg(name string) (*Cfg, error) {
	f, err := ioutil.ReadFile(name)
	if err != nil {
		return nil, err
	}

	cfg := &Cfg{}

	err = json.Unmarshal(f, &cfg)
	if err != nil {
		return nil, err
	}

	return cfg, nil
}

func main() {
	args := os.Args
	if len(args) < 2 {
		fmt.Printf(
			"usage: stt out.srt model in.[wav|flac]\n" +
			"       stt -l\n")
		return
	}

	cfg, err := loadCfg("./stt.cfg.json")
	if err != nil {
		log.Fatal(err)
	}

	w := watson.New(cfg.User, cfg.Pass)

	ml, err := w.GetModels()
	if err != nil {
		log.Fatal(err)
	}

	if args[1] == "-l" {
		if len(args) > 2 {
			log.Fatal("too many args")
		}

		for _, m := range ml.Models {
			fmt.Printf("%s %-8d=> %s\n", m.Lang, m.Rate, m.Name)
		}

		return
	}

	out := args[1]
	model := args[2]
	in := args[3]
	ext := ""
	switch path.Ext(in) {
	case ".wav":
		ext = "wav"
	case ".flac":
		ext = "flac"
	case ".json":
		ext = "json"
	default:
		log.Fatal("stt: unknown file format: ", in)
	}

	found := false
	for _, m := range ml.Models {
		if m.Name == model {
			found = true
		}
	}
	if !found {
		log.Fatal("model not found")
	}

	is, err := os.Open(in)
	if err != nil {
		log.Fatal(err)
	}
	defer is.Close()

	os, err := os.Create(out)
	if err != nil {
		log.Fatal(err)
	}
	defer os.Close()

	tt, err := w.Recognize(is, model, ext)
	if err != nil {
		log.Fatal(err)
	}

	for _, w := range tt.Words {
		fmt.Printf("%v\n", w)
	}
}
<|endoftext|>


<commit_before>package video

import "image"

var color_channel = [12]int{0, 0, 0, 0, 1, 2, 1, 2, 1, 2, 1, 2}

type Macroblock struct {
	macroblock_address_increment int
	macroblock_type              *MacroblockType
	spatial_temporal_weight_code uint32
	frame_motion_type            uint32
	field_motion_type            uint32
	dct_type                     bool

	cbp codedBlockPattern
}

func (br *VideoSequence) macroblock(
	// location
	mb_address, mb_row int,
	// dct predictors
	dcp *dcDctPredictors, resetDCPredictors dcDctPredictorResetter,
	mvd *motionVectorData,
	qsc *uint32,
	frameSlice *image.YCbCr) (int, error) {

	mb := Macroblock{}

	for {
		if nextbits, err := br.Peek32(11); err != nil {
			return 0, err
		} else if nextbits == 0x08 { // 0000 0001 000
			br.Trash(11)
			mb.macroblock_address_increment += 33
		} else {
			break
		}
	}

	if incr, err := macroblockAddressIncrementDecoder.Decode(br); err != nil {
		return 0, err
	} else {
		mb.macroblock_address_increment += incr
	}

	// Copy skipped macroblocks for PFrames and BFrames
	if mb.macroblock_address_increment > 1 {
		switch br.PictureHeader.picture_coding_type {
		case PFrame:
			pframe_copy_macroblocks(
				mb_row, mb_address+1,
				mb.macroblock_address_increment-1,
				frameSlice, br.frameStore.past)
		case BFrame:
			bframe_copy_macroblocks(
				mb_row, mb_address+1,
				mb.macroblock_address_increment-1,
				*mvd,
				br.frameStore,
				frameSlice)
		}
	}

	// Reset dcDctPredictors: whenever a macroblock is skipped. (7.2.1)
	if mb.macroblock_address_increment > 1 {
		resetDCPredictors()
	}

	// Reset motion vector predictors: P-picture with a skipped macroblock (7.6.4.3)
	if br.PictureHeader.picture_coding_type == PFrame &&
		mb.macroblock_address_increment > 1 {
		mvd.reset()
	}

	if err := br.macroblock_mode(&mb); err != nil {
		return 0, err
	}

	// Reset dcDctPredictors: whenever a non-intra macroblock is decoded. (7.2.1)
	if mb.macroblock_type.macroblock_intra == false {
		resetDCPredictors()
	}

	// Reset motion vector predictors: intra macroblock without concealment motion vectors (7.6.4.3)
	if mb.macroblock_type.macroblock_intra == true &&
		br.PictureCodingExtension.concealment_motion_vectors == false {
		mvd.reset()
	}

	// Reset motion vector predictors: non-intra P-picture with no forward motion vectors (7.6.4.3)
	if br.PictureHeader.picture_coding_type == PFrame &&
		mb.macroblock_type.macroblock_intra == false &&
		mb.macroblock_type.macroblock_motion_forward == false {
		mvd.reset()
	}

	if mb.macroblock_type.macroblock_quant {
		if mb_qsc, err := br.Read32(5); err != nil {
			return 0, err
		} else {
			*qsc = mb_qsc
		}
	}

	if mb.macroblock_type.macroblock_motion_forward ||
		(mb.macroblock_type.macroblock_intra && br.PictureCodingExtension.concealment_motion_vectors) {
		if err := br.motion_vectors(0, &mb, mvd); err != nil {
			return 0, err
		}
	}

	if mb.macroblock_type.macroblock_motion_backward {
		if err := br.motion_vectors(1, &mb, mvd); err != nil {
			return 0, err
		}
	}

	mvd.previous.set(mb.macroblock_type, br.PictureHeader.picture_coding_type)

	if mb.macroblock_type.macroblock_intra && br.PictureCodingExtension.concealment_motion_vectors {
		if err := marker_bit(br); err != nil {
			return 0, err
		}
	}

	if mb.macroblock_type.macroblock_pattern {
		if cbp, err := coded_block_pattern(br, br.SequenceExtension.chroma_format); err != nil {
			return 0, nil
		} else {
			mb.cbp = cbp
		}
	}

	var block_count int
	switch br.SequenceExtension.chroma_format {
	case ChromaFormat420:
		block_count = 6
	case ChromaFormat422:
		block_count = 8
	case ChromaFormat444:
		block_count = 12
	}

	mb_address += mb.macroblock_address_increment
	pattern_code := mb.cbp.decode(mb.macroblock_type.macroblock_intra, mb.macroblock_type.macroblock_pattern, br.SequenceExtension.chroma_format)

	var b block
	var cb clampedblock

	for i := 0; i < block_count; i++ {
		cc := color_channel[i]

		if pattern_code[i] {
			if err := b.read(br, dcp, br.PictureCodingExtension.intra_vlc_format, cc, mb.macroblock_type.macroblock_intra); err != nil {
				return 0, err
			}
			b.decode_block(br, cc, *qsc, mb.macroblock_type.macroblock_intra)
			b.idct()
		} else {
			b.zero()
		}

		b.motion_compensation(*mvd, i, mb_row, mb_address, br.frameStore)
		b.clamp(&cb)
		updateFrameSlice(i, mb_address, mb.dct_type, frameSlice, &cb)
	}

	return mb_address, nil
}

func updateFrameSlice(i, mb_address int, interlaced bool, frameSlice *image.YCbCr, cb *clampedblock) {

	var (
		base_i  int
		channel []uint8
		stride  int
	)

	// channel switch
	switch i {
	case 0, 1, 2, 3:
		channel = frameSlice.Y
	case 4:
		channel = frameSlice.Cb
	case 5:
		channel = frameSlice.Cr
	}

	// base address and stride switch
	switch i {
	case 0, 1, 2, 3:
		stride = frameSlice.YStride
		base_i = mb_address * 16
	case 4, 5:
		stride = frameSlice.CStride
		base_i = mb_address * 8
	}

	// position switch
	if interlaced {
		// Field DCT coding alternates lines from each block:
		//
		//  <-8px-> <-8px->
		//  ───0───│───1───
		//  ───2───│───3───
		//  ───0───│───1───
		//  ───2───│───3───
		//  ───0───│───1───
		//  ───2───│───3───
		//  ───0───│───1───
		//  ───2───│───3───
		switch i {
		case 0, 1, 2, 3:
			base_i += (i & 1) << 3
			base_i += ((i & 2) >> 1) * stride
			stride *= 2
		}
	} else {
		// Frame DCT coding are mapped in the follow order:
		//
		//  <-8px-> <-8px->
		//  ───0───│───1───
		//  ───0───│───1───
		//  ───0───│───1───
		//  ───0───│───1───
		//  ───2───│───3───
		//  ───2───│───3───
		//  ───2───│───3───
		//  ───2───│───3───
		switch i {
		case 0, 1, 2, 3:
			base_i += (i & 1) << 3            // horiztonal positioning
			base_i += ((i & 2) << 2) * stride // vertical positioning
		}
	}

	// perform copy
	for y := 0; y < 8; y++ {
		si := y * 8
		di := base_i + (y * stride)
		copy(channel[di:di+8], cb[si:si+8])
	}

}

func (br *VideoSequence) macroblock_mode(mb *Macroblock) (err error) {

	var typeDecoder macroblockTypeDecoderFn
	switch br.PictureHeader.picture_coding_type {
	case IFrame:
		typeDecoder = macroblockTypeDecoder.IFrame
	case PFrame:
		typeDecoder = macroblockTypeDecoder.PFrame
	case BFrame:
		typeDecoder = macroblockTypeDecoder.BFrame
	default:
		panic("not implemented: macroblock type decoder")
	}

	mb.macroblock_type, err = typeDecoder(br)
	if err != nil {
		return err
	}

	if mb.macroblock_type.spatial_temporal_weight_code_flag &&
		false /* ( spatial_temporal_weight_code_table_index != ‘00’) */ {
		mb.spatial_temporal_weight_code, err = br.Read32(2)
		if err != nil {
			return err
		}
	}

	if mb.macroblock_type.macroblock_motion_forward ||
		mb.macroblock_type.macroblock_motion_backward {
		if br.PictureCodingExtension.picture_structure == PictureStructure_FramePicture {
			if br.PictureCodingExtension.frame_pred_frame_dct == 0 {
				mb.frame_motion_type, err = br.Read32(2)
				if err != nil {
					return err
				}
			}
		} else {
			mb.field_motion_type, err = br.Read32(2)
			if err != nil {
				return err
			}
		}
	}

	if br.PictureCodingExtension.picture_structure == PictureStructure_FramePicture &&
		br.PictureCodingExtension.frame_pred_frame_dct == 0 &&
		(mb.macroblock_type.macroblock_intra || mb.macroblock_type.macroblock_pattern) {
		mb.dct_type, err = br.ReadBit() //dct_type 1 uimsbf
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>dont copy macroblocks if its the first macroblock of the slice that has been shifted.<commit_after>package video

import "image"

var color_channel = [12]int{0, 0, 0, 0, 1, 2, 1, 2, 1, 2, 1, 2}

type Macroblock struct {
	macroblock_address_increment int
	macroblock_type              *MacroblockType
	spatial_temporal_weight_code uint32
	frame_motion_type            uint32
	field_motion_type            uint32
	dct_type                     bool

	cbp codedBlockPattern
}

func (br *VideoSequence) macroblock(
	// location
	mb_address, mb_row int,
	// dct predictors
	dcp *dcDctPredictors, resetDCPredictors dcDctPredictorResetter,
	mvd *motionVectorData,
	qsc *uint32,
	frameSlice *image.YCbCr) (int, error) {

	mb := Macroblock{}

	for {
		if nextbits, err := br.Peek32(11); err != nil {
			return 0, err
		} else if nextbits == 0x08 { // 0000 0001 000
			br.Trash(11)
			mb.macroblock_address_increment += 33
		} else {
			break
		}
	}

	if incr, err := macroblockAddressIncrementDecoder.Decode(br); err != nil {
		return 0, err
	} else {
		mb.macroblock_address_increment += incr
	}

	// Copy skipped macroblocks for PFrames and BFrames
	if mb_address != -1 && mb.macroblock_address_increment > 1 {
		switch br.PictureHeader.picture_coding_type {
		case PFrame:
			pframe_copy_macroblocks(
				mb_row, mb_address+1,
				mb.macroblock_address_increment-1,
				frameSlice, br.frameStore.past)
		case BFrame:
			bframe_copy_macroblocks(
				mb_row, mb_address+1,
				mb.macroblock_address_increment-1,
				*mvd,
				br.frameStore,
				frameSlice)
		}
	}

	// Reset dcDctPredictors: whenever a macroblock is skipped. (7.2.1)
	if mb.macroblock_address_increment > 1 {
		resetDCPredictors()
	}

	// Reset motion vector predictors: P-picture with a skipped macroblock (7.6.4.3)
	if br.PictureHeader.picture_coding_type == PFrame &&
		mb.macroblock_address_increment > 1 {
		mvd.reset()
	}

	if err := br.macroblock_mode(&mb); err != nil {
		return 0, err
	}

	// Reset dcDctPredictors: whenever a non-intra macroblock is decoded. (7.2.1)
	if mb.macroblock_type.macroblock_intra == false {
		resetDCPredictors()
	}

	// Reset motion vector predictors: intra macroblock without concealment motion vectors (7.6.4.3)
	if mb.macroblock_type.macroblock_intra == true &&
		br.PictureCodingExtension.concealment_motion_vectors == false {
		mvd.reset()
	}

	// Reset motion vector predictors: non-intra P-picture with no forward motion vectors (7.6.4.3)
	if br.PictureHeader.picture_coding_type == PFrame &&
		mb.macroblock_type.macroblock_intra == false &&
		mb.macroblock_type.macroblock_motion_forward == false {
		mvd.reset()
	}

	if mb.macroblock_type.macroblock_quant {
		if mb_qsc, err := br.Read32(5); err != nil {
			return 0, err
		} else {
			*qsc = mb_qsc
		}
	}

	if mb.macroblock_type.macroblock_motion_forward ||
		(mb.macroblock_type.macroblock_intra && br.PictureCodingExtension.concealment_motion_vectors) {
		if err := br.motion_vectors(0, &mb, mvd); err != nil {
			return 0, err
		}
	}

	if mb.macroblock_type.macroblock_motion_backward {
		if err := br.motion_vectors(1, &mb, mvd); err != nil {
			return 0, err
		}
	}

	mvd.previous.set(mb.macroblock_type, br.PictureHeader.picture_coding_type)

	if mb.macroblock_type.macroblock_intra && br.PictureCodingExtension.concealment_motion_vectors {
		if err := marker_bit(br); err != nil {
			return 0, err
		}
	}

	if mb.macroblock_type.macroblock_pattern {
		if cbp, err := coded_block_pattern(br, br.SequenceExtension.chroma_format); err != nil {
			return 0, nil
		} else {
			mb.cbp = cbp
		}
	}

	var block_count int
	switch br.SequenceExtension.chroma_format {
	case ChromaFormat420:
		block_count = 6
	case ChromaFormat422:
		block_count = 8
	case ChromaFormat444:
		block_count = 12
	}

	mb_address += mb.macroblock_address_increment
	pattern_code := mb.cbp.decode(mb.macroblock_type.macroblock_intra, mb.macroblock_type.macroblock_pattern, br.SequenceExtension.chroma_format)

	var b block
	var cb clampedblock

	for i := 0; i < block_count; i++ {
		cc := color_channel[i]

		if pattern_code[i] {
			if err := b.read(br, dcp, br.PictureCodingExtension.intra_vlc_format, cc, mb.macroblock_type.macroblock_intra); err != nil {
				return 0, err
			}
			b.decode_block(br, cc, *qsc, mb.macroblock_type.macroblock_intra)
			b.idct()
		} else {
			b.zero()
		}

		b.motion_compensation(*mvd, i, mb_row, mb_address, br.frameStore)
		b.clamp(&cb)
		updateFrameSlice(i, mb_address, mb.dct_type, frameSlice, &cb)
	}

	return mb_address, nil
}

func updateFrameSlice(i, mb_address int, interlaced bool, frameSlice *image.YCbCr, cb *clampedblock) {

	var (
		base_i  int
		channel []uint8
		stride  int
	)

	// channel switch
	switch i {
	case 0, 1, 2, 3:
		channel = frameSlice.Y
	case 4:
		channel = frameSlice.Cb
	case 5:
		channel = frameSlice.Cr
	}

	// base address and stride switch
	switch i {
	case 0, 1, 2, 3:
		stride = frameSlice.YStride
		base_i = mb_address * 16
	case 4, 5:
		stride = frameSlice.CStride
		base_i = mb_address * 8
	}

	// position switch
	if interlaced {
		// Field DCT coding alternates lines from each block:
		//
		//  <-8px-> <-8px->
		//  ───0───│───1───
		//  ───2───│───3───
		//  ───0───│───1───
		//  ───2───│───3───
		//  ───0───│───1───
		//  ───2───│───3───
		//  ───0───│───1───
		//  ───2───│───3───
		switch i {
		case 0, 1, 2, 3:
			base_i += (i & 1) << 3
			base_i += ((i & 2) >> 1) * stride
			stride *= 2
		}
	} else {
		// Frame DCT coding are mapped in the follow order:
		//
		//  <-8px-> <-8px->
		//  ───0───│───1───
		//  ───0───│───1───
		//  ───0───│───1───
		//  ───0───│───1───
		//  ───2───│───3───
		//  ───2───│───3───
		//  ───2───│───3───
		//  ───2───│───3───
		switch i {
		case 0, 1, 2, 3:
			base_i += (i & 1) << 3            // horiztonal positioning
			base_i += ((i & 2) << 2) * stride // vertical positioning
		}
	}

	// perform copy
	for y := 0; y < 8; y++ {
		si := y * 8
		di := base_i + (y * stride)
		copy(channel[di:di+8], cb[si:si+8])
	}

}

func (br *VideoSequence) macroblock_mode(mb *Macroblock) (err error) {

	var typeDecoder macroblockTypeDecoderFn
	switch br.PictureHeader.picture_coding_type {
	case IFrame:
		typeDecoder = macroblockTypeDecoder.IFrame
	case PFrame:
		typeDecoder = macroblockTypeDecoder.PFrame
	case BFrame:
		typeDecoder = macroblockTypeDecoder.BFrame
	default:
		panic("not implemented: macroblock type decoder")
	}

	mb.macroblock_type, err = typeDecoder(br)
	if err != nil {
		return err
	}

	if mb.macroblock_type.spatial_temporal_weight_code_flag &&
		false /* ( spatial_temporal_weight_code_table_index != ‘00’) */ {
		mb.spatial_temporal_weight_code, err = br.Read32(2)
		if err != nil {
			return err
		}
	}

	if mb.macroblock_type.macroblock_motion_forward ||
		mb.macroblock_type.macroblock_motion_backward {
		if br.PictureCodingExtension.picture_structure == PictureStructure_FramePicture {
			if br.PictureCodingExtension.frame_pred_frame_dct == 0 {
				mb.frame_motion_type, err = br.Read32(2)
				if err != nil {
					return err
				}
			}
		} else {
			mb.field_motion_type, err = br.Read32(2)
			if err != nil {
				return err
			}
		}
	}

	if br.PictureCodingExtension.picture_structure == PictureStructure_FramePicture &&
		br.PictureCodingExtension.frame_pred_frame_dct == 0 &&
		(mb.macroblock_type.macroblock_intra || mb.macroblock_type.macroblock_pattern) {
		mb.dct_type, err = br.ReadBit() //dct_type 1 uimsbf
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// +build !plan9,!solaris

package main

import (
	"log"
	"os"
	"path/filepath"
	"time"

	"github.com/fsnotify/fsnotify"
)

func WaitForReplacement(filename string, watcher *fsnotify.Watcher) {
	for i := 0; i < 20; i++ {
		time.Sleep(100 * time.Millisecond)

		if _, err := os.Stat(filename); err == nil {
			if err := watcher.Add(filename); err == nil {
				log.Printf("watching resumed for %s", filename)
				return
			}
		}
	}
	log.Printf("failed to resume watching for %s", filename)
}

func WatchForUpdates(filename string, done <-chan bool, action func()) {
	filename = filepath.Clean(filename)
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Fatal("failed to create watcher for ", filename, ": ", err)
	}
	go func() {
		for {
			select {
			case _ = <-done:
				log.Printf("Shutting down watcher for: %s", filename)
				watcher.Close()
				return
			case event := <-watcher.Events:
				// On Arch Linux, it appears Chmod events precede Remove events,
				// which causes a race between action() and the coming Remove event.
				if event.Op == fsnotify.Chmod {
					continue
				}
				if event.Op&(fsnotify.Remove|fsnotify.Rename) != 0 {
					log.Printf("watching interrupted on event: %s", event)
					watcher.Remove(filename)
					WaitForReplacement(filename, watcher)
				}
				log.Printf("reloading after event: %s", event)
				action()
			case err := <-watcher.Errors:
				log.Printf("error watching %s: %s", filename, err)
			}
		}
	}()
	if err = watcher.Add(filename); err != nil {
		log.Fatal("failed to add ", filename, " to watcher: ", err)
	}
	log.Printf("watching %s for updates", filename)
}
<commit_msg>watcher: minor refactor<commit_after>// +build !plan9,!solaris

package main

import (
	"log"
	"os"
	"path/filepath"
	"time"

	"github.com/fsnotify/fsnotify"
)

func WaitForReplacement(filename string, watcher *fsnotify.Watcher) {
	for i := 0; i < 50; i++ {
		time.Sleep(100 * time.Millisecond)

		if _, err := os.Stat(filename); err == nil {
			if err := watcher.Add(filename); err == nil {
				log.Printf("watching resumed for %s", filename)
				return
			}
		}
	}
	log.Printf("failed to resume watching for %s", filename)
}

func watchLoop(filename string, watcher *fsnotify.Watcher, done <-chan bool, action func()) {
	for {
		select {
		case _ = <-done:
			log.Printf("Shutting down watcher for: %s", filename)
			watcher.Close()
			return
		case event := <-watcher.Events:
			// On Arch Linux, it appears Chmod events precede Remove events,
			// which causes a race between action() and the coming Remove event.
			if event.Op == fsnotify.Chmod {
				continue
			}
			if event.Op&(fsnotify.Remove|fsnotify.Rename) != 0 {
				log.Printf("watching interrupted on event: %s", event)
				watcher.Remove(filename)
				WaitForReplacement(filename, watcher)
			}
			log.Printf("reloading after event: %s", event)
			action()
		case err := <-watcher.Errors:
			log.Printf("error watching %s: %s", filename, err)
		}
	}
}

func WatchForUpdates(filename string, done <-chan bool, action func()) {
	filename = filepath.Clean(filename)
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Fatal("failed to create watcher for ", filename, ": ", err)
	}
	if err = watcher.Add(filename); err != nil {
		log.Fatal("failed to add ", filename, " to watcher: ", err)
	}
	go watchLoop(filename, watcher, done, action)
	log.Printf("watching %s for updates", filename)
}
<|endoftext|>


<commit_before>package main_test

import (
	"crypto/tls"
	"fmt"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"io/ioutil"
	"net/http"
	"net/url"
	"os/exec"
	"strings"
)

var _ = Describe("Assuming chaos-galago is deployed", func() {
	var client *http.Client

	BeforeEach(func() {
		tr := &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
		client = &http.Client{Transport: tr}
	})

	Describe("service instances", func() {
		Context("when the service instance exists", func() {
			var (
				dashboardURL string
			)

			BeforeEach(func() {
				output, err := exec.Command("cf", "create-service", "chaos-galago", "default", serviceInstanceName).Output()
				freakOutDebug(output, err)
				output, err = exec.Command("cf", "service", serviceInstanceName).Output()
				freakOutDebug(output, err)
				firstSplit := strings.SplitAfter(string(output), "Dashboard: ")[1]
				dashboardURL = strings.TrimSpace(strings.SplitAfter(firstSplit, "\n")[0])
			})

			AfterEach(func() {
				output, err := exec.Command("cf", "delete-service", "-f", serviceInstanceName).Output()
				freakOutDebug(output, err)
			})

			It("creates a service instance", func() {
				output, _ := exec.Command("cf", "create-service", "chaos-galago", "default", serviceInstanceName).Output()
				Expect(string(output)).To(MatchRegexp("OK"))
				Expect(string(output)).To(MatchRegexp("already exists"))
				output, _ = exec.Command("cf", "services").Output()
				Expect(string(output)).To(MatchRegexp(serviceInstanceName))
			})

			It("updates a service instance", func() {
				resp, _ := client.PostForm(dashboardURL, url.Values{"probability": {"1"}, "frequency": {"1"}})
				defer resp.Body.Close()
				body, _ := ioutil.ReadAll(resp.Body)
				Expect(string(body)).To(MatchRegexp("Probability: 1"))
				Expect(string(body)).To(MatchRegexp("Frequency: 1"))
			})

			It("deletes a service instance", func() {
				output, _ := exec.Command("cf", "delete-service", "-f", serviceInstanceName).Output()
				Expect(string(output)).To(MatchRegexp("OK"))
				Expect(string(output)).ToNot(MatchRegexp("does not exist"))
				output, _ = exec.Command("cf", "services").Output()
				Expect(string(output)).ToNot(MatchRegexp(serviceInstanceName))
			})
		})

		Context("When the service instance does not exist", func() {
			AfterEach(func() {
				output, err := exec.Command("cf", "delete-service", "-f", serviceInstanceName).Output()
				freakOutDebug(output, err)
			})

			It("creates a service instance", func() {
				output, _ := exec.Command("cf", "create-service", "chaos-galago", "default", serviceInstanceName).Output()
				Expect(string(output)).To(MatchRegexp("OK"))
				Expect(string(output)).ToNot(MatchRegexp("already exists"))
				output, _ = exec.Command("cf", "services").Output()
				Expect(string(output)).To(MatchRegexp(serviceInstanceName))
			})

			It("deletes a service instance", func() {
				output, _ := exec.Command("cf", "delete-service", "-f", serviceInstanceName).Output()
				Expect(string(output)).To(MatchRegexp("OK"))
				Expect(string(output)).To(MatchRegexp("does not exist"))
				output, _ = exec.Command("cf", "services").Output()
				Expect(string(output)).ToNot(MatchRegexp(serviceInstanceName))
			})
		})

	})

	Describe("service bindings", func() {
		BeforeEach(func() {
			output, err := exec.Command("cf", "create-service", "chaos-galago", "default", serviceInstanceName).Output()
			freakOutDebug(output, err)
		})

		AfterEach(func() {
			output, err := exec.Command("cf", "delete-service", "-f", serviceInstanceName).Output()
			freakOutDebug(output, err)
		})

		Context("when an app is bound", func() {
			BeforeEach(func() {
				output, err := exec.Command("cf", "bind-service", appName, serviceInstanceName).Output()
				freakOutDebug(output, err)
			})

			AfterEach(func() {
				output, err := exec.Command("cf", "unbind-service", appName, serviceInstanceName).Output()
				freakOutDebug(output, err)
			})

			It("bind a service instance", func() {
				output, _ := exec.Command("cf", "bind-service", appName, serviceInstanceName).Output()
				Expect(string(output)).To(MatchRegexp("OK"))
				Expect(string(output)).To(MatchRegexp("already bound"))
				output, _ = exec.Command("cf", "env", appName).Output()
				Expect(string(output)).To(MatchRegexp(`label": "chaos-galago"`))
				Expect(string(output)).To(MatchRegexp(`frequency": 5`))
				Expect(string(output)).To(MatchRegexp(`probability": 0.2`))
			})

			It("unbind a service instance", func() {
				output, _ := exec.Command("cf", "unbind-service", appName, serviceInstanceName).Output()
				Expect(string(output)).To(MatchRegexp("OK"))
				Expect(string(output)).ToNot(MatchRegexp("did not exist"))
				output, _ = exec.Command("cf", "env", appName).Output()
				Expect(string(output)).ToNot(MatchRegexp(`label": "chaos-galago"`))
				Expect(string(output)).ToNot(MatchRegexp(`frequency": 5`))
				Expect(string(output)).ToNot(MatchRegexp(`probability": 0.2`))
			})

			Context("the processor", func() {
				var (
					appGUID      string
					dashboardURL string
				)

				BeforeEach(func() {
					output, err := exec.Command("cf", "app", appName, "--guid").Output()
					freakOutDebug(output, err)
					appGUID = strings.TrimSpace(string(output))
					output, err = exec.Command("cf", "service", serviceInstanceName).Output()
					freakOutDebug(output, err)
					firstSplit := strings.SplitAfter(string(output), "Dashboard: ")[1]
					dashboardURL = strings.TrimSpace(strings.SplitAfter(firstSplit, "\n")[0])
					client.PostForm(dashboardURL, url.Values{"probability": {"1"}, "frequency": {"1"}})
					exec.Command("cf", "target", "-o", "chaos-galago", "-s", "chaos-galago").Run()
				})

				AfterEach(func() {
					output, err := exec.Command("cf", "target", "-o", orgName, "-s", spaceName).Output()
					freakOutDebug(output, err)
				})

				It("acts on bound aplications", func() {
					Eventually(func() string {
						output, _ := exec.Command("cf", "curl", fmt.Sprintf("v2/apps/%s/instances", appGUID)).Output()
						return string(output)
					}, "120s", "1s").Should(MatchRegexp(`"state": "RUNNING"`))
					Eventually(func() string {
						output, _ := exec.Command("cf", "logs", "chaos-galago-processor", "--recent").Output()
						return string(output)
					}, "120s", "1s").Should(MatchRegexp(fmt.Sprintf("About to kill app instance: %s at index: 0", appGUID)))
					Eventually(func() string {
						output, _ := exec.Command("cf", "curl", fmt.Sprintf("v2/apps/%s/instances", appGUID)).Output()
						return string(output)
					}, "120s", "1s").Should(MatchRegexp(`"state": "DOWN"`))
				})
			})
		})

		Context("when an app is not bound", func() {
			AfterEach(func() {
				output, err := exec.Command("cf", "unbind-service", appName, serviceInstanceName).Output()
				freakOutDebug(output, err)
			})

			It("bind a service instance", func() {
				output, _ := exec.Command("cf", "bind-service", appName, serviceInstanceName).Output()
				Expect(string(output)).To(MatchRegexp("OK"))
				Expect(string(output)).ToNot(MatchRegexp("already bound"))
				output, _ = exec.Command("cf", "env", appName).Output()
				Expect(string(output)).To(MatchRegexp(`label": "chaos-galago"`))
				Expect(string(output)).To(MatchRegexp(`frequency": 5`))
				Expect(string(output)).To(MatchRegexp(`probability": 0.2`))
			})

			It("unbind a service instance", func() {
				output, _ := exec.Command("cf", "unbind-service", appName, serviceInstanceName).Output()
				Expect(string(output)).To(MatchRegexp("OK"))
				Expect(string(output)).To(MatchRegexp("did not exist"))
				output, _ = exec.Command("cf", "env", appName).Output()
				Expect(string(output)).ToNot(MatchRegexp(`label": "chaos-galago"`))
				Expect(string(output)).ToNot(MatchRegexp(`frequency": 5`))
				Expect(string(output)).ToNot(MatchRegexp(`probability": 0.2`))
			})
		})
	})
})
<commit_msg>Allow extra time to start application due to slower diego start times.<commit_after>package main_test

import (
	"crypto/tls"
	"fmt"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"io/ioutil"
	"net/http"
	"net/url"
	"os/exec"
	"strings"
)

var _ = Describe("Assuming chaos-galago is deployed", func() {
	var client *http.Client

	BeforeEach(func() {
		tr := &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
		client = &http.Client{Transport: tr}
	})

	Describe("service instances", func() {
		Context("when the service instance exists", func() {
			var (
				dashboardURL string
			)

			BeforeEach(func() {
				output, err := exec.Command("cf", "create-service", "chaos-galago", "default", serviceInstanceName).Output()
				freakOutDebug(output, err)
				output, err = exec.Command("cf", "service", serviceInstanceName).Output()
				freakOutDebug(output, err)
				firstSplit := strings.SplitAfter(string(output), "Dashboard: ")[1]
				dashboardURL = strings.TrimSpace(strings.SplitAfter(firstSplit, "\n")[0])
			})

			AfterEach(func() {
				output, err := exec.Command("cf", "delete-service", "-f", serviceInstanceName).Output()
				freakOutDebug(output, err)
			})

			It("creates a service instance", func() {
				output, _ := exec.Command("cf", "create-service", "chaos-galago", "default", serviceInstanceName).Output()
				Expect(string(output)).To(MatchRegexp("OK"))
				Expect(string(output)).To(MatchRegexp("already exists"))
				output, _ = exec.Command("cf", "services").Output()
				Expect(string(output)).To(MatchRegexp(serviceInstanceName))
			})

			It("updates a service instance", func() {
				resp, _ := client.PostForm(dashboardURL, url.Values{"probability": {"1"}, "frequency": {"1"}})
				defer resp.Body.Close()
				body, _ := ioutil.ReadAll(resp.Body)
				Expect(string(body)).To(MatchRegexp("Probability: 1"))
				Expect(string(body)).To(MatchRegexp("Frequency: 1"))
			})

			It("deletes a service instance", func() {
				output, _ := exec.Command("cf", "delete-service", "-f", serviceInstanceName).Output()
				Expect(string(output)).To(MatchRegexp("OK"))
				Expect(string(output)).ToNot(MatchRegexp("does not exist"))
				output, _ = exec.Command("cf", "services").Output()
				Expect(string(output)).ToNot(MatchRegexp(serviceInstanceName))
			})
		})

		Context("When the service instance does not exist", func() {
			AfterEach(func() {
				output, err := exec.Command("cf", "delete-service", "-f", serviceInstanceName).Output()
				freakOutDebug(output, err)
			})

			It("creates a service instance", func() {
				output, _ := exec.Command("cf", "create-service", "chaos-galago", "default", serviceInstanceName).Output()
				Expect(string(output)).To(MatchRegexp("OK"))
				Expect(string(output)).ToNot(MatchRegexp("already exists"))
				output, _ = exec.Command("cf", "services").Output()
				Expect(string(output)).To(MatchRegexp(serviceInstanceName))
			})

			It("deletes a service instance", func() {
				output, _ := exec.Command("cf", "delete-service", "-f", serviceInstanceName).Output()
				Expect(string(output)).To(MatchRegexp("OK"))
				Expect(string(output)).To(MatchRegexp("does not exist"))
				output, _ = exec.Command("cf", "services").Output()
				Expect(string(output)).ToNot(MatchRegexp(serviceInstanceName))
			})
		})

	})

	Describe("service bindings", func() {
		BeforeEach(func() {
			output, err := exec.Command("cf", "create-service", "chaos-galago", "default", serviceInstanceName).Output()
			freakOutDebug(output, err)
		})

		AfterEach(func() {
			output, err := exec.Command("cf", "delete-service", "-f", serviceInstanceName).Output()
			freakOutDebug(output, err)
		})

		Context("when an app is bound", func() {
			BeforeEach(func() {
				output, err := exec.Command("cf", "bind-service", appName, serviceInstanceName).Output()
				freakOutDebug(output, err)
			})

			AfterEach(func() {
				output, err := exec.Command("cf", "unbind-service", appName, serviceInstanceName).Output()
				freakOutDebug(output, err)
			})

			It("bind a service instance", func() {
				output, _ := exec.Command("cf", "bind-service", appName, serviceInstanceName).Output()
				Expect(string(output)).To(MatchRegexp("OK"))
				Expect(string(output)).To(MatchRegexp("already bound"))
				output, _ = exec.Command("cf", "env", appName).Output()
				Expect(string(output)).To(MatchRegexp(`label": "chaos-galago"`))
				Expect(string(output)).To(MatchRegexp(`frequency": 5`))
				Expect(string(output)).To(MatchRegexp(`probability": 0.2`))
			})

			It("unbind a service instance", func() {
				output, _ := exec.Command("cf", "unbind-service", appName, serviceInstanceName).Output()
				Expect(string(output)).To(MatchRegexp("OK"))
				Expect(string(output)).ToNot(MatchRegexp("did not exist"))
				output, _ = exec.Command("cf", "env", appName).Output()
				Expect(string(output)).ToNot(MatchRegexp(`label": "chaos-galago"`))
				Expect(string(output)).ToNot(MatchRegexp(`frequency": 5`))
				Expect(string(output)).ToNot(MatchRegexp(`probability": 0.2`))
			})

			Context("the processor", func() {
				var (
					appGUID      string
					dashboardURL string
				)

				BeforeEach(func() {
					output, err := exec.Command("cf", "app", appName, "--guid").Output()
					freakOutDebug(output, err)
					appGUID = strings.TrimSpace(string(output))
					output, err = exec.Command("cf", "service", serviceInstanceName).Output()
					freakOutDebug(output, err)
					firstSplit := strings.SplitAfter(string(output), "Dashboard: ")[1]
					dashboardURL = strings.TrimSpace(strings.SplitAfter(firstSplit, "\n")[0])
					client.PostForm(dashboardURL, url.Values{"probability": {"1"}, "frequency": {"1"}})
					exec.Command("cf", "target", "-o", "chaos-galago", "-s", "chaos-galago").Run()
				})

				AfterEach(func() {
					output, err := exec.Command("cf", "target", "-o", orgName, "-s", spaceName).Output()
					freakOutDebug(output, err)
				})

				It("acts on bound aplications", func() {
					Eventually(func() string {
						output, _ := exec.Command("cf", "curl", fmt.Sprintf("v2/apps/%s/instances", appGUID)).Output()
						return string(output)
					}, "180s", "1s").Should(MatchRegexp(`"state": "RUNNING"`))
					Eventually(func() string {
						output, _ := exec.Command("cf", "logs", "chaos-galago-processor", "--recent").Output()
						return string(output)
					}, "180s", "1s").Should(MatchRegexp(fmt.Sprintf("About to kill app instance: %s at index: 0", appGUID)))
					Eventually(func() string {
						output, _ := exec.Command("cf", "curl", fmt.Sprintf("v2/apps/%s/instances", appGUID)).Output()
						return string(output)
					}, "180s", "1s").Should(MatchRegexp(`"state": "DOWN"`))
				})
			})
		})

		Context("when an app is not bound", func() {
			AfterEach(func() {
				output, err := exec.Command("cf", "unbind-service", appName, serviceInstanceName).Output()
				freakOutDebug(output, err)
			})

			It("bind a service instance", func() {
				output, _ := exec.Command("cf", "bind-service", appName, serviceInstanceName).Output()
				Expect(string(output)).To(MatchRegexp("OK"))
				Expect(string(output)).ToNot(MatchRegexp("already bound"))
				output, _ = exec.Command("cf", "env", appName).Output()
				Expect(string(output)).To(MatchRegexp(`label": "chaos-galago"`))
				Expect(string(output)).To(MatchRegexp(`frequency": 5`))
				Expect(string(output)).To(MatchRegexp(`probability": 0.2`))
			})

			It("unbind a service instance", func() {
				output, _ := exec.Command("cf", "unbind-service", appName, serviceInstanceName).Output()
				Expect(string(output)).To(MatchRegexp("OK"))
				Expect(string(output)).To(MatchRegexp("did not exist"))
				output, _ = exec.Command("cf", "env", appName).Output()
				Expect(string(output)).ToNot(MatchRegexp(`label": "chaos-galago"`))
				Expect(string(output)).ToNot(MatchRegexp(`frequency": 5`))
				Expect(string(output)).ToNot(MatchRegexp(`probability": 0.2`))
			})
		})
	})
})
<|endoftext|>


<commit_before>package web

import (
	"fmt"
	"net/http"

	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/workers/mails"
	"github.com/cozy/cozy-stack/web/statik"
	"github.com/cozy/echo"
)

// devMailHandler allow to easily render a mail from a route of the stack. The
// query parameters are used as data input for the mail template. The
// ContentType query parameter allow to render the mail in "text/html" or
// "text/plain".
func devMailsHandler(c echo.Context) error {
	name := c.Param("name")
	locale := c.QueryParam("locale")
	if locale == "" {
		locale = statik.GetLanguageFromHeader(c.Request().Header)
	}

	recipientName := c.QueryParam("RecipientName")
	if recipientName == "" {
		recipientName = "Jean Dupont"
	}

	_, parts, err := mails.RenderMail(name, locale, recipientName, devData(c))
	if err != nil {
		return err
	}

	contentType := c.QueryParam("ContentType")
	if contentType == "" {
		contentType = "text/html"
	}

	var part *mails.Part
	for _, p := range parts {
		if p.Type == contentType {
			part = p
		}
	}
	if part == nil {
		return echo.NewHTTPError(http.StatusNotFound,
			fmt.Errorf("Could not find template %q with content-type %q", name, contentType))
	}

	// Remove all CSP policies to display HTML email. this is a dev-only
	// handler, no need to worry.
	c.Response().Header().Set(echo.HeaderContentSecurityPolicy, "")
	if part.Type == "text/html" {
		return c.HTML(http.StatusOK, part.Body)
	}
	return c.String(http.StatusOK, part.Body)
}

// devTemplatesHandler allow to easily render a given template from a route of
// the stack. The query parameters are used as data input for the template.
func devTemplatesHandler(c echo.Context) error {
	name := c.Param("name")
	return c.Render(http.StatusOK, name, devData(c))
}

func devData(c echo.Context) echo.Map {
	data := make(echo.Map)
	for k, v := range c.QueryParams() {
		if len(v) > 0 {
			data[k] = v[0]
		}
	}
	if _, ok := data["Domain"]; !ok {
		data["Domain"] = c.Request().Host
	}
	if _, ok := data["ContextName"]; !ok {
		data["ContextName"] = config.DefaultInstanceContext
	}
	return data
}
<commit_msg>Fix some helpers for /dev/templates/... (#1790)<commit_after>package web

import (
	"fmt"
	"net/http"

	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/workers/mails"
	"github.com/cozy/cozy-stack/web/middlewares"
	"github.com/cozy/cozy-stack/web/statik"
	"github.com/cozy/echo"
)

// devMailHandler allow to easily render a mail from a route of the stack. The
// query parameters are used as data input for the mail template. The
// ContentType query parameter allow to render the mail in "text/html" or
// "text/plain".
func devMailsHandler(c echo.Context) error {
	name := c.Param("name")
	locale := c.QueryParam("locale")
	if locale == "" {
		locale = statik.GetLanguageFromHeader(c.Request().Header)
	}

	recipientName := c.QueryParam("RecipientName")
	if recipientName == "" {
		recipientName = "Jean Dupont"
	}

	_, parts, err := mails.RenderMail(name, locale, recipientName, devData(c))
	if err != nil {
		return err
	}

	contentType := c.QueryParam("ContentType")
	if contentType == "" {
		contentType = "text/html"
	}

	var part *mails.Part
	for _, p := range parts {
		if p.Type == contentType {
			part = p
		}
	}
	if part == nil {
		return echo.NewHTTPError(http.StatusNotFound,
			fmt.Errorf("Could not find template %q with content-type %q", name, contentType))
	}

	// Remove all CSP policies to display HTML email. this is a dev-only
	// handler, no need to worry.
	c.Response().Header().Set(echo.HeaderContentSecurityPolicy, "")
	if part.Type == "text/html" {
		return c.HTML(http.StatusOK, part.Body)
	}
	return c.String(http.StatusOK, part.Body)
}

// devTemplatesHandler allow to easily render a given template from a route of
// the stack. The query parameters are used as data input for the template.
func devTemplatesHandler(c echo.Context) error {
	name := c.Param("name")
	return c.Render(http.StatusOK, name, devData(c))
}

func devData(c echo.Context) echo.Map {
	data := make(echo.Map)
	for k, v := range c.QueryParams() {
		if len(v) > 0 {
			data[k] = v[0]
		}
	}
	if _, ok := data["Domain"]; !ok {
		data["Domain"] = c.Request().Host
	}
	if _, ok := data["ContextName"]; !ok {
		data["ContextName"] = config.DefaultInstanceContext
	}
	if i, err := instance.Get(c.Request().Host); err == nil {
		data["CozyUI"] = middlewares.CozyUI(i)
		data["ThemeCSS"] = middlewares.ThemeCSS(i)
		data["Favicon"] = middlewares.Favicon(i)
	}
	return data
}
<|endoftext|>


<commit_before>package inigo_test

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"fmt"
	"github.com/cloudfoundry-incubator/inigo/loggredile"
	"github.com/fraenkel/candiedyaml"
	"io"
	"io/ioutil"

	"github.com/cloudfoundry-incubator/inigo/inigo_server"
	"github.com/cloudfoundry-incubator/inigo/stager_runner"
	"github.com/cloudfoundry-incubator/inigo/zipper"
	"github.com/cloudfoundry-incubator/runtime-schema/models/factories"
	"github.com/cloudfoundry/yagnats"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Stager", func() {
	var otherStagerRunner *stager_runner.StagerRunner

	BeforeEach(func() {
		fileServerRunner.Start()
		otherStagerRunner = stager_runner.New(
			stagerPath,
			etcdRunner.NodeURLS(),
			[]string{fmt.Sprintf("127.0.0.1:%d", natsPort)},
		)
	})

	Context("when unable to find an appropriate compiler", func() {
		BeforeEach(func() {
			executorRunner.Start()
			stagerRunner.Start()
		})

		It("returns an error", func() {
			receivedMessages := make(chan *yagnats.Message)
			natsRunner.MessageBus.Subscribe("compiler-stagers-test", func(message *yagnats.Message) {
				receivedMessages <- message
			})

			natsRunner.MessageBus.PublishWithReplyTo(
				"diego.staging.start",
				"compiler-stagers-test",
				[]byte(`{"app_id": "some-app-guid", "task_id": "some-task-id", "stack": "no-compiler"}`))

			var receivedMessage *yagnats.Message
			Eventually(receivedMessages, 2.0).Should(Receive(&receivedMessage))
			Ω(receivedMessage.Payload).Should(ContainSubstring("no compiler defined for requested stack"))
			Consistently(receivedMessages, 2.0).ShouldNot(Receive())
		})
	})

	Describe("Staging", func() {
		var outputGuid string
		var stagingMessage []byte
		var buildpackToUse string

		BeforeEach(func() {
			executorRunner.Start()

			buildpackToUse = "admin_buildpack.zip"
			outputGuid = factories.GenerateGuid()

			fileServerRunner.ServeFile("smelter.zip", smelterZipPath)

			//make and upload an app
			var appFiles = []zipper.ZipFile{
				{"my-app", "scooby-doo"},
			}
			zipper.CreateZipFile("/tmp/app.zip", appFiles)
			inigoserver.UploadFile("app.zip", "/tmp/app.zip")

			//make and upload a buildpack
			var adminBuildpackFiles = []zipper.ZipFile{
				{"bin/detect", `#!/bin/bash
				echo My Buildpack
				`},
				{"bin/compile", `#!/bin/bash
				echo COMPILING BUILDPACK
				echo $SOME_STAGING_ENV
				touch $1/compiled
				`},
				{"bin/release", `#!/bin/bash
cat <<EOF
---
default_process_types:
  web: start-command
EOF
				`},
			}
			zipper.CreateZipFile("/tmp/admin_buildpack.zip", adminBuildpackFiles)
			inigoserver.UploadFile("admin_buildpack.zip", "/tmp/admin_buildpack.zip")

			var bustedAdminBuildpackFiles = []zipper.ZipFile{
				{"bin/detect", `#!/bin/bash]
				exit 1
				`},
				{"bin/compile", `#!/bin/bash`},
				{"bin/release", `#!/bin/bash`},
			}
			zipper.CreateZipFile("/tmp/busted_admin_buildpack.zip", bustedAdminBuildpackFiles)
			inigoserver.UploadFile("busted_admin_buildpack.zip", "/tmp/busted_admin_buildpack.zip")
		})

		JustBeforeEach(func() {
			stagingMessage = []byte(
				fmt.Sprintf(
					`{
						"app_id": "some-app-guid",
						"task_id": "some-task-id",
						"memory_mb": 128,
						"disk_mb": 128,
						"file_descriptors": 1024,
						"stack": "default",
						"app_bits_download_uri": "%s",
						"buildpacks" : [{ "key": "test-buildpack", "url": "%s" }],
						"environment": [["SOME_STAGING_ENV", "%s"]]
					}`,
					inigoserver.DownloadUrl("app.zip"),
					inigoserver.DownloadUrl(buildpackToUse),
					outputGuid,
				),
			)
		})

		Context("with one stager running", func() {
			BeforeEach(func() {
				stagerRunner.Start("--compilers", `{"default":"smelter.zip"}`)
			})

			It("runs the compiler on the executor with the correct environment variables, bits and log tag, and responds with the detected buildpack", func() {
				//listen for NATS response
				payloads := make(chan []byte)

				natsRunner.MessageBus.Subscribe("stager-test", func(msg *yagnats.Message) {
					payloads <- msg.Payload
				})

				//stream logs
				messages, stop := loggredile.StreamMessages(
					loggregatorRunner.Config.OutgoingPort,
					"/tail/?app=some-app-guid",
				)
				defer close(stop)

				logOutput := ""
				go func() {
					for message := range messages {
						Ω(message.GetSourceName()).To(Equal("STG"))
						logOutput += string(message.GetMessage()) + "\n"
					}
				}()

				//publish the staging message
				err := natsRunner.MessageBus.PublishWithReplyTo(
					"diego.staging.start",
					"stager-test",
					stagingMessage,
				)
				Ω(err).ShouldNot(HaveOccurred())

				//wait for staging to complete
				var payload []byte
				Eventually(payloads, 10.0).Should(Receive(&payload))

				//Assert on the staging output (detected buildpack)
				Ω(string(payload)).Should(Equal(`{"detected_buildpack":"My Buildpack"}`))

				//Asser the user saw reasonable output
				Eventually(func() string {
					return logOutput
				}).Should(ContainSubstring("COMPILING BUILDPACK"))
				Ω(logOutput).Should(ContainSubstring(outputGuid))

				//Fetch the compiled droplet from the fakeCC
				dropletData, ok := fakeCC.UploadedDroplets["some-app-guid"]
				Ω(ok).Should(BeTrue())
				Ω(dropletData).ShouldNot(BeEmpty())

				//Unzip the droplet
				ungzippedDropletData, err := gzip.NewReader(bytes.NewReader(dropletData))
				Ω(err).ShouldNot(HaveOccurred())

				//Untar the droplet
				untarredDropletData := tar.NewReader(ungzippedDropletData)
				dropletContents := map[string][]byte{}
				for {
					hdr, err := untarredDropletData.Next()
					if err == io.EOF {
						break
					}
					Ω(err).ShouldNot(HaveOccurred())

					content, err := ioutil.ReadAll(untarredDropletData)
					Ω(err).ShouldNot(HaveOccurred())

					dropletContents[hdr.Name] = content
				}

				//Assert the droplet has the right files in it
				Ω(dropletContents).Should(HaveKey("./"))
				Ω(dropletContents).Should(HaveKey("./staging_info.yml"))
				Ω(dropletContents).Should(HaveKey("./logs/"))
				Ω(dropletContents).Should(HaveKey("./tmp/"))
				Ω(dropletContents).Should(HaveKey("./app/"))
				Ω(dropletContents).Should(HaveKey("./app/my-app"))
				Ω(dropletContents).Should(HaveKey("./app/compiled"))

				//Assert the files contain the right content
				Ω(string(dropletContents["./app/my-app"])).Should(Equal("scooby-doo"))

				//In particular, staging_info.yml should have the correct detected_buildpack and start_command
				yamlDecoder := candiedyaml.NewDecoder(bytes.NewReader(dropletContents["./staging_info.yml"]))
				stagingInfo := map[string]string{}
				err = yamlDecoder.Decode(&stagingInfo)
				Ω(err).ShouldNot(HaveOccurred())

				Ω(stagingInfo["detected_buildpack"]).Should(Equal("My Buildpack"))
				Ω(stagingInfo["start_command"]).Should(Equal("start-command"))

				//Assert nothing else crept into the droplet
				Ω(dropletContents).Should(HaveLen(7))
			})

			Context("when compilation fails", func() {
				BeforeEach(func() {
					buildpackToUse = "busted_admin_buildpack.zip"
				})

				It("responds with the error, and no detected buildpack present", func() {
					payloads := make(chan []byte)

					natsRunner.MessageBus.Subscribe("stager-test", func(msg *yagnats.Message) {
						payloads <- msg.Payload
					})

					messages, stop := loggredile.StreamMessages(
						loggregatorRunner.Config.OutgoingPort,
						"/tail/?app=some-app-guid",
					)
					defer close(stop)

					logOutput := ""
					go func() {
						for message := range messages {
							Ω(message.GetSourceName()).To(Equal("STG"))
							logOutput += string(message.GetMessage()) + "\n"
						}
					}()

					err := natsRunner.MessageBus.PublishWithReplyTo(
						"diego.staging.start",
						"stager-test",
						stagingMessage,
					)
					Ω(err).ShouldNot(HaveOccurred())

					var payload []byte
					Eventually(payloads, 10.0).Should(Receive(&payload))
					Ω(string(payload)).Should(Equal(`{"error":"process exited with status 1"}`))

					Eventually(func() string {
						return logOutput
					}, 5.0).Should(ContainSubstring("no buildpack detected"))
				})
			})
		})

		Context("with two stagers running", func() {
			BeforeEach(func() {
				stagerRunner.Start("--compilers", `{"default":"smelter.zip"}`)
				otherStagerRunner.Start("--compilers", `{"default":"smelter.zip"}`)
			})

			AfterEach(func() {
				otherStagerRunner.Stop()
			})

			It("only one returns a staging completed response", func() {
				received := make(chan bool)
				natsRunner.MessageBus.Subscribe("two-stagers-test", func(message *yagnats.Message) {
					received <- true
				})

				natsRunner.MessageBus.PublishWithReplyTo(
					"diego.staging.start",
					"two-stagers-test",
					stagingMessage,
				)

				Eventually(received, 10.0).Should(Receive())
				Consistently(received, 2.0).ShouldNot(Receive())
			})
		})
	})
})
<commit_msg>specify now-required app bits download uri<commit_after>package inigo_test

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"fmt"
	"github.com/cloudfoundry-incubator/inigo/loggredile"
	"github.com/fraenkel/candiedyaml"
	"io"
	"io/ioutil"

	"github.com/cloudfoundry-incubator/inigo/inigo_server"
	"github.com/cloudfoundry-incubator/inigo/stager_runner"
	"github.com/cloudfoundry-incubator/inigo/zipper"
	"github.com/cloudfoundry-incubator/runtime-schema/models/factories"
	"github.com/cloudfoundry/yagnats"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Stager", func() {
	var otherStagerRunner *stager_runner.StagerRunner

	BeforeEach(func() {
		fileServerRunner.Start()
		otherStagerRunner = stager_runner.New(
			stagerPath,
			etcdRunner.NodeURLS(),
			[]string{fmt.Sprintf("127.0.0.1:%d", natsPort)},
		)
	})

	Context("when unable to find an appropriate compiler", func() {
		BeforeEach(func() {
			executorRunner.Start()
			stagerRunner.Start()
		})

		It("returns an error", func() {
			receivedMessages := make(chan *yagnats.Message)
			natsRunner.MessageBus.Subscribe("compiler-stagers-test", func(message *yagnats.Message) {
				receivedMessages <- message
			})

			natsRunner.MessageBus.PublishWithReplyTo(
				"diego.staging.start",
				"compiler-stagers-test",
				[]byte(`{
					"app_id": "some-app-guid",
					"task_id": "some-task-id",
					"app_bits_download_uri": "some-download-uri",
					"stack": "no-compiler"
				}`),
			)

			var receivedMessage *yagnats.Message
			Eventually(receivedMessages, 2.0).Should(Receive(&receivedMessage))
			Ω(receivedMessage.Payload).Should(ContainSubstring("no compiler defined for requested stack"))
			Consistently(receivedMessages, 2.0).ShouldNot(Receive())
		})
	})

	Describe("Staging", func() {
		var outputGuid string
		var stagingMessage []byte
		var buildpackToUse string

		BeforeEach(func() {
			executorRunner.Start()

			buildpackToUse = "admin_buildpack.zip"
			outputGuid = factories.GenerateGuid()

			fileServerRunner.ServeFile("smelter.zip", smelterZipPath)

			//make and upload an app
			var appFiles = []zipper.ZipFile{
				{"my-app", "scooby-doo"},
			}
			zipper.CreateZipFile("/tmp/app.zip", appFiles)
			inigoserver.UploadFile("app.zip", "/tmp/app.zip")

			//make and upload a buildpack
			var adminBuildpackFiles = []zipper.ZipFile{
				{"bin/detect", `#!/bin/bash
				echo My Buildpack
				`},
				{"bin/compile", `#!/bin/bash
				echo COMPILING BUILDPACK
				echo $SOME_STAGING_ENV
				touch $1/compiled
				`},
				{"bin/release", `#!/bin/bash
cat <<EOF
---
default_process_types:
  web: start-command
EOF
				`},
			}
			zipper.CreateZipFile("/tmp/admin_buildpack.zip", adminBuildpackFiles)
			inigoserver.UploadFile("admin_buildpack.zip", "/tmp/admin_buildpack.zip")

			var bustedAdminBuildpackFiles = []zipper.ZipFile{
				{"bin/detect", `#!/bin/bash]
				exit 1
				`},
				{"bin/compile", `#!/bin/bash`},
				{"bin/release", `#!/bin/bash`},
			}
			zipper.CreateZipFile("/tmp/busted_admin_buildpack.zip", bustedAdminBuildpackFiles)
			inigoserver.UploadFile("busted_admin_buildpack.zip", "/tmp/busted_admin_buildpack.zip")
		})

		JustBeforeEach(func() {
			stagingMessage = []byte(
				fmt.Sprintf(
					`{
						"app_id": "some-app-guid",
						"task_id": "some-task-id",
						"memory_mb": 128,
						"disk_mb": 128,
						"file_descriptors": 1024,
						"stack": "default",
						"app_bits_download_uri": "%s",
						"buildpacks" : [{ "key": "test-buildpack", "url": "%s" }],
						"environment": [["SOME_STAGING_ENV", "%s"]]
					}`,
					inigoserver.DownloadUrl("app.zip"),
					inigoserver.DownloadUrl(buildpackToUse),
					outputGuid,
				),
			)
		})

		Context("with one stager running", func() {
			BeforeEach(func() {
				stagerRunner.Start("--compilers", `{"default":"smelter.zip"}`)
			})

			It("runs the compiler on the executor with the correct environment variables, bits and log tag, and responds with the detected buildpack", func() {
				//listen for NATS response
				payloads := make(chan []byte)

				natsRunner.MessageBus.Subscribe("stager-test", func(msg *yagnats.Message) {
					payloads <- msg.Payload
				})

				//stream logs
				messages, stop := loggredile.StreamMessages(
					loggregatorRunner.Config.OutgoingPort,
					"/tail/?app=some-app-guid",
				)
				defer close(stop)

				logOutput := ""
				go func() {
					for message := range messages {
						Ω(message.GetSourceName()).To(Equal("STG"))
						logOutput += string(message.GetMessage()) + "\n"
					}
				}()

				//publish the staging message
				err := natsRunner.MessageBus.PublishWithReplyTo(
					"diego.staging.start",
					"stager-test",
					stagingMessage,
				)
				Ω(err).ShouldNot(HaveOccurred())

				//wait for staging to complete
				var payload []byte
				Eventually(payloads, 10.0).Should(Receive(&payload))

				//Assert on the staging output (detected buildpack)
				Ω(string(payload)).Should(Equal(`{"detected_buildpack":"My Buildpack"}`))

				//Asser the user saw reasonable output
				Eventually(func() string {
					return logOutput
				}).Should(ContainSubstring("COMPILING BUILDPACK"))
				Ω(logOutput).Should(ContainSubstring(outputGuid))

				//Fetch the compiled droplet from the fakeCC
				dropletData, ok := fakeCC.UploadedDroplets["some-app-guid"]
				Ω(ok).Should(BeTrue())
				Ω(dropletData).ShouldNot(BeEmpty())

				//Unzip the droplet
				ungzippedDropletData, err := gzip.NewReader(bytes.NewReader(dropletData))
				Ω(err).ShouldNot(HaveOccurred())

				//Untar the droplet
				untarredDropletData := tar.NewReader(ungzippedDropletData)
				dropletContents := map[string][]byte{}
				for {
					hdr, err := untarredDropletData.Next()
					if err == io.EOF {
						break
					}
					Ω(err).ShouldNot(HaveOccurred())

					content, err := ioutil.ReadAll(untarredDropletData)
					Ω(err).ShouldNot(HaveOccurred())

					dropletContents[hdr.Name] = content
				}

				//Assert the droplet has the right files in it
				Ω(dropletContents).Should(HaveKey("./"))
				Ω(dropletContents).Should(HaveKey("./staging_info.yml"))
				Ω(dropletContents).Should(HaveKey("./logs/"))
				Ω(dropletContents).Should(HaveKey("./tmp/"))
				Ω(dropletContents).Should(HaveKey("./app/"))
				Ω(dropletContents).Should(HaveKey("./app/my-app"))
				Ω(dropletContents).Should(HaveKey("./app/compiled"))

				//Assert the files contain the right content
				Ω(string(dropletContents["./app/my-app"])).Should(Equal("scooby-doo"))

				//In particular, staging_info.yml should have the correct detected_buildpack and start_command
				yamlDecoder := candiedyaml.NewDecoder(bytes.NewReader(dropletContents["./staging_info.yml"]))
				stagingInfo := map[string]string{}
				err = yamlDecoder.Decode(&stagingInfo)
				Ω(err).ShouldNot(HaveOccurred())

				Ω(stagingInfo["detected_buildpack"]).Should(Equal("My Buildpack"))
				Ω(stagingInfo["start_command"]).Should(Equal("start-command"))

				//Assert nothing else crept into the droplet
				Ω(dropletContents).Should(HaveLen(7))
			})

			Context("when compilation fails", func() {
				BeforeEach(func() {
					buildpackToUse = "busted_admin_buildpack.zip"
				})

				It("responds with the error, and no detected buildpack present", func() {
					payloads := make(chan []byte)

					natsRunner.MessageBus.Subscribe("stager-test", func(msg *yagnats.Message) {
						payloads <- msg.Payload
					})

					messages, stop := loggredile.StreamMessages(
						loggregatorRunner.Config.OutgoingPort,
						"/tail/?app=some-app-guid",
					)
					defer close(stop)

					logOutput := ""
					go func() {
						for message := range messages {
							Ω(message.GetSourceName()).To(Equal("STG"))
							logOutput += string(message.GetMessage()) + "\n"
						}
					}()

					err := natsRunner.MessageBus.PublishWithReplyTo(
						"diego.staging.start",
						"stager-test",
						stagingMessage,
					)
					Ω(err).ShouldNot(HaveOccurred())

					var payload []byte
					Eventually(payloads, 10.0).Should(Receive(&payload))
					Ω(string(payload)).Should(Equal(`{"error":"process exited with status 1"}`))

					Eventually(func() string {
						return logOutput
					}, 5.0).Should(ContainSubstring("no buildpack detected"))
				})
			})
		})

		Context("with two stagers running", func() {
			BeforeEach(func() {
				stagerRunner.Start("--compilers", `{"default":"smelter.zip"}`)
				otherStagerRunner.Start("--compilers", `{"default":"smelter.zip"}`)
			})

			AfterEach(func() {
				otherStagerRunner.Stop()
			})

			It("only one returns a staging completed response", func() {
				received := make(chan bool)
				natsRunner.MessageBus.Subscribe("two-stagers-test", func(message *yagnats.Message) {
					received <- true
				})

				natsRunner.MessageBus.PublishWithReplyTo(
					"diego.staging.start",
					"two-stagers-test",
					stagingMessage,
				)

				Eventually(received, 10.0).Should(Receive())
				Consistently(received, 2.0).ShouldNot(Receive())
			})
		})
	})
})
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"os"
	"os/signal"
	"sync"

	log "github.com/golang/glog"
	osquery "github.com/kolide/osquery-go"

	"github.com/google/fleetspeak/fleetspeak/src/client/daemonservice/client"
	"github.com/google/fleetspeak/fleetspeak/src/osquery/plugin"

	fspb "github.com/google/fleetspeak/fleetspeak/src/common/proto/fleetspeak"
)

const version = "0.1"

var (
	socketPath  = flag.String("socket", "", "path to osqueryd extensions socket")
	logService  = flag.String("log_service", "", "If set, a logger extention will be registered which logs to this Fleetspeak service.")
	managerName = flag.String("manager_name", "Fleetspeak", "Name to register the extension manager as, also used as a prefix for the plugin names.")
)

func main() {
	flag.Parse()

	ch, err := client.Init(version)
	if err != nil {
		log.Exitf("Unable to initialize FS connection: %v", err)
	}

	server, err := osquery.NewExtensionManagerServer(*managerName, *socketPath)
	if err != nil {
		log.Exitf("Unable to create osquery extension manager: %v", err)
	}

	stop := make(chan struct{})
	var working sync.WaitGroup
	defer func() {
		close(stop)
		working.Wait()
	}()
	in := make(chan *fspb.Message, 20)

	working.Add(1)
	go func() {
		defer working.Done()
		for {
			select {
			case m := <-ch.In:
				select {
				case <-stop:
					return
				case err := <-ch.Err:
					log.Exitf("Error from channel: %v", err)
				case in <- m:
				}
			case <-stop:
				return
			case err := <-ch.Err:
				log.Exitf("Error from channel: %v", err)
			}
		}
	}()

	server.RegisterPlugin(plugin.MakeDistributed(*managerName+"Queries", in, ch.Out))

	if *logService != "" {
		server.RegisterPlugin(plugin.MakeLogger(*managerName+"Logger", *logService, ch.Out))
	}

	done := make(chan struct{})
	go func() {
		if err := server.Start(); err != nil {
			log.Errorf("server.Start() returned error: %v", err)
		} else {
			log.Infof("server.Start() terminated normally")
		}
		close(done)
	}()

	s := make(chan os.Signal)
	signal.Notify(s, os.Interrupt)
	select {
	case <-s:
		server.Shutdown(context.Background())
		log.Infof("Interrupt received, waiting for server to finish.")
		<-done
	case <-done:
	}
	signal.Reset(os.Interrupt)
}
<commit_msg>Fix spelling.<commit_after>package main

import (
	"context"
	"flag"
	"os"
	"os/signal"
	"sync"

	log "github.com/golang/glog"
	osquery "github.com/kolide/osquery-go"

	"github.com/google/fleetspeak/fleetspeak/src/client/daemonservice/client"
	"github.com/google/fleetspeak/fleetspeak/src/osquery/plugin"

	fspb "github.com/google/fleetspeak/fleetspeak/src/common/proto/fleetspeak"
)

const version = "0.1"

var (
	socketPath  = flag.String("socket", "", "path to osqueryd extensions socket")
	logService  = flag.String("log_service", "", "If set, a logger extension will be registered which logs to this Fleetspeak service.")
	managerName = flag.String("manager_name", "Fleetspeak", "Name to register the extension manager as, also used as a prefix for the plugin names.")
)

func main() {
	flag.Parse()

	ch, err := client.Init(version)
	if err != nil {
		log.Exitf("Unable to initialize FS connection: %v", err)
	}

	server, err := osquery.NewExtensionManagerServer(*managerName, *socketPath)
	if err != nil {
		log.Exitf("Unable to create osquery extension manager: %v", err)
	}

	stop := make(chan struct{})
	var working sync.WaitGroup
	defer func() {
		close(stop)
		working.Wait()
	}()
	in := make(chan *fspb.Message, 20)

	working.Add(1)
	go func() {
		defer working.Done()
		for {
			select {
			case m := <-ch.In:
				select {
				case <-stop:
					return
				case err := <-ch.Err:
					log.Exitf("Error from channel: %v", err)
				case in <- m:
				}
			case <-stop:
				return
			case err := <-ch.Err:
				log.Exitf("Error from channel: %v", err)
			}
		}
	}()

	server.RegisterPlugin(plugin.MakeDistributed(*managerName+"Queries", in, ch.Out))

	if *logService != "" {
		server.RegisterPlugin(plugin.MakeLogger(*managerName+"Logger", *logService, ch.Out))
	}

	done := make(chan struct{})
	go func() {
		if err := server.Start(); err != nil {
			log.Errorf("server.Start() returned error: %v", err)
		} else {
			log.Infof("server.Start() terminated normally")
		}
		close(done)
	}()

	s := make(chan os.Signal)
	signal.Notify(s, os.Interrupt)
	select {
	case <-s:
		server.Shutdown(context.Background())
		log.Infof("Interrupt received, waiting for server to finish.")
		<-done
	case <-done:
	}
	signal.Reset(os.Interrupt)
}
<|endoftext|>