<commit_before>package chroot

import (
	"fmt"
	"github.com/mitchellh/goamz/ec2"
	"github.com/mitchellh/multistep"
	awscommon "github.com/mitchellh/packer/builder/amazon/common"
	"github.com/mitchellh/packer/packer"
	"log"
)

// StepCreateVolume creates a new volume from the snapshot of the root
// device of the AMI.
//
// Produces:
//   volume_id string - The ID of the created volume
type StepCreateVolume struct {
	volumeId string
}

func (s *StepCreateVolume) Run(state map[string]interface{}) multistep.StepAction {
	ec2conn := state["ec2"].(*ec2.EC2)
	image := state["source_image"].(*ec2.Image)
	instance := state["instance"].(*ec2.Instance)
	ui := state["ui"].(packer.Ui)

	// Determine the root device snapshot
	log.Printf("Searching for root device of the image (%s)", image.RootDeviceName)
	var rootDevice *ec2.BlockDeviceMapping
	for _, device := range image.BlockDevices {
		if device.DeviceName == image.RootDeviceName {
			rootDevice = &device
			break
		}
	}

	if rootDevice == nil {
		err := fmt.Errorf("Couldn't find root device!")
		state["error"] = err
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	ui.Say("Creating the root volume...")
	createVolume := &ec2.CreateVolume{
		AvailZone:  instance.AvailZone,
		Size:       rootDevice.VolumeSize,
		SnapshotId: rootDevice.SnapshotId,
		VolumeType: rootDevice.VolumeType,
		IOPS:       rootDevice.IOPS,
	}

	createVolumeResp, err := ec2conn.CreateVolume(createVolume)
	if err != nil {
		err := fmt.Errorf("Error creating root volume: %s", err)
		state["error"] = err
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	// Set the volume ID so we remember to delete it later
	s.volumeId = createVolumeResp.VolumeId
	log.Printf("Volume ID: %s", s.volumeId)

	// Wait for the volume to become ready
	stateChange := awscommon.StateChangeConf{
		Conn:      ec2conn,
		Pending:   []string{"creating"},
		StepState: state,
		Target:    "available",
		Refresh: func() (interface{}, string, error) {
			resp, err := ec2conn.Volumes([]string{s.volumeId}, ec2.NewFilter())
			if err != nil {
				return nil, "", err
			}

			return nil, resp.Volumes[0].Status, nil
		},
	}

	_, err = awscommon.WaitForState(&stateChange)
	if err != nil {
		err := fmt.Errorf("Error waiting for volume: %s", err)
		state["error"] = err
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	state["volume_id"] = s.volumeId
	return multistep.ActionContinue
}

func (s *StepCreateVolume) Cleanup(state map[string]interface{}) {
	if s.volumeId == "" {
		return
	}

	ec2conn := state["ec2"].(*ec2.EC2)
	ui := state["ui"].(packer.Ui)

	ui.Say("Deleting the created EBS volume...")
	_, err := ec2conn.DeleteVolume(s.volumeId)
	if err != nil {
		ui.Error(fmt.Sprintf("Error deleting EBS volume: %s", err))
	}
}
<commit_msg>builder/amazon/chroot: improved logging about what volume created<commit_after>package chroot

import (
	"fmt"
	"github.com/mitchellh/goamz/ec2"
	"github.com/mitchellh/multistep"
	awscommon "github.com/mitchellh/packer/builder/amazon/common"
	"github.com/mitchellh/packer/packer"
	"log"
)

// StepCreateVolume creates a new volume from the snapshot of the root
// device of the AMI.
//
// Produces:
//   volume_id string - The ID of the created volume
type StepCreateVolume struct {
	volumeId string
}

func (s *StepCreateVolume) Run(state map[string]interface{}) multistep.StepAction {
	ec2conn := state["ec2"].(*ec2.EC2)
	image := state["source_image"].(*ec2.Image)
	instance := state["instance"].(*ec2.Instance)
	ui := state["ui"].(packer.Ui)

	// Determine the root device snapshot
	log.Printf("Searching for root device of the image (%s)", image.RootDeviceName)
	var rootDevice *ec2.BlockDeviceMapping
	for _, device := range image.BlockDevices {
		if device.DeviceName == image.RootDeviceName {
			rootDevice = &device
			break
		}
	}

	if rootDevice == nil {
		err := fmt.Errorf("Couldn't find root device!")
		state["error"] = err
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	ui.Say("Creating the root volume...")
	createVolume := &ec2.CreateVolume{
		AvailZone:  instance.AvailZone,
		Size:       rootDevice.VolumeSize,
		SnapshotId: rootDevice.SnapshotId,
		VolumeType: rootDevice.VolumeType,
		IOPS:       rootDevice.IOPS,
	}
	log.Printf("Create args: %#v", createVolume)

	createVolumeResp, err := ec2conn.CreateVolume(createVolume)
	if err != nil {
		err := fmt.Errorf("Error creating root volume: %s", err)
		state["error"] = err
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	// Set the volume ID so we remember to delete it later
	s.volumeId = createVolumeResp.VolumeId
	log.Printf("Volume ID: %s", s.volumeId)

	// Wait for the volume to become ready
	stateChange := awscommon.StateChangeConf{
		Conn:      ec2conn,
		Pending:   []string{"creating"},
		StepState: state,
		Target:    "available",
		Refresh: func() (interface{}, string, error) {
			resp, err := ec2conn.Volumes([]string{s.volumeId}, ec2.NewFilter())
			if err != nil {
				return nil, "", err
			}

			return nil, resp.Volumes[0].Status, nil
		},
	}

	_, err = awscommon.WaitForState(&stateChange)
	if err != nil {
		err := fmt.Errorf("Error waiting for volume: %s", err)
		state["error"] = err
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	state["volume_id"] = s.volumeId
	return multistep.ActionContinue
}

func (s *StepCreateVolume) Cleanup(state map[string]interface{}) {
	if s.volumeId == "" {
		return
	}

	ec2conn := state["ec2"].(*ec2.EC2)
	ui := state["ui"].(packer.Ui)

	ui.Say("Deleting the created EBS volume...")
	_, err := ec2conn.DeleteVolume(s.volumeId)
	if err != nil {
		ui.Error(fmt.Sprintf("Error deleting EBS volume: %s", err))
	}
}
<|endoftext|>


<commit_before><commit_msg>audio: add Mixer doc<commit_after><|endoftext|>


<commit_before>package models

import (
	"fmt"
	"github.com/roadrunners/go-url-shortener/app/db"
	"github.com/roadrunners/go-url-shortener/app/models/key"
	"github.com/roadrunners/go-url-shortener/app/redis"
	"github.com/robfig/revel"
)

type ShortUrl struct {
	Id   int64  `db:"id" json:"-"`
	Slug string `db:"-" json:"slug"`
	URL  string `db:"url" json:"url"`
}

func (s ShortUrl) String() string {
	return fmt.Sprintf("(%s, %s)", s.Slug, s.URL)
}

func (s *ShortUrl) pushToRedis() {
	k := fmt.Sprintf("shorturl:%d:url", s.Id)
	err := redis.Client.Set(k, []byte(s.URL))
	if err != nil {
		revel.ERROR.Fatal("Could not push short url to redis")
	}
}

func ShortUrlById(id int64) (*ShortUrl, error) {
	v, err := db.DbMap.Get(ShortUrl{}, id)
	if err != nil || v == nil {
		return nil, err
	}
	s := v.(*ShortUrl)
	s.Slug = key.GenKey(s.Id)
	return s, nil
}

func ShortUrlBySlug(slug string) (*ShortUrl, error) {
	id := key.GenId(slug)
	return ShortUrlById(id)
}

func CachedShortUrlBySlug(slug string) (*ShortUrl, error) {
	id := key.GenId(slug)
	k := fmt.Sprintf("shorturl:%d:url", id)
	data, err := redis.Client.Get(k)
	if err == nil {
		s := ShortUrl{Id: id, Slug: slug, URL: string(data)}
		return &s, nil
	}
	revel.WARN.Printf("Missed cache for slug %v (id %v, key %v)", slug, id, k)
	s, err := ShortUrlById(id)
	if s != nil && err == nil {
		go s.pushToRedis()
	}
	return s, err
}

func ShortUrlCreate(url string) (*ShortUrl, error) {
	s := &ShortUrl{URL: url}
	if err := db.DbMap.Insert(s); err != nil {
		return nil, err
	}
	s.Slug = key.GenKey(s.Id)
	go s.pushToRedis()
	return s, nil
}
<commit_msg>shorturl.go: use the slug for lookup in redis<commit_after>package models

import (
	"fmt"
	"github.com/roadrunners/go-url-shortener/app/db"
	"github.com/roadrunners/go-url-shortener/app/models/key"
	"github.com/roadrunners/go-url-shortener/app/redis"
	"github.com/robfig/revel"
)

type ShortUrl struct {
	Id   int64  `db:"id" json:"-"`
	Slug string `db:"-" json:"slug"`
	URL  string `db:"url" json:"url"`
}

func (s ShortUrl) String() string {
	return fmt.Sprintf("(%s, %s)", s.Slug, s.URL)
}

func (s *ShortUrl) pushToRedis() {
	k := fmt.Sprintf("shorturl:%s:url", s.Slug)
	err := redis.Client.Set(k, []byte(s.URL))
	if err != nil {
		revel.ERROR.Fatal("Could not push short url to redis")
	}
}

func ShortUrlById(id int64) (*ShortUrl, error) {
	v, err := db.DbMap.Get(ShortUrl{}, id)
	if err != nil || v == nil {
		return nil, err
	}
	s := v.(*ShortUrl)
	s.Slug = key.GenKey(s.Id)
	return s, nil
}

func ShortUrlBySlug(slug string) (*ShortUrl, error) {
	id := key.GenId(slug)
	return ShortUrlById(id)
}

func CachedShortUrlBySlug(slug string) (*ShortUrl, error) {
	k := fmt.Sprintf("shorturl:%s:url", slug)
	data, err := redis.Client.Get(k)
	if err == nil {
		s := ShortUrl{Slug: slug, URL: string(data)}
		return &s, nil
	}
	id := key.GenId(slug)
	revel.WARN.Printf("Missed cache for slug %v (id %v, key %v)", slug, id, k)
	s, err := ShortUrlById(id)
	if s != nil && err == nil {
		go s.pushToRedis()
	}
	return s, err
}

func ShortUrlCreate(url string) (*ShortUrl, error) {
	s := &ShortUrl{URL: url}
	if err := db.DbMap.Insert(s); err != nil {
		return nil, err
	}
	s.Slug = key.GenKey(s.Id)
	go s.pushToRedis()
	return s, nil
}
<|endoftext|>


<commit_before><commit_msg>change hide mechanism to (no hide until set)<commit_after><|endoftext|>


<commit_before>package main

import (
	"go/build"
	"log"
	"strings"
)

// contexts are the default contexts which are scanned, unless
// overridden by the -contexts flag.
var contexts = []*build.Context{
	{GOOS: "linux", GOARCH: "386", CgoEnabled: true},
	{GOOS: "linux", GOARCH: "386"},
	{GOOS: "linux", GOARCH: "amd64", CgoEnabled: true},
	{GOOS: "linux", GOARCH: "amd64"},
	{GOOS: "linux", GOARCH: "arm", CgoEnabled: true},
	{GOOS: "linux", GOARCH: "arm"},
	{GOOS: "darwin", GOARCH: "386", CgoEnabled: true},
	{GOOS: "darwin", GOARCH: "386"},
	{GOOS: "darwin", GOARCH: "amd64", CgoEnabled: true},
	{GOOS: "darwin", GOARCH: "amd64"},
	{GOOS: "windows", GOARCH: "amd64", CgoEnabled: true},
	{GOOS: "windows", GOARCH: "amd64"},
	{GOOS: "windows", GOARCH: "386", CgoEnabled: true},
	{GOOS: "windows", GOARCH: "386"},
	{GOOS: "freebsd", GOARCH: "386", CgoEnabled: true},
	{GOOS: "freebsd", GOARCH: "386"},
	{GOOS: "freebsd", GOARCH: "amd64", CgoEnabled: true},
	{GOOS: "freebsd", GOARCH: "amd64"},
	{GOOS: "freebsd", GOARCH: "arm", CgoEnabled: true},
	{GOOS: "freebsd", GOARCH: "arm"},
	{GOOS: "netbsd", GOARCH: "386", CgoEnabled: true},
	{GOOS: "netbsd", GOARCH: "386"},
	{GOOS: "netbsd", GOARCH: "amd64", CgoEnabled: true},
	{GOOS: "netbsd", GOARCH: "amd64"},
	{GOOS: "netbsd", GOARCH: "arm", CgoEnabled: true},
	{GOOS: "netbsd", GOARCH: "arm"},
	{GOOS: "openbsd", GOARCH: "386", CgoEnabled: true},
	{GOOS: "openbsd", GOARCH: "386"},
	{GOOS: "openbsd", GOARCH: "amd64", CgoEnabled: true},
	{GOOS: "openbsd", GOARCH: "amd64"},
}

func contextName(c *build.Context) string {
	s := c.GOOS + "-" + c.GOARCH
	if c.CgoEnabled {
		return s + "-cgo"
	}
	return s
}

func osArchName(c *build.Context) string {
	return c.GOOS + "-" + c.GOARCH
}

func parseContext(c string) *build.Context {
	parts := strings.Split(c, "-")
	if len(parts) < 2 {
		log.Fatalf("bad context: %q", c)
	}
	bc := &build.Context{
		GOOS:   parts[0],
		GOARCH: parts[1],
	}
	if len(parts) == 3 {
		if parts[2] == "cgo" {
			bc.CgoEnabled = true
		} else {
			log.Fatalf("bad context: %q", c)
		}
	}
	return bc
}

func setCustomContexts(customctx string) {
	contexts = []*build.Context{}
	for _, c := range strings.Split(customctx, ",") {
		contexts = append(contexts, parseContext(c))
	}
}
<commit_msg>qexport update context<commit_after>package main

import (
	"go/build"
	"log"
	"strings"
)

// contexts are the default contexts which are scanned, unless
// overridden by the -contexts flag.
var contexts = []*build.Context{
	{GOOS: "linux", GOARCH: "386", CgoEnabled: true},
	{GOOS: "linux", GOARCH: "amd64", CgoEnabled: true},
	{GOOS: "linux", GOARCH: "arm", CgoEnabled: true},
	{GOOS: "darwin", GOARCH: "386", CgoEnabled: true},
	{GOOS: "darwin", GOARCH: "amd64", CgoEnabled: true},
	{GOOS: "windows", GOARCH: "amd64", CgoEnabled: true},
	{GOOS: "windows", GOARCH: "386", CgoEnabled: true},
	{GOOS: "freebsd", GOARCH: "386", CgoEnabled: true},
	{GOOS: "freebsd", GOARCH: "amd64", CgoEnabled: true},
	{GOOS: "freebsd", GOARCH: "arm", CgoEnabled: true},
	{GOOS: "netbsd", GOARCH: "386", CgoEnabled: true},
	{GOOS: "netbsd", GOARCH: "amd64", CgoEnabled: true},
	{GOOS: "netbsd", GOARCH: "arm", CgoEnabled: true},
	{GOOS: "openbsd", GOARCH: "386", CgoEnabled: true},
	{GOOS: "openbsd", GOARCH: "amd64", CgoEnabled: true},
}

func contextName(c *build.Context) string {
	s := c.GOOS + "-" + c.GOARCH
	if c.CgoEnabled {
		return s + "-cgo"
	}
	return s
}

func osArchName(c *build.Context) string {
	return c.GOOS + "-" + c.GOARCH
}

func parseContext(c string) *build.Context {
	parts := strings.Split(c, "-")
	if len(parts) < 2 {
		log.Fatalf("bad context: %q", c)
	}
	bc := &build.Context{
		GOOS:   parts[0],
		GOARCH: parts[1],
	}
	if len(parts) == 3 {
		if parts[2] == "cgo" {
			bc.CgoEnabled = true
		} else {
			log.Fatalf("bad context: %q", c)
		}
	}
	return bc
}

func setCustomContexts(customctx string) {
	contexts = []*build.Context{}
	for _, c := range strings.Split(customctx, ",") {
		contexts = append(contexts, parseContext(c))
	}
}
<|endoftext|>


<commit_before>package controller

import (
	"errors"
	"fmt"
	mongomodels "koding/db/models"
	"koding/db/mongodb/modelhelper"
	"socialapi/models"
	"strings"

	"github.com/VerbalExpressions/GoVerbalExpressions"
	"github.com/koding/logging"
	"labix.org/v2/mgo/bson"
)

var (
	ErrMigrated     = errors.New("already migrated")
	kodingChannelId int64
	tagRegex        = verbalexpressions.New().
			BeginCapture().
			Find("|#:JTag:").
			Anything().
			Then(":").
			Anything().
			Then("|").
			EndCapture().
			Regex()
)

type Controller struct {
	log logging.Logger
}

func New(log logging.Logger) (*Controller, error) {
	wc := &Controller{
		log: log,
	}

	return wc, nil
}

func (mwc *Controller) Start() error {
	if err := mwc.migrateAllTags(); err != nil {
		return err
	}

	if err := mwc.migrateAllPosts(); err != nil {
		return err
	}

	return nil
}

func (mwc *Controller) migrateAllPosts() error {
	o := modelhelper.Options{
		Sort: "meta.createdAt",
	}
	s := modelhelper.Selector{
		"socialMessageId": modelhelper.Selector{"$exists": false},
	}
	kodingChannel, err := mwc.createGroupChannel("koding")
	if err != nil {
		return fmt.Errorf("Koding channel cannot be created: %s", err)
	}
	kodingChannelId = kodingChannel.Id

	errCount := 0
	successCount := 0

	handleError := func(su *mongomodels.StatusUpdate, err error) {
		mwc.log.Error("an error occured for %s: %s", su.Id.Hex(), err)
		errCount++
	}

	iter := modelhelper.GetStatusUpdateIter(s, o)
	defer iter.Close()

	var su mongomodels.StatusUpdate
	for iter.Next(&su) {
		channelId, err := mwc.fetchGroupChannelId(su.Group)
		if err != nil {
			return fmt.Errorf("Post migration is interrupted with %d errors: channel id cannot be fetched :%s", errCount, err)
		}

		// create channel message
		cm := mapStatusUpdateToChannelMessage(&su)
		cm.InitialChannelId = channelId
		if err := insertChannelMessage(cm, su.OriginId.Hex()); err != nil {
			handleError(&su, err)
			continue
		}

		if err := addChannelMessageToMessageList(cm); err != nil {
			handleError(&su, err)
			continue
		}

		// create reply messages
		if err := mwc.migrateComments(cm, &su, channelId); err != nil {
			handleError(&su, err)
			continue
		}

		if err := mwc.migrateLikes(cm, su.Id); err != nil {
			handleError(&su, err)
			continue
		}

		if err := mwc.migrateTags(cm, su.Id); err != nil {
			handleError(&su, err)
			continue
		}

		// update mongo status update channelMessageId field
		if err := completePostMigration(&su, cm); err != nil {
			handleError(&su, err)
			continue
		}
		successCount++
	}

	if err := iter.Err(); err != nil {
		return fmt.Errorf("Post migration is interrupted with %d errors: %s", errCount, err)
	}

	mwc.log.Notice("Post migration completed for %d status updates with %d errors", successCount, errCount)

	return nil
}

func (mwc *Controller) createGroupChannel(groupName string) (*models.Channel, error) {
	c := models.NewChannel()
	c.Name = groupName
	c.GroupName = groupName
	c.TypeConstant = models.Channel_TYPE_GROUP

	group, err := modelhelper.GetGroup(groupName)
	if err != nil {
		return nil, err
	}

	// TODO check it causes any error
	c.PrivacyConstant = group.Privacy

	// find group owner
	creatorId, err := fetchGroupOwnerId(group)
	if err != nil {
		return nil, err
	}

	c.CreatorId = creatorId
	// create channel
	if err := c.Create(); err != nil {
		return nil, err
	}

	return c, nil
}

func fetchGroupOwnerId(g *mongomodels.Group) (int64, error) {
	// fetch owner relationship
	s := modelhelper.Selector{
		"targetName": "JAccount",
		"as":         "owner",
		"sourceId":   g.Id,
	}
	r, err := modelhelper.GetRelationship(s)
	if err != nil {
		return 0, err
	}

	a := models.NewAccount()
	a.OldId = r.TargetId.Hex()
	if err := a.FetchOrCreate(); err != nil {
		return 0, err
	}

	return a.Id, nil
}

func insertChannelMessage(cm *models.ChannelMessage, accountOldId string) error {

	if err := prepareMessageAccount(cm, accountOldId); err != nil {
		return err
	}

	if err := cm.CreateRaw(); err != nil {
		return err
	}

	return nil
}

func addChannelMessageToMessageList(cm *models.ChannelMessage) error {
	cml := models.NewChannelMessageList()
	cml.ChannelId = cm.InitialChannelId
	cml.MessageId = cm.Id
	cml.AddedAt = cm.CreatedAt

	return cml.CreateRaw()
}

func (mwc *Controller) migrateComments(parentMessage *models.ChannelMessage, su *mongomodels.StatusUpdate, channelId int64) error {

	s := modelhelper.Selector{
		"sourceId":   su.Id,
		"targetName": "JComment",
	}
	rels, err := modelhelper.GetAllRelationships(s)
	if err != nil {
		if err == modelhelper.ErrNotFound {
			return nil
		}
		return fmt.Errorf("comment relationships cannot be fetched: %s", err)
	}

	for _, r := range rels {
		comment, err := modelhelper.GetCommentById(r.TargetId.Hex())
		if err != nil {
			return fmt.Errorf("comment cannot be fetched %s", err)
		}
		// comment is already migrated
		if comment.SocialMessageId != 0 {
			continue
		}

		reply := mapCommentToChannelMessage(comment)
		reply.InitialChannelId = channelId
		// insert as channel message
		if err := insertChannelMessage(reply, comment.OriginId.Hex()); err != nil {
			return fmt.Errorf("comment cannot be inserted %s", err)
		}

		// insert as message reply
		mr := models.NewMessageReply()
		mr.MessageId = parentMessage.Id
		mr.ReplyId = reply.Id
		mr.CreatedAt = reply.CreatedAt
		if err := mr.CreateRaw(); err != nil {
			return fmt.Errorf("comment cannot be inserted to message reply %s", err)
		}

		if err := mwc.migrateLikes(reply, comment.Id); err != nil {
			return fmt.Errorf("likes cannot be migrated %s", err)
		}

		if err := completeCommentMigration(comment, reply); err != nil {
			return fmt.Errorf("old comment cannot be flagged with new message id %s", err)
		}
	}

	return nil
}

func (mwc *Controller) migrateLikes(cm *models.ChannelMessage, oldId bson.ObjectId) error {
	s := modelhelper.Selector{
		"sourceId": oldId,
		"as":       "like",
	}
	rels, err := modelhelper.GetAllRelationships(s)
	if err != nil {
		return fmt.Errorf("likes cannot be fetched %s", err)
	}
	for _, r := range rels {
		a := models.NewAccount()
		a.OldId = r.TargetId.Hex()
		if err := a.FetchOrCreate(); err != nil {
			mwc.log.Error("interactor account could not found: %s", err)
			continue
		}
		i := models.NewInteraction()
		i.MessageId = cm.Id
		i.AccountId = a.Id
		i.TypeConstant = models.Interaction_TYPE_LIKE
		i.CreatedAt = r.TimeStamp
		if err := i.CreateRaw(); err != nil {
			mwc.log.Error("interaction could not created: %s", err)
		}
	}

	return nil
}

func prepareMessageAccount(cm *models.ChannelMessage, accountOldId string) error {
	a := models.NewAccount()
	a.OldId = accountOldId
	if err := a.FetchOrCreate(); err != nil {
		return fmt.Errorf("account could not found: %s", err)
	}

	cm.AccountId = a.Id

	return nil
}

func (mwc *Controller) fetchGroupChannelId(groupName string) (int64, error) {
	// koding group channel id is prefetched
	if groupName == "koding" {
		return kodingChannelId, nil
	}

	c, err := mwc.createGroupChannel(groupName)
	if err != nil {
		return 0, err
	}

	return c.Id, nil
}

func mapStatusUpdateToChannelMessage(su *mongomodels.StatusUpdate) *models.ChannelMessage {
	cm := models.NewChannelMessage()
	cm.Slug = su.Slug
	prepareBody(cm, su.Body)
	cm.TypeConstant = models.ChannelMessage_TYPE_POST
	cm.CreatedAt = su.Meta.CreatedAt
	prepareMessageMetaDates(cm, &su.Meta)

	return cm
}

func mapCommentToChannelMessage(c *mongomodels.Comment) *models.ChannelMessage {
	cm := models.NewChannelMessage()
	cm.Body = c.Body
	cm.TypeConstant = models.ChannelMessage_TYPE_REPLY
	cm.CreatedAt = c.Meta.CreatedAt
	prepareMessageMetaDates(cm, &c.Meta)

	return cm
}

func prepareMessageMetaDates(cm *models.ChannelMessage, meta *mongomodels.Meta) {
	// this is added because status update->modified at field is before createdAt
	if cm.CreatedAt.After(meta.ModifiedAt) {
		cm.UpdatedAt = cm.CreatedAt
	} else {
		cm.UpdatedAt = meta.ModifiedAt
	}
}

func prepareBody(cm *models.ChannelMessage, body string) {
	res := tagRegex.FindAllStringSubmatch(body, -1)
	cm.Body = body
	if len(res) == 0 {
		return
	}

	for _, element := range res {
		tag := element[1][1 : len(element[1])-1]
		tag = strings.Split(tag, ":")[3]
		tag = "#" + tag
		cm.Body = verbalexpressions.New().Find(element[1]).Replace(cm.Body, tag)
	}

}

func completePostMigration(su *mongomodels.StatusUpdate, cm *models.ChannelMessage) error {
	su.SocialMessageId = cm.Id

	return modelhelper.UpdateStatusUpdate(su)
}

func completeCommentMigration(reply *mongomodels.Comment, cm *models.ChannelMessage) error {
	reply.SocialMessageId = cm.Id

	return modelhelper.UpdateComment(reply)
}
<commit_msg>Migration: Group SocialApiChannelId field is set when group channel is created<commit_after>package controller

import (
	"errors"
	"fmt"
	mongomodels "koding/db/models"
	"koding/db/mongodb/modelhelper"
	"socialapi/models"
	"strconv"
	"strings"

	"github.com/VerbalExpressions/GoVerbalExpressions"
	"github.com/koding/logging"
	"labix.org/v2/mgo/bson"
)

var (
	ErrMigrated     = errors.New("already migrated")
	kodingChannelId int64
	tagRegex        = verbalexpressions.New().
			BeginCapture().
			Find("|#:JTag:").
			Anything().
			Then(":").
			Anything().
			Then("|").
			EndCapture().
			Regex()
)

type Controller struct {
	log logging.Logger
}

func New(log logging.Logger) (*Controller, error) {
	wc := &Controller{
		log: log,
	}

	return wc, nil
}

func (mwc *Controller) Start() error {
	if err := mwc.migrateAllTags(); err != nil {
		return err
	}

	if err := mwc.migrateAllPosts(); err != nil {
		return err
	}

	return nil
}

func (mwc *Controller) migrateAllPosts() error {
	o := modelhelper.Options{
		Sort: "meta.createdAt",
	}
	s := modelhelper.Selector{
		"socialMessageId": modelhelper.Selector{"$exists": false},
	}
	kodingChannel, err := mwc.createGroupChannel("koding")
	if err != nil {
		return fmt.Errorf("Koding channel cannot be created: %s", err)
	}
	kodingChannelId = kodingChannel.Id

	errCount := 0
	successCount := 0

	handleError := func(su *mongomodels.StatusUpdate, err error) {
		mwc.log.Error("an error occured for %s: %s", su.Id.Hex(), err)
		errCount++
	}

	iter := modelhelper.GetStatusUpdateIter(s, o)
	defer iter.Close()

	var su mongomodels.StatusUpdate
	for iter.Next(&su) {
		channelId, err := mwc.fetchGroupChannelId(su.Group)
		if err != nil {
			return fmt.Errorf("Post migration is interrupted with %d errors: channel id cannot be fetched :%s", errCount, err)
		}

		// create channel message
		cm := mapStatusUpdateToChannelMessage(&su)
		cm.InitialChannelId = channelId
		if err := insertChannelMessage(cm, su.OriginId.Hex()); err != nil {
			handleError(&su, err)
			continue
		}

		if err := addChannelMessageToMessageList(cm); err != nil {
			handleError(&su, err)
			continue
		}

		// create reply messages
		if err := mwc.migrateComments(cm, &su, channelId); err != nil {
			handleError(&su, err)
			continue
		}

		if err := mwc.migrateLikes(cm, su.Id); err != nil {
			handleError(&su, err)
			continue
		}

		if err := mwc.migrateTags(cm, su.Id); err != nil {
			handleError(&su, err)
			continue
		}

		// update mongo status update channelMessageId field
		if err := completePostMigration(&su, cm); err != nil {
			handleError(&su, err)
			continue
		}
		successCount++
	}

	if err := iter.Err(); err != nil {
		return fmt.Errorf("Post migration is interrupted with %d errors: %s", errCount, err)
	}

	mwc.log.Notice("Post migration completed for %d status updates with %d errors", successCount, errCount)

	return nil
}

func (mwc *Controller) createGroupChannel(groupName string) (*models.Channel, error) {
	c := models.NewChannel()
	c.Name = groupName
	c.GroupName = groupName
	c.TypeConstant = models.Channel_TYPE_GROUP

	group, err := modelhelper.GetGroup(groupName)
	if err != nil {
		return nil, err
	}

	// TODO check it causes any error
	c.PrivacyConstant = group.Privacy

	// find group owner
	creatorId, err := fetchGroupOwnerId(group)
	if err != nil {
		return nil, err
	}

	c.CreatorId = creatorId
	// create channel
	if err := c.Create(); err != nil {
		return nil, err
	}

	group.SocialApiChannelId = strconv.FormatInt(c.Id, 10)
	if err := modelhelper.UpdateGroup(group); err != nil {
		mwc.log.Error("Group document cannot be updated: %s", err)
	}

	return c, nil
}

func fetchGroupOwnerId(g *mongomodels.Group) (int64, error) {
	// fetch owner relationship
	s := modelhelper.Selector{
		"targetName": "JAccount",
		"as":         "owner",
		"sourceId":   g.Id,
	}
	r, err := modelhelper.GetRelationship(s)
	if err != nil {
		return 0, err
	}

	a := models.NewAccount()
	a.OldId = r.TargetId.Hex()
	if err := a.FetchOrCreate(); err != nil {
		return 0, err
	}

	return a.Id, nil
}

func insertChannelMessage(cm *models.ChannelMessage, accountOldId string) error {

	if err := prepareMessageAccount(cm, accountOldId); err != nil {
		return err
	}

	if err := cm.CreateRaw(); err != nil {
		return err
	}

	return nil
}

func addChannelMessageToMessageList(cm *models.ChannelMessage) error {
	cml := models.NewChannelMessageList()
	cml.ChannelId = cm.InitialChannelId
	cml.MessageId = cm.Id
	cml.AddedAt = cm.CreatedAt

	return cml.CreateRaw()
}

func (mwc *Controller) migrateComments(parentMessage *models.ChannelMessage, su *mongomodels.StatusUpdate, channelId int64) error {

	s := modelhelper.Selector{
		"sourceId":   su.Id,
		"targetName": "JComment",
	}
	rels, err := modelhelper.GetAllRelationships(s)
	if err != nil {
		if err == modelhelper.ErrNotFound {
			return nil
		}
		return fmt.Errorf("comment relationships cannot be fetched: %s", err)
	}

	for _, r := range rels {
		comment, err := modelhelper.GetCommentById(r.TargetId.Hex())
		if err != nil {
			return fmt.Errorf("comment cannot be fetched %s", err)
		}
		// comment is already migrated
		if comment.SocialMessageId != 0 {
			continue
		}

		reply := mapCommentToChannelMessage(comment)
		reply.InitialChannelId = channelId
		// insert as channel message
		if err := insertChannelMessage(reply, comment.OriginId.Hex()); err != nil {
			return fmt.Errorf("comment cannot be inserted %s", err)
		}

		// insert as message reply
		mr := models.NewMessageReply()
		mr.MessageId = parentMessage.Id
		mr.ReplyId = reply.Id
		mr.CreatedAt = reply.CreatedAt
		if err := mr.CreateRaw(); err != nil {
			return fmt.Errorf("comment cannot be inserted to message reply %s", err)
		}

		if err := mwc.migrateLikes(reply, comment.Id); err != nil {
			return fmt.Errorf("likes cannot be migrated %s", err)
		}

		if err := completeCommentMigration(comment, reply); err != nil {
			return fmt.Errorf("old comment cannot be flagged with new message id %s", err)
		}
	}

	return nil
}

func (mwc *Controller) migrateLikes(cm *models.ChannelMessage, oldId bson.ObjectId) error {
	s := modelhelper.Selector{
		"sourceId": oldId,
		"as":       "like",
	}
	rels, err := modelhelper.GetAllRelationships(s)
	if err != nil {
		return fmt.Errorf("likes cannot be fetched %s", err)
	}
	for _, r := range rels {
		a := models.NewAccount()
		a.OldId = r.TargetId.Hex()
		if err := a.FetchOrCreate(); err != nil {
			mwc.log.Error("interactor account could not found: %s", err)
			continue
		}
		i := models.NewInteraction()
		i.MessageId = cm.Id
		i.AccountId = a.Id
		i.TypeConstant = models.Interaction_TYPE_LIKE
		i.CreatedAt = r.TimeStamp
		if err := i.CreateRaw(); err != nil {
			mwc.log.Error("interaction could not created: %s", err)
		}
	}

	return nil
}

func prepareMessageAccount(cm *models.ChannelMessage, accountOldId string) error {
	a := models.NewAccount()
	a.OldId = accountOldId
	if err := a.FetchOrCreate(); err != nil {
		return fmt.Errorf("account could not found: %s", err)
	}

	cm.AccountId = a.Id

	return nil
}

func (mwc *Controller) fetchGroupChannelId(groupName string) (int64, error) {
	// koding group channel id is prefetched
	if groupName == "koding" {
		return kodingChannelId, nil
	}

	c, err := mwc.createGroupChannel(groupName)
	if err != nil {
		return 0, err
	}

	return c.Id, nil
}

func mapStatusUpdateToChannelMessage(su *mongomodels.StatusUpdate) *models.ChannelMessage {
	cm := models.NewChannelMessage()
	cm.Slug = su.Slug
	prepareBody(cm, su.Body)
	cm.TypeConstant = models.ChannelMessage_TYPE_POST
	cm.CreatedAt = su.Meta.CreatedAt
	prepareMessageMetaDates(cm, &su.Meta)

	return cm
}

func mapCommentToChannelMessage(c *mongomodels.Comment) *models.ChannelMessage {
	cm := models.NewChannelMessage()
	cm.Body = c.Body
	cm.TypeConstant = models.ChannelMessage_TYPE_REPLY
	cm.CreatedAt = c.Meta.CreatedAt
	prepareMessageMetaDates(cm, &c.Meta)

	return cm
}

func prepareMessageMetaDates(cm *models.ChannelMessage, meta *mongomodels.Meta) {
	// this is added because status update->modified at field is before createdAt
	if cm.CreatedAt.After(meta.ModifiedAt) {
		cm.UpdatedAt = cm.CreatedAt
	} else {
		cm.UpdatedAt = meta.ModifiedAt
	}
}

func prepareBody(cm *models.ChannelMessage, body string) {
	res := tagRegex.FindAllStringSubmatch(body, -1)
	cm.Body = body
	if len(res) == 0 {
		return
	}

	for _, element := range res {
		tag := element[1][1 : len(element[1])-1]
		tag = strings.Split(tag, ":")[3]
		tag = "#" + tag
		cm.Body = verbalexpressions.New().Find(element[1]).Replace(cm.Body, tag)
	}

}

func completePostMigration(su *mongomodels.StatusUpdate, cm *models.ChannelMessage) error {
	su.SocialMessageId = cm.Id

	return modelhelper.UpdateStatusUpdate(su)
}

func completeCommentMigration(reply *mongomodels.Comment, cm *models.ChannelMessage) error {
	reply.SocialMessageId = cm.Id

	return modelhelper.UpdateComment(reply)
}
<|endoftext|>


<commit_before>package notification

import (
	"fmt"
	mongomodels "koding/db/models"
	"koding/db/mongodb/modelhelper"
	"socialapi/config"
	"socialapi/models"
	socialapimodels "socialapi/models"
	"socialapi/workers/common/tests"
	"testing"
	"time"

	"github.com/koding/runner"
	. "github.com/smartystreets/goconvey/convey"
	"labix.org/v2/mgo/bson"
)

func TestCleanup(t *testing.T) {
	testData := []struct {
		definition string
		usernames  []string
		expected   []string
	}{
		{
			"should remove aliases",
			[]string{"team", "all"},
			[]string{"all"},
		},
		{
			"should return same usernames",
			[]string{"foo", "bar", "zaar"},
			[]string{"foo", "bar", "zaar"},
		},
		{
			"should remove duplicates",
			[]string{"admins", "admins", "ff"},
			[]string{"admins", "ff"},
		},
		{
			"should remove specific ones if have a general one",
			[]string{"admins", "admins", "team"},
			[]string{"all"},
		},
		{
			"should reduce to global alias",
			[]string{"team", "all", "group"},
			[]string{"all"},
		},
		{
			// some of the admins may not be in the channel
			"should keep channel and admins",
			[]string{"channel", "bar", "admins"},
			[]string{"channel", "bar", "admins"},
		},
	}

	for _, test := range testData {
		responses := cleanup(test.usernames)
		exists := false
		for _, response := range responses {
			for _, exc := range test.expected {
				if exc == response {
					exists = true
					break
				}
			}
		}

		if !exists {
			t.Fatalf("%s. expected: %+v, got: %+v", test.definition, responses)
		}

		if len(test.expected) != len(responses) {
			t.Fatalf("%s, %s. expected: %+v, got: %+v", test.definition, "expected lengths are not same", test.expected, responses)
		}
	}
}

func TestNormalize(t *testing.T) {
	tests.WithRunner(t, func(r *runner.Runner) {

		appConfig := config.MustRead(r.Conf.Path)
		modelhelper.Initialize(appConfig.Mongo)
		defer modelhelper.Close()

		Convey("while normalizing the usernames to their original nicks", t, func() {
			adminAccount, groupChannel, _ := models.CreateRandomGroupDataWithChecks()
			account1 := models.CreateAccountInBothDbsWithCheck()
			account2 := models.CreateAccountInBothDbsWithCheck()
			account3 := models.CreateAccountInBothDbsWithCheck()

			_, err := groupChannel.AddParticipant(account1.Id)
			So(err, ShouldBeNil)
			_, err = groupChannel.AddParticipant(account2.Id)
			So(err, ShouldBeNil)
			_, err = groupChannel.AddParticipant(account3.Id)
			So(err, ShouldBeNil)

			topicChan := socialapimodels.CreateTypedGroupedChannelWithTest(account1.Id, socialapimodels.Channel_TYPE_TOPIC, groupChannel.GroupName)

			Convey("@all should return all the members of the team", func() {
				body := "hi @all i am really excited to join this team!"
				cm := models.CreateMessageWithBody(groupChannel.Id, adminAccount.Id, models.ChannelMessage_TYPE_POST, body)

				usernames, err := NewNormalizer(cm, []string{"all"}, r.Log).Do()
				So(err, ShouldBeNil)
				So(len(usernames), ShouldEqual, 3)

				Convey("poster should not be in the mention list", func() {
					So(socialapimodels.IsIn(adminAccount.Nick, usernames...), ShouldBeFalse)
				})
			})

			Convey("@team should return all the members of the team", func() {
				_, err := topicChan.AddParticipant(adminAccount.Id)
				So(err, ShouldBeNil)
				_, err = topicChan.AddParticipant(account2.Id)
				So(err, ShouldBeNil)
				_, err = topicChan.AddParticipant(account3.Id)
				So(err, ShouldBeNil)

				body := "hi @team i am really excited to join this chan!"
				cm := models.CreateMessageWithBody(topicChan.Id, adminAccount.Id, models.ChannelMessage_TYPE_POST, body)

				usernames, err := NewNormalizer(cm, []string{"team"}, r.Log).Do()
				So(err, ShouldBeNil)
				So(len(usernames), ShouldEqual, 3)
			})

			Convey("@all + any username should return all the members of the team", func() {
				body := "hi @all i am really excited to join this team! how are you @" + account3.Nick
				cm := models.CreateMessageWithBody(groupChannel.Id, adminAccount.Id, models.ChannelMessage_TYPE_POST, body)

				usernames, err := NewNormalizer(cm, []string{"all", account3.Nick}, r.Log).Do()
				So(err, ShouldBeNil)
				So(len(usernames), ShouldEqual, 3)
			})

			Convey("@channel should return all the members of the channel", func() {

				body := "hi @channel"
				cm := socialapimodels.CreateMessageWithBody(topicChan.Id, account1.Id, models.ChannelMessage_TYPE_POST, body)

				Convey("if channel doesnt have any members", func() {
					Convey("should return 0 username", func() {
						usernames, err := NewNormalizer(cm, []string{"channel"}, r.Log).Do()
						So(err, ShouldBeNil)
						So(len(usernames), ShouldEqual, 0)
					})
				})

				Convey("if channel have member", func() {
					Convey("should return them", func() {
						_, err := topicChan.AddParticipant(account2.Id)
						So(err, ShouldBeNil)

						usernames, err := NewNormalizer(cm, []string{"channel"}, r.Log).Do()
						So(err, ShouldBeNil)
						So(len(usernames), ShouldEqual, 1)
						So(usernames[0], ShouldEqual, account2.Nick)
					})
				})
			})

			Convey("@channel + @group should return all the members of the team", func() {
				body := "hi @channel i am glad that i joined @group"
				cm := socialapimodels.CreateMessageWithBody(topicChan.Id, account1.Id, models.ChannelMessage_TYPE_POST, body)

				usernames, err := NewNormalizer(cm, []string{"channel", "group"}, r.Log).Do()
				So(err, ShouldBeNil)
				So(len(usernames), ShouldEqual, 3)
			})

			Convey("@admins should return all the admins of the team", func() {

				// create the group
				group, err := createGroup(groupChannel.GroupName)
				So(err, ShouldBeNil)

				err = makeAdmin(bson.ObjectIdHex(account1.OldId), group.Id)
				So(err, ShouldBeNil)

				body := "hi @admins make me mod plzz"
				cm := socialapimodels.CreateMessageWithBody(topicChan.Id, account2.Id, models.ChannelMessage_TYPE_POST, body)

				usernames, err := NewNormalizer(cm, []string{"admins"}, r.Log).Do()
				So(err, ShouldBeNil)
				So(len(usernames), ShouldEqual, 1)
				So(usernames[0], ShouldEqual, account1.Nick)

				Convey("adding another user to mention list should work", func() {
					_, err := topicChan.AddParticipant(account2.Id)
					So(err, ShouldBeNil)

					body := fmt.Sprintf("hi @%s do you know who are in @admins ? i believe @%s is in", account3.Nick, account2.Nick)
					cm := socialapimodels.CreateMessageWithBody(topicChan.Id, account2.Id, models.ChannelMessage_TYPE_POST, body)

					usernames, err := NewNormalizer(cm, []string{"admins", account3.Nick}, r.Log).Do()
					So(err, ShouldBeNil)
					So(len(usernames), ShouldEqual, 2)
				})
			})

			Convey("non members of channel should not be in mention list", func() {
				_, err := topicChan.AddParticipant(adminAccount.Id)
				So(err, ShouldBeNil)

				_, err = topicChan.AddParticipant(account1.Id)
				So(err, ShouldBeNil)

				body := fmt.Sprintf("hi @%s i heard that @%s is not in this channel?", account1.Nick, account2.Nick)

				cm := socialapimodels.CreateMessageWithBody(topicChan.Id, adminAccount.Id, models.ChannelMessage_TYPE_POST, body)

				usernames, err := NewNormalizer(cm, []string{account1.Nick, account2.Nick}, r.Log).Do()
				So(err, ShouldBeNil)
				So(len(usernames), ShouldEqual, 2)
			})

			Convey("non members of team should not be in mention list", func() {
				body := "hi @nonmember how are things with your @girlfriend?"
				cm := socialapimodels.CreateMessageWithBody(topicChan.Id, account2.Id, models.ChannelMessage_TYPE_POST, body)

				usernames, err := NewNormalizer(cm, []string{"nonmember", "girlfriend"}, r.Log).Do()
				So(err, ShouldBeNil)
				So(len(usernames), ShouldEqual, 0)
			})
		})
	})
}

func createGroup(groupName string) (*mongomodels.Group, error) {
	g := &mongomodels.Group{
		Id:         bson.NewObjectId(),
		Body:       bson.NewObjectId().Hex(),
		Title:      bson.NewObjectId().Hex(),
		Slug:       groupName,
		Privacy:    "private",
		Visibility: "hidden",
	}
	return g, modelhelper.CreateGroup(g)
}

func makeAdmin(accountId, groupId bson.ObjectId) error {
	r := &mongomodels.Relationship{
		Id:         bson.NewObjectId(),
		TargetId:   accountId,
		TargetName: "JAccount",
		SourceId:   groupId,
		SourceName: "JGroup",
		As:         "admin",
		TimeStamp:  time.Now().UTC(),
	}

	return modelhelper.AddRelationship(r)
}
<commit_msg>Socialapi:added more test cases<commit_after>package notification

import (
	"fmt"
	mongomodels "koding/db/models"
	"koding/db/mongodb/modelhelper"
	"socialapi/config"
	"socialapi/models"
	socialapimodels "socialapi/models"
	"socialapi/workers/common/tests"
	"testing"
	"time"

	"github.com/koding/runner"
	. "github.com/smartystreets/goconvey/convey"
	"labix.org/v2/mgo/bson"
)

func TestCleanup(t *testing.T) {
	testData := []struct {
		definition string
		usernames  []string
		expected   []string
	}{
		{
			"should remove aliases",
			[]string{"team", "all"},
			[]string{"all"},
		},
		{
			"should return same usernames",
			[]string{"foo", "bar", "zaar"},
			[]string{"foo", "bar", "zaar"},
		},
		{
			"should remove duplicates",
			[]string{"admins", "admins", "ff"},
			[]string{"admins", "ff"},
		},
		{
			"should remove specific ones if have a general one",
			[]string{"admins", "admins", "team"},
			[]string{"all"},
		},
		{
			"should reduce to global alias",
			[]string{"team", "all", "group"},
			[]string{"all"},
		},
		{
			// some of the admins may not be in the channel
			"should keep channel and admins",
			[]string{"channel", "bar", "admins"},
			[]string{"channel", "bar", "admins"},
		},
	}

	for _, test := range testData {
		responses := cleanup(test.usernames)
		exists := false
		for _, response := range responses {
			for _, exc := range test.expected {
				if exc == response {
					exists = true
					break
				}
			}
		}

		if !exists {
			t.Fatalf("%s. expected: %+v, got: %+v", test.definition, responses)
		}

		if len(test.expected) != len(responses) {
			t.Fatalf("%s, %s. expected: %+v, got: %+v", test.definition, "expected lengths are not same", test.expected, responses)
		}
	}
}

func TestNormalize(t *testing.T) {
	tests.WithRunner(t, func(r *runner.Runner) {

		appConfig := config.MustRead(r.Conf.Path)
		modelhelper.Initialize(appConfig.Mongo)
		defer modelhelper.Close()

		Convey("while normalizing the usernames to their original nicks", t, func() {
			adminAccount, groupChannel, _ := models.CreateRandomGroupDataWithChecks()
			account1 := models.CreateAccountInBothDbsWithCheck()
			account2 := models.CreateAccountInBothDbsWithCheck()
			account3 := models.CreateAccountInBothDbsWithCheck()

			_, err := groupChannel.AddParticipant(account1.Id)
			So(err, ShouldBeNil)
			_, err = groupChannel.AddParticipant(account2.Id)
			So(err, ShouldBeNil)
			_, err = groupChannel.AddParticipant(account3.Id)
			So(err, ShouldBeNil)

			topicChan := socialapimodels.CreateTypedGroupedChannelWithTest(account1.Id, socialapimodels.Channel_TYPE_TOPIC, groupChannel.GroupName)

			Convey("@all should return all the members of the team", func() {
				body := "hi @all i am really excited to join this team!"
				cm := models.CreateMessageWithBody(groupChannel.Id, adminAccount.Id, models.ChannelMessage_TYPE_POST, body)

				usernames, err := NewNormalizer(cm, []string{"all"}, r.Log).Do()
				So(err, ShouldBeNil)
				So(len(usernames), ShouldEqual, 3)

				Convey("poster should not be in the mention list", func() {
					So(socialapimodels.IsIn(adminAccount.Nick, usernames...), ShouldBeFalse)
				})
			})

			Convey("@team should return all the members of the team in a non-group channel", func() {
				_, err := topicChan.AddParticipant(adminAccount.Id)
				So(err, ShouldBeNil)
				_, err = topicChan.AddParticipant(account2.Id)
				So(err, ShouldBeNil)
				_, err = topicChan.AddParticipant(account3.Id)
				So(err, ShouldBeNil)

				body := "hi @team i am really excited to join this chan!"
				cm := models.CreateMessageWithBody(topicChan.Id, adminAccount.Id, models.ChannelMessage_TYPE_POST, body)

				usernames, err := NewNormalizer(cm, []string{"team"}, r.Log).Do()
				So(err, ShouldBeNil)
				So(len(usernames), ShouldEqual, 3)
			})

			Convey("@all + any username should return all the members of the team", func() {
				body := "hi @all i am really excited to join this team! how are you @" + account3.Nick
				cm := models.CreateMessageWithBody(groupChannel.Id, adminAccount.Id, models.ChannelMessage_TYPE_POST, body)

				usernames, err := NewNormalizer(cm, []string{"all", account3.Nick}, r.Log).Do()
				So(err, ShouldBeNil)
				So(len(usernames), ShouldEqual, 3)
			})

			Convey("@channel should return all the members of the channel", func() {

				body := "hi @channel"
				cm := socialapimodels.CreateMessageWithBody(topicChan.Id, account1.Id, models.ChannelMessage_TYPE_POST, body)

				Convey("if channel doesnt have any members", func() {
					Convey("should return 0 username", func() {
						usernames, err := NewNormalizer(cm, []string{"channel"}, r.Log).Do()
						So(err, ShouldBeNil)
						So(len(usernames), ShouldEqual, 0)
					})
				})

				Convey("if channel have member", func() {
					Convey("should return them", func() {
						_, err := topicChan.AddParticipant(account2.Id)
						So(err, ShouldBeNil)

						usernames, err := NewNormalizer(cm, []string{"channel"}, r.Log).Do()
						So(err, ShouldBeNil)
						So(len(usernames), ShouldEqual, 1)
						So(usernames[0], ShouldEqual, account2.Nick)
					})
				})
			})

			Convey("@channel + @group should return all the members of the team", func() {
				body := "hi @channel i am glad that i joined @group"
				cm := socialapimodels.CreateMessageWithBody(topicChan.Id, account1.Id, models.ChannelMessage_TYPE_POST, body)

				usernames, err := NewNormalizer(cm, []string{"channel", "group"}, r.Log).Do()
				So(err, ShouldBeNil)
				So(len(usernames), ShouldEqual, 3)
			})

			Convey("@admins should return all the admins of the team", func() {

				// create the group
				group, err := createGroup(groupChannel.GroupName)
				So(err, ShouldBeNil)

				err = makeAdmin(bson.ObjectIdHex(account1.OldId), group.Id)
				So(err, ShouldBeNil)

				body := "hi @admins make me mod plzz"
				cm := socialapimodels.CreateMessageWithBody(topicChan.Id, account2.Id, models.ChannelMessage_TYPE_POST, body)

				usernames, err := NewNormalizer(cm, []string{"admins"}, r.Log).Do()
				So(err, ShouldBeNil)
				So(len(usernames), ShouldEqual, 1)
				So(usernames[0], ShouldEqual, account1.Nick)

				Convey("adding another user to mention list should work", func() {
					_, err := topicChan.AddParticipant(account2.Id)
					So(err, ShouldBeNil)

					body := fmt.Sprintf("hi @%s do you know who are in @admins ? i believe @%s is in", account3.Nick, account2.Nick)
					cm := socialapimodels.CreateMessageWithBody(topicChan.Id, account2.Id, models.ChannelMessage_TYPE_POST, body)

					usernames, err := NewNormalizer(cm, []string{"admins", account3.Nick}, r.Log).Do()
					So(err, ShouldBeNil)
					So(len(usernames), ShouldEqual, 2)
				})
			})

			Convey("non members of channel should not be in mention list", func() {
				_, err := topicChan.AddParticipant(adminAccount.Id)
				So(err, ShouldBeNil)

				_, err = topicChan.AddParticipant(account1.Id)
				So(err, ShouldBeNil)

				body := fmt.Sprintf("hi @%s i heard that @%s is not in this channel?", account1.Nick, account2.Nick)

				cm := socialapimodels.CreateMessageWithBody(topicChan.Id, adminAccount.Id, models.ChannelMessage_TYPE_POST, body)

				usernames, err := NewNormalizer(cm, []string{account1.Nick, account2.Nick}, r.Log).Do()
				So(err, ShouldBeNil)
				So(len(usernames), ShouldEqual, 2)
			})

			Convey("non members of team should not be in mention list", func() {
				nonmember := models.CreateAccountInBothDbsWithCheck()

				body := "hi @" + nonmember.Nick
				cm := socialapimodels.CreateMessageWithBody(topicChan.Id, account2.Id, models.ChannelMessage_TYPE_POST, body)

				usernames, err := NewNormalizer(cm, []string{nonmember.Nick}, r.Log).Do()
				So(err, ShouldBeNil)
				So(len(usernames), ShouldEqual, 0)
			})

			Convey("non existing members of team should not be in mention list", func() {
				body := "hi @nonmember how are things with your @girlfriend?"
				cm := socialapimodels.CreateMessageWithBody(topicChan.Id, account2.Id, models.ChannelMessage_TYPE_POST, body)

				usernames, err := NewNormalizer(cm, []string{"nonmember", "girlfriend"}, r.Log).Do()
				So(err, ShouldBeNil)
				So(len(usernames), ShouldEqual, 0)
			})

			Convey("non members of a private channel should not be in mention list", func() {
				nonmember := models.CreateAccountInBothDbsWithCheck()
				_, err := groupChannel.AddParticipant(nonmember.Id)
				So(err, ShouldBeNil)

				pmChan := socialapimodels.CreateTypedGroupedChannelWithTest(account1.Id, socialapimodels.Channel_TYPE_PRIVATE_MESSAGE, groupChannel.GroupName)
				_, err = pmChan.AddParticipant(account1.Id)
				So(err, ShouldBeNil)
				_, err = pmChan.AddParticipant(account2.Id)
				So(err, ShouldBeNil)

				body := "hi @" + nonmember.Nick + " and @" + account1.Nick
				cm := socialapimodels.CreateMessageWithBody(pmChan.Id, account2.Id, models.ChannelMessage_TYPE_PRIVATE_MESSAGE, body)

				usernames, err := NewNormalizer(cm, []string{nonmember.Nick, account1.Nick}, r.Log).Do()
				So(err, ShouldBeNil)
				So(len(usernames), ShouldEqual, 1)
			})

		})
	})
}

func createGroup(groupName string) (*mongomodels.Group, error) {
	g := &mongomodels.Group{
		Id:         bson.NewObjectId(),
		Body:       bson.NewObjectId().Hex(),
		Title:      bson.NewObjectId().Hex(),
		Slug:       groupName,
		Privacy:    "private",
		Visibility: "hidden",
	}
	return g, modelhelper.CreateGroup(g)
}

func makeAdmin(accountId, groupId bson.ObjectId) error {
	r := &mongomodels.Relationship{
		Id:         bson.NewObjectId(),
		TargetId:   accountId,
		TargetName: "JAccount",
		SourceId:   groupId,
		SourceName: "JGroup",
		As:         "admin",
		TimeStamp:  time.Now().UTC(),
	}

	return modelhelper.AddRelationship(r)
}
<|endoftext|>


<commit_before>package applier_test

import (
	. "bosh/agent/applier"
	fakeas "bosh/agent/applier/applyspec/fakes"
	fakeja "bosh/agent/applier/jobapplier/fakes"
	models "bosh/agent/applier/models"
	fakepa "bosh/agent/applier/packageapplier/fakes"
	fakejobsuper "bosh/jobsupervisor/fakes"
	boshsettings "bosh/settings"
	boshdirs "bosh/settings/directories"
	"errors"
	. "github.com/onsi/ginkgo"
	"github.com/stretchr/testify/assert"
)

type FakeLogRotateDelegate struct {
	SetupLogrotateErr  error
	SetupLogrotateArgs SetupLogrotateArgs
}

type SetupLogrotateArgs struct {
	GroupName string
	BasePath  string
	Size      string
}

func (d *FakeLogRotateDelegate) SetupLogrotate(groupName, basePath, size string) (err error) {
	d.SetupLogrotateArgs = SetupLogrotateArgs{groupName, basePath, size}

	if d.SetupLogrotateErr != nil {
		err = d.SetupLogrotateErr
	}

	return
}

func buildApplier() (
	*fakeja.FakeJobApplier,
	*fakepa.FakePackageApplier,
	*FakeLogRotateDelegate,
	*fakejobsuper.FakeJobSupervisor,
	Applier,
) {
	jobApplier := fakeja.NewFakeJobApplier()
	packageApplier := fakepa.NewFakePackageApplier()
	platform := &FakeLogRotateDelegate{}
	jobSupervisor := fakejobsuper.NewFakeJobSupervisor()
	applier := NewConcreteApplier(jobApplier, packageApplier, platform, jobSupervisor, boshdirs.NewDirectoriesProvider("/fake-base-dir"))
	return jobApplier, packageApplier, platform, jobSupervisor, applier
}

func buildJob() models.Job {
	return models.Job{Name: "fake-job-name", Version: "fake-version-name"}
}

func buildPackage() models.Package {
	return models.Package{Name: "fake-package-name", Version: "fake-package-name"}
}

func init() {
	Describe("Testing with Ginkgo", func() {
		It("apply applies jobs", func() {
			jobApplier, _, _, _, applier := buildApplier()
			job := buildJob()

			err := applier.Apply(&fakeas.FakeApplySpec{JobResults: []models.Job{job}})
			assert.NoError(GinkgoT(), err)
			assert.Equal(GinkgoT(), jobApplier.AppliedJobs, []models.Job{job})
		})
		It("apply errs when applying jobs errs", func() {

			jobApplier, _, _, _, applier := buildApplier()
			job := buildJob()

			jobApplier.ApplyError = errors.New("fake-apply-job-error")

			err := applier.Apply(&fakeas.FakeApplySpec{JobResults: []models.Job{job}})
			assert.Error(GinkgoT(), err)
			assert.Contains(GinkgoT(), err.Error(), "fake-apply-job-error")
		})
		It("apply applies packages", func() {

			_, packageApplier, _, _, applier := buildApplier()

			pkg1 := buildPackage()
			pkg2 := buildPackage()

			err := applier.Apply(&fakeas.FakeApplySpec{PackageResults: []models.Package{pkg1, pkg2}})
			assert.NoError(GinkgoT(), err)
			assert.Equal(GinkgoT(), packageApplier.AppliedPackages, []models.Package{pkg1, pkg2})
		})
		It("apply errs when applying packages errs", func() {

			_, packageApplier, _, _, applier := buildApplier()
			pkg := buildPackage()

			packageApplier.ApplyError = errors.New("fake-apply-package-error")

			err := applier.Apply(&fakeas.FakeApplySpec{PackageResults: []models.Package{pkg}})
			assert.Error(GinkgoT(), err)
			assert.Contains(GinkgoT(), err.Error(), "fake-apply-package-error")
		})
		It("apply configures jobs", func() {

			jobApplier, _, _, jobSupervisor, applier := buildApplier()

			job1 := models.Job{Name: "fake-job-name-1", Version: "fake-version-name-1"}
			job2 := models.Job{Name: "fake-job-name-2", Version: "fake-version-name-2"}
			jobs := []models.Job{job1, job2}

			err := applier.Apply(&fakeas.FakeApplySpec{JobResults: jobs})
			assert.NoError(GinkgoT(), err)
			assert.Equal(GinkgoT(), jobApplier.ConfiguredJobs, []models.Job{job2, job1})
			assert.Equal(GinkgoT(), jobApplier.ConfiguredJobIndices, []int{0, 1})

			assert.True(GinkgoT(), jobSupervisor.Reloaded)
		})
		It("apply errs if monitor fails reload", func() {

			_, _, _, jobSupervisor, applier := buildApplier()
			jobs := []models.Job{}
			jobSupervisor.ReloadErr = errors.New("error reloading monit")

			err := applier.Apply(&fakeas.FakeApplySpec{JobResults: jobs})
			assert.Error(GinkgoT(), err)
			assert.Contains(GinkgoT(), err.Error(), "error reloading monit")
		})
		It("apply errs if a job fails configuring", func() {

			jobApplier, _, _, _, applier := buildApplier()
			jobApplier.ConfigureError = errors.New("error configuring job")

			job := models.Job{Name: "fake-job-name-1", Version: "fake-version-name-1"}

			err := applier.Apply(&fakeas.FakeApplySpec{JobResults: []models.Job{job}})
			assert.Error(GinkgoT(), err)
			assert.Contains(GinkgoT(), err.Error(), "error configuring job")
		})
		It("apply sets up logrotation", func() {

			_, _, platform, _, applier := buildApplier()

			err := applier.Apply(&fakeas.FakeApplySpec{MaxLogFileSizeResult: "fake-size"})
			assert.NoError(GinkgoT(), err)
			assert.Equal(GinkgoT(), platform.SetupLogrotateArgs, SetupLogrotateArgs{
				GroupName: boshsettings.VCAP_USERNAME,
				BasePath:  "/fake-base-dir",
				Size:      "fake-size",
			})
		})
		It("apply errs if setup logrotate fails", func() {

			_, _, platform, _, applier := buildApplier()

			platform.SetupLogrotateErr = errors.New("fake-msg")

			err := applier.Apply(&fakeas.FakeApplySpec{})
			assert.Error(GinkgoT(), err)
			assert.Contains(GinkgoT(), err.Error(), "Logrotate setup failed: fake-msg")
		})
	})
}
<commit_msg>formatting concrete_applier_test.go<commit_after>package applier_test

import (
	"errors"

	. "github.com/onsi/ginkgo"
	"github.com/stretchr/testify/assert"

	. "bosh/agent/applier"
	fakeas "bosh/agent/applier/applyspec/fakes"
	fakeja "bosh/agent/applier/jobapplier/fakes"
	models "bosh/agent/applier/models"
	fakepa "bosh/agent/applier/packageapplier/fakes"
	fakejobsuper "bosh/jobsupervisor/fakes"
	boshsettings "bosh/settings"
	boshdirs "bosh/settings/directories"
)

type FakeLogRotateDelegate struct {
	SetupLogrotateErr  error
	SetupLogrotateArgs SetupLogrotateArgs
}

type SetupLogrotateArgs struct {
	GroupName string
	BasePath  string
	Size      string
}

func (d *FakeLogRotateDelegate) SetupLogrotate(groupName, basePath, size string) error {
	d.SetupLogrotateArgs = SetupLogrotateArgs{groupName, basePath, size}
	if d.SetupLogrotateErr != nil {
		return d.SetupLogrotateErr
	}
	return nil
}

func buildJob() models.Job {
	return models.Job{Name: "fake-job-name", Version: "fake-version-name"}
}

func buildPackage() models.Package {
	return models.Package{Name: "fake-package-name", Version: "fake-package-name"}
}

func init() {
	Describe("concreteApplier", func() {
		var (
			jobApplier *fakeja.FakeJobApplier
			packageApplier *fakepa.FakePackageApplier
			logRotateDelegate *FakeLogRotateDelegate
			jobSupervisor *fakejobsuper.FakeJobSupervisor
			applier Applier
		)

		BeforeEach(func() {
			jobApplier = fakeja.NewFakeJobApplier()
			packageApplier = fakepa.NewFakePackageApplier()
			logRotateDelegate = &FakeLogRotateDelegate{}
			jobSupervisor = fakejobsuper.NewFakeJobSupervisor()
			applier = NewConcreteApplier(
				jobApplier,
				packageApplier,
				logRotateDelegate,
				jobSupervisor,
				boshdirs.NewDirectoriesProvider("/fake-base-dir"),
			)
		})

		It("apply applies jobs", func() {
			job := buildJob()

			err := applier.Apply(&fakeas.FakeApplySpec{JobResults: []models.Job{job}})
			assert.NoError(GinkgoT(), err)
			assert.Equal(GinkgoT(), jobApplier.AppliedJobs, []models.Job{job})
		})

		It("apply errs when applying jobs errs", func() {
			job := buildJob()

			jobApplier.ApplyError = errors.New("fake-apply-job-error")

			err := applier.Apply(&fakeas.FakeApplySpec{JobResults: []models.Job{job}})
			assert.Error(GinkgoT(), err)
			assert.Contains(GinkgoT(), err.Error(), "fake-apply-job-error")
		})

		It("apply applies packages", func() {
			pkg1 := buildPackage()
			pkg2 := buildPackage()

			err := applier.Apply(&fakeas.FakeApplySpec{PackageResults: []models.Package{pkg1, pkg2}})
			assert.NoError(GinkgoT(), err)
			assert.Equal(GinkgoT(), packageApplier.AppliedPackages, []models.Package{pkg1, pkg2})
		})

		It("apply errs when applying packages errs", func() {
			pkg := buildPackage()

			packageApplier.ApplyError = errors.New("fake-apply-package-error")

			err := applier.Apply(&fakeas.FakeApplySpec{PackageResults: []models.Package{pkg}})
			assert.Error(GinkgoT(), err)
			assert.Contains(GinkgoT(), err.Error(), "fake-apply-package-error")
		})

		It("apply configures jobs", func() {
			job1 := models.Job{Name: "fake-job-name-1", Version: "fake-version-name-1"}
			job2 := models.Job{Name: "fake-job-name-2", Version: "fake-version-name-2"}
			jobs := []models.Job{job1, job2}

			err := applier.Apply(&fakeas.FakeApplySpec{JobResults: jobs})
			assert.NoError(GinkgoT(), err)
			assert.Equal(GinkgoT(), jobApplier.ConfiguredJobs, []models.Job{job2, job1})
			assert.Equal(GinkgoT(), jobApplier.ConfiguredJobIndices, []int{0, 1})

			assert.True(GinkgoT(), jobSupervisor.Reloaded)
		})

		It("apply errs if monitor fails reload", func() {
			jobs := []models.Job{}
			jobSupervisor.ReloadErr = errors.New("error reloading monit")

			err := applier.Apply(&fakeas.FakeApplySpec{JobResults: jobs})
			assert.Error(GinkgoT(), err)
			assert.Contains(GinkgoT(), err.Error(), "error reloading monit")
		})

		It("apply errs if a job fails configuring", func() {
			jobApplier.ConfigureError = errors.New("error configuring job")

			job := models.Job{Name: "fake-job-name-1", Version: "fake-version-name-1"}

			err := applier.Apply(&fakeas.FakeApplySpec{JobResults: []models.Job{job}})
			assert.Error(GinkgoT(), err)
			assert.Contains(GinkgoT(), err.Error(), "error configuring job")
		})

		It("apply sets up logrotation", func() {
			err := applier.Apply(&fakeas.FakeApplySpec{MaxLogFileSizeResult: "fake-size"})
			assert.NoError(GinkgoT(), err)
			assert.Equal(GinkgoT(), logRotateDelegate.SetupLogrotateArgs, SetupLogrotateArgs{
				GroupName: boshsettings.VCAP_USERNAME,
				BasePath:  "/fake-base-dir",
				Size:      "fake-size",
			})
		})

		It("apply errs if setup logrotate fails", func() {
			logRotateDelegate.SetupLogrotateErr = errors.New("fake-msg")

			err := applier.Apply(&fakeas.FakeApplySpec{})
			assert.Error(GinkgoT(), err)
			assert.Contains(GinkgoT(), err.Error(), "Logrotate setup failed: fake-msg")
		})
	})
}
<|endoftext|>


<commit_before>package fileserver

import (
	"github.com/yarf-framework/yarf"
	"net/http"
	"os"
	"strings"
)

type File struct {
	// Implements Resource
	yarf.Resource

	// Points to www_root
	Path string

	// Prefix to exclude on path construction
	Prefix string
}

// Implement the GET handler
func (f *File) Get(c *yarf.Context) error {
	// Construct path
	path := f.Path + strings.TrimPrefix(c.Request.URL.EscapedPath(), f.Prefix)

	// Check if file exists
	if _, err := os.Stat(path); os.IsNotExist(err) {
		// Check that isn't index request
		if _, err := os.Stat(path + "/index.html"); os.IsNotExist(err) {
			return yarf.ErrorNotFound()
		}
	}

	http.ServeFile(c.Response, c.Request, path)

	return nil
}

func New(path, prefix string) *yarf.Yarf {
	// Init server
	y := yarf.New()

	// Init resource
	f := new(File)
	f.Path = path
	f.Prefix = prefix

	y.Add("/", f)

	return y
}
<commit_msg>Fix fileserver<commit_after>package fileserver

import (
	"github.com/yarf-framework/yarf"
	"net/http"
	"os"
	"strings"
)

type File struct {
	// Implements Resource
	yarf.Resource

	// Points to www_root
	Path string

	// Prefix to exclude on path construction
	Prefix string
}

// Implement the GET handler
func (f *File) Get(c *yarf.Context) error {
	// Construct path
	path := f.Path + strings.TrimPrefix(c.Request.URL.EscapedPath(), f.Prefix)

	// Check if file exists
	if _, err := os.Stat(path); os.IsNotExist(err) {
		// Check that isn't index request
		if _, err := os.Stat(path + "/index.html"); os.IsNotExist(err) {
			return yarf.ErrorNotFound()
		}
	}

	http.ServeFile(c.Response, c.Request, path)

	return nil
}

func New(path, prefix string) *yarf.Yarf {
	// Init server
	y := yarf.New()

	// Set follow to file server
	y.Follow = http.StripPrefix(prefix, http.FileServer(http.Dir(path)))

	// Return without routes.
	return y
}
<|endoftext|>


<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package main

import (
	"encoding/json"
	"github.com/gorilla/mux"
	"log"
	"mime"
	"net/http"
	"org/apache/htrace/common"
	"org/apache/htrace/conf"
	"org/apache/htrace/resource"
	"path/filepath"
	"strconv"
	"strings"
)

type serverInfoHandler struct {
}

func (handler *serverInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	version := common.ServerInfo{ReleaseVersion: common.RELEASE_VERSION,
		GitVersion: common.GIT_VERSION}
	buf, err := json.Marshal(&version)
	if err != nil {
		log.Printf("error marshalling ServerInfo: %s\n", err.Error())
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	w.Write(buf)
}

type dataStoreHandler struct {
	store *dataStore
}

func (hand *dataStoreHandler) parse64(w http.ResponseWriter, str string) (int64, bool) {
	val, err := strconv.ParseUint(str, 16, 64)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte("Error parsing : " + err.Error()))
		return -1, false
	}
	return int64(val), true
}

func (hand *dataStoreHandler) getReqField32(fieldName string, w http.ResponseWriter,
	req *http.Request) (int32, bool) {
	str := req.FormValue(fieldName)
	if str == "" {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte("No " + fieldName + " specified."))
		return -1, false
	}
	val, err := strconv.ParseUint(str, 16, 32)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte("Error parsing " + fieldName + ": " + err.Error()))
		return -1, false
	}
	return int32(val), true
}

type findSidHandler struct {
	dataStoreHandler
}

func (hand *findSidHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	req.ParseForm()
	vars := mux.Vars(req)
	stringSid := vars["id"]
	sid, ok := hand.parse64(w, stringSid)
	if !ok {
		return
	}
	span := hand.store.FindSpan(sid)
	if span == nil {
		w.WriteHeader(http.StatusNoContent)
		return
	}
	w.Write(span.ToJson())
}

type findChildrenHandler struct {
	dataStoreHandler
}

func (hand *findChildrenHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	req.ParseForm()
	vars := mux.Vars(req)
	stringSid := vars["id"]
	sid, ok := hand.parse64(w, stringSid)
	if !ok {
		return
	}
	var lim int32
	lim, ok = hand.getReqField32("lim", w, req)
	if !ok {
		return
	}
	children := hand.store.FindChildren(sid, lim)
	if len(children) == 0 {
		w.WriteHeader(http.StatusNoContent)
		return
	}
	jbytes, err := json.Marshal(children)
	if err != nil {
		panic(err)
	}
	w.Write(jbytes)
}

type defaultServeHandler struct {
}

func (hand *defaultServeHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	ident := strings.TrimLeft(req.URL.Path, "/")
	ident = strings.Replace(ident, "/", "__", -1)
	rsc := resource.Catalog[ident]
	if rsc == "" {
		log.Printf("failed to find entry for %s\n", ident)
		w.WriteHeader(http.StatusNotFound)
		return
	}
	file_ext := filepath.Ext(req.URL.Path)
	mime_type := mime.TypeByExtension(file_ext)
	w.Header().Set("Content-Type", mime_type)
	w.Write([]byte(rsc))
}

func startRestServer(cnf *conf.Config, store *dataStore) {

	r := mux.NewRouter().StrictSlash(false)
	// Default Handler. This will serve requests for static requests.
	r.Handle("/", &defaultServeHandler{})

	r.Handle("/server/info", &serverInfoHandler{}).Methods("GET")

	span := r.PathPrefix("/span").Subrouter()
	findSidH := &findSidHandler{dataStoreHandler: dataStoreHandler{store: store}}
	span.Handle("/{id}", findSidH).Methods("GET")

	findChildrenH := &findChildrenHandler{dataStoreHandler: dataStoreHandler{store: store}}
	span.Handle("/{id}/children", findChildrenH).Methods("GET")

	http.ListenAndServe(cnf.Get(conf.HTRACE_WEB_ADDRESS), r)
	log.Println("Started REST server...")
}
<commit_msg>HTRACE-37. htraced: serve index.html if the root URL was given (cmccabe)<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package main

import (
	"encoding/json"
	"github.com/gorilla/mux"
	"log"
	"mime"
	"net/http"
	"org/apache/htrace/common"
	"org/apache/htrace/conf"
	"org/apache/htrace/resource"
	"path/filepath"
	"strconv"
	"strings"
)

type serverInfoHandler struct {
}

func (handler *serverInfoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	version := common.ServerInfo{ReleaseVersion: common.RELEASE_VERSION,
		GitVersion: common.GIT_VERSION}
	buf, err := json.Marshal(&version)
	if err != nil {
		log.Printf("error marshalling ServerInfo: %s\n", err.Error())
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	w.Write(buf)
}

type dataStoreHandler struct {
	store *dataStore
}

func (hand *dataStoreHandler) parse64(w http.ResponseWriter, str string) (int64, bool) {
	val, err := strconv.ParseUint(str, 16, 64)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte("Error parsing : " + err.Error()))
		return -1, false
	}
	return int64(val), true
}

func (hand *dataStoreHandler) getReqField32(fieldName string, w http.ResponseWriter,
	req *http.Request) (int32, bool) {
	str := req.FormValue(fieldName)
	if str == "" {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte("No " + fieldName + " specified."))
		return -1, false
	}
	val, err := strconv.ParseUint(str, 16, 32)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte("Error parsing " + fieldName + ": " + err.Error()))
		return -1, false
	}
	return int32(val), true
}

type findSidHandler struct {
	dataStoreHandler
}

func (hand *findSidHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	req.ParseForm()
	vars := mux.Vars(req)
	stringSid := vars["id"]
	sid, ok := hand.parse64(w, stringSid)
	if !ok {
		return
	}
	span := hand.store.FindSpan(sid)
	if span == nil {
		w.WriteHeader(http.StatusNoContent)
		return
	}
	w.Write(span.ToJson())
}

type findChildrenHandler struct {
	dataStoreHandler
}

func (hand *findChildrenHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	req.ParseForm()
	vars := mux.Vars(req)
	stringSid := vars["id"]
	sid, ok := hand.parse64(w, stringSid)
	if !ok {
		return
	}
	var lim int32
	lim, ok = hand.getReqField32("lim", w, req)
	if !ok {
		return
	}
	children := hand.store.FindChildren(sid, lim)
	if len(children) == 0 {
		w.WriteHeader(http.StatusNoContent)
		return
	}
	jbytes, err := json.Marshal(children)
	if err != nil {
		panic(err)
	}
	w.Write(jbytes)
}

type defaultServeHandler struct {
}

func (hand *defaultServeHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	ident := strings.TrimLeft(req.URL.Path, "/")
	if ident == "" {
		ident = "index.html" // default to index.html
	}
	ident = strings.Replace(ident, "/", "__", -1)
	rsc := resource.Catalog[ident]
	if rsc == "" {
		log.Printf("failed to find entry for %s\n", ident)
		w.WriteHeader(http.StatusNotFound)
		return
	}
	file_ext := filepath.Ext(req.URL.Path)
	mime_type := mime.TypeByExtension(file_ext)
	w.Header().Set("Content-Type", mime_type)
	w.Write([]byte(rsc))
}

func startRestServer(cnf *conf.Config, store *dataStore) {

	r := mux.NewRouter().StrictSlash(false)
	// Default Handler. This will serve requests for static requests.
	r.Handle("/", &defaultServeHandler{})

	r.Handle("/server/info", &serverInfoHandler{}).Methods("GET")

	span := r.PathPrefix("/span").Subrouter()
	findSidH := &findSidHandler{dataStoreHandler: dataStoreHandler{store: store}}
	span.Handle("/{id}", findSidH).Methods("GET")

	findChildrenH := &findChildrenHandler{dataStoreHandler: dataStoreHandler{store: store}}
	span.Handle("/{id}/children", findChildrenH).Methods("GET")

	http.ListenAndServe(cnf.Get(conf.HTRACE_WEB_ADDRESS), r)
	log.Println("Started REST server...")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"io/ioutil"
	"log"
	"net/http"
	"time"

	yaml "gopkg.in/yaml.v2"
)

type (
	// Reply stores reply for entrypoint
	Reply struct {
		Status int
		Body   string
		Delay  time.Duration
	}
)

func main() {
	addr := flag.String("url", ":8080", "URL to listen to")
	config := flag.String("config", "./mockhttp.yml", "Config file")
	flag.Parse()
	content, e := ioutil.ReadFile(*config)
	if e != nil {
		log.Fatal(e)
	}
	ymlCfg := make(map[string]Reply, 0)
	e = yaml.Unmarshal(content, &ymlCfg)
	if e != nil {
		log.Fatal(e)
	}
	for k, route := range ymlCfg {
		log.Println(k, route)
		http.HandleFunc(k, func(w http.ResponseWriter, r *http.Request) {
			v, ok := ymlCfg[r.URL.String()]
			if !ok {
				w.WriteHeader(404)
				return
			}
			log.Println("Requested URL:", r.URL.String(), v)
			if v.Delay > 0 {
				<-time.After(v.Delay)
			}
			if v.Status != 0 {
				w.WriteHeader(v.Status)
			}
			if v.Body != "" {
				w.Write([]byte(v.Body))
			}
		})
	}
	http.ListenAndServe(*addr, nil)
}
<commit_msg>use URL.Path to select route<commit_after>package main

import (
	"flag"
	"io/ioutil"
	"log"
	"net/http"
	"time"

	yaml "gopkg.in/yaml.v2"
)

type (
	// Reply stores reply for entrypoint
	Reply struct {
		Status int
		Body   string
		Delay  time.Duration
	}
)

func main() {
	addr := flag.String("url", ":8080", "URL to listen to")
	config := flag.String("config", "./mockhttp.yml", "Config file")
	flag.Parse()
	content, e := ioutil.ReadFile(*config)
	if e != nil {
		log.Fatal(e)
	}
	ymlCfg := make(map[string]Reply, 0)
	e = yaml.Unmarshal(content, &ymlCfg)
	if e != nil {
		log.Fatal(e)
	}
	for k, route := range ymlCfg {
		log.Println(k, route)
		http.HandleFunc(k, func(w http.ResponseWriter, r *http.Request) {

			v, ok := ymlCfg[r.URL.Path]
			if !ok {
				log.Println("No route found", r.URL.Path)
				w.WriteHeader(404)
				return
			}
			log.Println("Requested URL:", r.URL.String(), v)
			if v.Delay > 0 {
				<-time.After(v.Delay)
			}
			if v.Status != 0 {
				w.WriteHeader(v.Status)
			}
			if v.Body != "" {
				w.Write([]byte(v.Body))
			}
		})
	}
	http.ListenAndServe(*addr, nil)
}
<|endoftext|>


<commit_before>package class

import (
    //"fmt"
    //"log"
    . "jvmgo/any"
    "jvmgo/classfile"
    "jvmgo/classpath"
)

const (
    jlObjectName = "java/lang/Object"
    jlClassName = "java/lang/Class"
    jlStringName = "java/lang/String"
)

type ClassLoader struct {
    classPath   *classpath.ClassPath
    classMap    map[string]*Class
}

func (self *ClassLoader) Init() {
    self.LoadClass(jlObjectName)
    self.LoadClass(jlClassName)
    jlClassClass := self.classMap[jlClassName]
    for _, class := range self.classMap {
        if class.jClass == nil {
            class.jClass = jlClassClass.NewObj()
        }
    }
}

// todo dangerous
func (self *ClassLoader) getClass(name string) (*Class) {
    class := self.classMap[name]
    if class == nil {
        panic("class not loaded! " + name)
    }
    return class
}

func (self *ClassLoader) StringClass() (*Class) {
    return self.LoadClass(jlStringName)
}

func (self *ClassLoader) LoadClass(name string) (*Class) {
    class := self.classMap[name]
    if class != nil {
        // already loaded
        return class
    } else {
        return self.reallyLoadClass(name)
    }
}

func (self *ClassLoader) reallyLoadClass(name string) (*Class) {
    //log.Printf("load: %v", name)
    class := self.parseClassFile(name)
    class.classLoader = self
    self.loadSuperClassAndInterfaces(class)
    self.initStaticFields(class)
    self.initInstanceFields(class)
    self.classMap[name] = class
    class.staticFieldValues = make([]Any, class.staticFieldCount)
    class.zeroStaticFields()

    jlClassClass := self.classMap[jlClassName]
    if jlClassClass != nil {
        class.jClass = jlClassClass.NewObj()
    }

    return class
}

func (self *ClassLoader) parseClassFile(name string) (class *Class) {
    classData, err := self.classPath.ReadClassData(name)
    if err != nil {
        // todo
        panic("class not found: " + name + "!")
    }

    cf, err := classfile.ParseClassFile(classData)
    if err != nil {
        // todo
        panic("failed to parse class file: " + name + "!" + err.Error())
    }

    return cf2class(cf)
}

// todo
func (self *ClassLoader) loadSuperClassAndInterfaces(class *Class) {
    if class.superClassName != "" {
        class.superClass = self.LoadClass(class.superClassName)
    }
    for _, interfaceName := range class.interfaceNames {
        self.LoadClass(interfaceName)
    }
}

func (self *ClassLoader) initStaticFields(class *Class) {
    slotId := uint(0)
    for _, f := range class.fields {
        if f.IsStatic() {
            f.slot = uint(slotId)
            slotId++
        }
    }
    class.staticFieldCount = slotId
}

func (self *ClassLoader) initInstanceFields(class *Class) {
    slotId := uint(0)
    if class.superClassName != "" {
        superClass := self.getClass(class.superClassName)
        slotId = superClass.instanceFieldCount
    }
    for _, field := range class.fields {
        if !field.IsStatic() {
            field.slot = slotId
            slotId++
        }
    }
    class.instanceFieldCount = slotId
}

func NewClassLoader(cp *classpath.ClassPath) (*ClassLoader) {
    classMap := map[string]*Class{}
    return &ClassLoader{cp, classMap}
}
<commit_msg>simply code<commit_after>package class

import (
    //"fmt"
    //"log"
    . "jvmgo/any"
    "jvmgo/classfile"
    "jvmgo/classpath"
)

const (
    jlObjectName = "java/lang/Object"
    jlClassName = "java/lang/Class"
    jlStringName = "java/lang/String"
)

type ClassLoader struct {
    classPath   *classpath.ClassPath
    classMap    map[string]*Class
}

func (self *ClassLoader) Init() {
    self.LoadClass(jlObjectName)
    self.LoadClass(jlClassName)
    jlClassClass := self.classMap[jlClassName]
    for _, class := range self.classMap {
        if class.jClass == nil {
            class.jClass = jlClassClass.NewObj()
        }
    }
}

// todo dangerous
func (self *ClassLoader) getClass(name string) (*Class) {
    class := self.classMap[name]
    if class == nil {
        panic("class not loaded! " + name)
    }
    return class
}

func (self *ClassLoader) StringClass() (*Class) {
    return self.LoadClass(jlStringName)
}

func (self *ClassLoader) LoadClass(name string) (*Class) {
    class := self.classMap[name]
    if class != nil {
        // already loaded
        return class
    } else {
        return self.reallyLoadClass(name)
    }
}

func (self *ClassLoader) reallyLoadClass(name string) (*Class) {
    //log.Printf("load: %v", name)
    class := self.parseClassFile(name)
    class.classLoader = self
    self.loadSuperClassAndInterfaces(class)
    self.initStaticFields(class)
    self.initInstanceFields(class)
    self.classMap[name] = class
    class.staticFieldValues = make([]Any, class.staticFieldCount)
    class.zeroStaticFields()

    jlClassClass := self.classMap[jlClassName]
    if jlClassClass != nil {
        class.jClass = jlClassClass.NewObj()
    }

    return class
}

func (self *ClassLoader) parseClassFile(name string) (class *Class) {
    classData, err := self.classPath.ReadClassData(name)
    if err != nil {
        // todo
        panic("class not found: " + name + "!")
    }

    cf, err := classfile.ParseClassFile(classData)
    if err != nil {
        // todo
        panic("failed to parse class file: " + name + "!" + err.Error())
    }

    return cf2class(cf)
}

// todo
func (self *ClassLoader) loadSuperClassAndInterfaces(class *Class) {
    if class.superClassName != "" {
        class.superClass = self.LoadClass(class.superClassName)
    }
    for _, interfaceName := range class.interfaceNames {
        self.LoadClass(interfaceName)
    }
}

func (self *ClassLoader) initStaticFields(class *Class) {
    slotId := uint(0)
    for _, f := range class.fields {
        if f.IsStatic() {
            f.slot = uint(slotId)
            slotId++
        }
    }
    class.staticFieldCount = slotId
}

func (self *ClassLoader) initInstanceFields(class *Class) {
    slotId := uint(0)
    if class.superClassName != "" {
        slotId = class.superClass.instanceFieldCount
    }
    for _, field := range class.fields {
        if !field.IsStatic() {
            field.slot = slotId
            slotId++
        }
    }
    class.instanceFieldCount = slotId
}

func NewClassLoader(cp *classpath.ClassPath) (*ClassLoader) {
    classMap := map[string]*Class{}
    return &ClassLoader{cp, classMap}
}
<|endoftext|>


<commit_before>package plan_builder_test

import (
	"github.com/cloudfoundry/cli/cf/actors/plan_builder"
	"github.com/cloudfoundry/cli/cf/api/fakes"
	testorg "github.com/cloudfoundry/cli/cf/api/organizations/fakes"
	"github.com/cloudfoundry/cli/cf/models"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Plan builder", func() {
	var (
		builder plan_builder.PlanBuilder

		planRepo       *fakes.FakeServicePlanRepo
		visibilityRepo *fakes.FakeServicePlanVisibilityRepository
		orgRepo        *testorg.FakeOrganizationRepository

		plan1 models.ServicePlanFields
		plan2 models.ServicePlanFields

		org1 models.Organization
		org2 models.Organization
	)

	BeforeEach(func() {
		planRepo = &fakes.FakeServicePlanRepo{}
		visibilityRepo = &fakes.FakeServicePlanVisibilityRepository{}
		orgRepo = &testorg.FakeOrganizationRepository{}

		builder = plan_builder.NewBuilder(planRepo, visibilityRepo, orgRepo)

		plan1 = models.ServicePlanFields{
			Name:                "service-plan1",
			Guid:                "service-plan1-guid",
			ServiceOfferingGuid: "service-guid1",
		}

		plan2 = models.ServicePlanFields{
			Name:                "service-plan2",
			Guid:                "service-plan2-guid",
			ServiceOfferingGuid: "service-guid1",
		}
		planRepo.SearchReturns = map[string][]models.ServicePlanFields{
			"service-guid1": []models.ServicePlanFields{plan1, plan2},
		}
		org1 = models.Organization{}
		org1.Name = "org1"
		org1.Guid = "org1-guid"

		org2 = models.Organization{}
		org2.Name = "org2"
		org2.Guid = "org2-guid"
		/**
			orgRepo.Organizations = []models.Organization{
				org1,
				org2,
			}
		**/
		visibilityRepo.ListReturns([]models.ServicePlanVisibilityFields{
			{ServicePlanGuid: "service-plan1-guid", OrganizationGuid: "org1-guid"},
			{ServicePlanGuid: "service-plan1-guid", OrganizationGuid: "org2-guid"},
		}, nil)
	})

	Describe(".AttachOrgsToPlans", func() {
		It("returns plans fully populated with the orgnames that have visibility", func() {
			orgRepo.ListOrgsReturns([]models.Organization{org1, org2}, nil)
			barePlans := []models.ServicePlanFields{plan1, plan2}

			plans, err := builder.AttachOrgsToPlans(barePlans)
			Expect(err).ToNot(HaveOccurred())

			Expect(plans[0].OrgNames).To(Equal([]string{"org1", "org2"}))
		})
	})

	Describe(".GetPlansForService", func() {
		It("returns all the plans for the service with the provided guid", func() {
			plans, err := builder.GetPlansForService("service-guid1")
			Expect(err).ToNot(HaveOccurred())

			Expect(len(plans)).To(Equal(2))
			Expect(plans[0].Name).To(Equal("service-plan1"))
			Expect(plans[0].OrgNames).To(Equal([]string{"org1", "org2"}))
			Expect(plans[1].Name).To(Equal("service-plan2"))
		})
	})

	Describe(".GetPlansVisibleToOrg", func() {
		It("returns all the plans visible to the named org", func() {

			plans, err := builder.GetPlansVisibleToOrg("org1")
			Expect(err).ToNot(HaveOccurred())

			Expect(len(plans)).To(Equal(1))
			Expect(plans[0].Name).To(Equal("service-plan1"))
			Expect(plans[0].OrgNames).To(Equal([]string{"org1", "org2"}))
		})
	})
})
<commit_msg>Move test setup into beforeEach of plan_builder_test<commit_after>package plan_builder_test

import (
	"github.com/cloudfoundry/cli/cf/actors/plan_builder"
	"github.com/cloudfoundry/cli/cf/api/fakes"
	testorg "github.com/cloudfoundry/cli/cf/api/organizations/fakes"
	"github.com/cloudfoundry/cli/cf/models"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Plan builder", func() {
	var (
		builder plan_builder.PlanBuilder

		planRepo       *fakes.FakeServicePlanRepo
		visibilityRepo *fakes.FakeServicePlanVisibilityRepository
		orgRepo        *testorg.FakeOrganizationRepository

		plan1 models.ServicePlanFields
		plan2 models.ServicePlanFields

		org1 models.Organization
		org2 models.Organization
	)

	BeforeEach(func() {
		planRepo = &fakes.FakeServicePlanRepo{}
		visibilityRepo = &fakes.FakeServicePlanVisibilityRepository{}
		orgRepo = &testorg.FakeOrganizationRepository{}
		builder = plan_builder.NewBuilder(planRepo, visibilityRepo, orgRepo)

		plan1 = models.ServicePlanFields{
			Name:                "service-plan1",
			Guid:                "service-plan1-guid",
			ServiceOfferingGuid: "service-guid1",
		}

		plan2 = models.ServicePlanFields{
			Name:                "service-plan2",
			Guid:                "service-plan2-guid",
			ServiceOfferingGuid: "service-guid1",
		}
		planRepo.SearchReturns = map[string][]models.ServicePlanFields{
			"service-guid1": []models.ServicePlanFields{plan1, plan2},
		}
		org1 = models.Organization{}
		org1.Name = "org1"
		org1.Guid = "org1-guid"

		org2 = models.Organization{}
		org2.Name = "org2"
		org2.Guid = "org2-guid"
		/**
			orgRepo.Organizations = []models.Organization{
				org1,
				org2,
			}
		**/
		visibilityRepo.ListReturns([]models.ServicePlanVisibilityFields{
			{ServicePlanGuid: "service-plan1-guid", OrganizationGuid: "org1-guid"},
			{ServicePlanGuid: "service-plan1-guid", OrganizationGuid: "org2-guid"},
		}, nil)
		orgRepo.ListOrgsReturns([]models.Organization{org1, org2}, nil)
	})

	Describe(".AttachOrgsToPlans", func() {
		It("returns plans fully populated with the orgnames that have visibility", func() {
			barePlans := []models.ServicePlanFields{plan1, plan2}

			plans, err := builder.AttachOrgsToPlans(barePlans)
			Expect(err).ToNot(HaveOccurred())

			Expect(plans[0].OrgNames).To(Equal([]string{"org1", "org2"}))
		})
	})

	Describe(".GetPlansForService", func() {
		It("returns all the plans for the service with the provided guid", func() {
			plans, err := builder.GetPlansForService("service-guid1")
			Expect(err).ToNot(HaveOccurred())

			Expect(len(plans)).To(Equal(2))
			Expect(plans[0].Name).To(Equal("service-plan1"))
			Expect(plans[0].OrgNames).To(Equal([]string{"org1", "org2"}))
			Expect(plans[1].Name).To(Equal("service-plan2"))
		})
	})

	Describe(".GetPlansVisibleToOrg", func() {
		It("returns all the plans visible to the named org", func() {
			plans, err := builder.GetPlansVisibleToOrg("org1")
			Expect(err).ToNot(HaveOccurred())

			Expect(len(plans)).To(Equal(1))
			Expect(plans[0].Name).To(Equal("service-plan1"))
			Expect(plans[0].OrgNames).To(Equal([]string{"org1", "org2"}))
		})
	})
})
<|endoftext|>


<commit_before>package shutdown

import (
	"testing"
	"time"
)

func TestShutdown(t *testing.T) {
	s := New()
	go func() {
		time.Sleep(time.Millisecond)
		s.Start()
	}()
	s.WaitStart()

	go func() {
		time.Sleep(time.Millisecond)
		s.Done()
	}()
	s.WaitDown()
}
<commit_msg>fix ci<commit_after>package shutdown

import (
	"testing"
	"time"
)

func TestShutdown(t *testing.T) {
	s := New()
	go func() {
		time.Sleep(time.Millisecond)
		s.Start()
	}()
	s.WaitStart()

	go func() {
		time.Sleep(time.Millisecond)
		s.Done()
	}()
	s.WaitDone()
}
<|endoftext|>


<commit_before>package main

import (
	"math/rand"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/jeffail/tunny"
	cache "github.com/pmylund/go-cache"
)

const (
	defaultTimeout             = 10
	defaultWorkerPoolSize      = 50
	defaultSampletTriggerLimit = 3
)

var (
	HostCheckerClient = &http.Client{Timeout: 500 * time.Millisecond}

	hostCheckTicker = make(chan struct{})
)

type HostData struct {
	CheckURL string
	Method   string
	Headers  map[string]string
	Body     string
	MetaData map[string]string
}

type HostHealthReport struct {
	HostData
	ResponseCode int
	Latency      float64
	IsTCPError   bool
}

type HostUptimeChecker struct {
	failureCallback    func(HostHealthReport)
	upCallback         func(HostHealthReport)
	pingCallback       func(HostHealthReport)
	workerPoolSize     int
	sampleTriggerLimit int
	checkTimeout       int
	HostList           map[string]HostData
	unHealthyList      map[string]bool
	pool               *tunny.WorkPool

	errorChan       chan HostHealthReport
	okChan          chan HostHealthReport
	stopPollingChan chan bool
	sampleCache     *cache.Cache
	stopLoop        bool

	resetListMu sync.Mutex
	doResetList bool
	newList     map[string]HostData
}

func (h *HostUptimeChecker) getStaggeredTime() time.Duration {
	if h.checkTimeout <= 5 {
		return time.Duration(h.checkTimeout) * time.Second
	}

	rand.Seed(time.Now().Unix())
	min := h.checkTimeout - 3
	max := h.checkTimeout + 3

	dur := rand.Intn(max-min) + min

	return time.Duration(dur) * time.Second
}

func (h *HostUptimeChecker) HostCheckLoop() {
	for !h.stopLoop {
		if runningTests {
			<-hostCheckTicker
		}
		h.resetListMu.Lock()
		if h.doResetList && h.newList != nil {
			h.HostList = h.newList
			h.newList = nil
			h.doResetList = false
			log.Debug("[HOST CHECKER] Host list reset")
		}
		h.resetListMu.Unlock()
		for _, host := range h.HostList {
			_, err := h.pool.SendWork(host)
			if err != nil {
				log.Errorf("[HOST CHECKER] could not send work, error: %v", err)
			}
		}

		if !runningTests {
			time.Sleep(h.getStaggeredTime())
		}
	}
	log.Info("[HOST CHECKER] Checker stopped")
}

func (h *HostUptimeChecker) HostReporter() {
	for {
		select {
		case okHost := <-h.okChan:
			// Clear host from unhealthylist if it exists
			if h.unHealthyList[okHost.CheckURL] {
				h.upCallback(okHost)
				delete(h.unHealthyList, okHost.CheckURL)
			}
			go h.pingCallback(okHost)

		case failedHost := <-h.errorChan:
			newVal := 1
			if count, found := h.sampleCache.Get(failedHost.CheckURL); found {
				newVal = count.(int) + 1
			}

			h.sampleCache.Set(failedHost.CheckURL, newVal, cache.DefaultExpiration)

			if newVal >= h.sampleTriggerLimit {
				log.Debug("[HOST CHECKER] [HOST WARNING]: ", failedHost.CheckURL)
				// Reset the count
				h.sampleCache.Set(failedHost.CheckURL, 1, cache.DefaultExpiration)
				// track it
				h.unHealthyList[failedHost.CheckURL] = true
				// Call the custom callback hook
				go h.failureCallback(failedHost)
			}
			go h.pingCallback(failedHost)

		case <-h.stopPollingChan:
			log.Debug("[HOST CHECKER] Received kill signal")
			return
		}
	}
}

func (h *HostUptimeChecker) CheckHost(toCheck HostData) {
	log.Debug("[HOST CHECKER] Checking: ", toCheck.CheckURL)

	t1 := time.Now()

	useMethod := toCheck.Method
	if toCheck.Method == "" {
		useMethod = "GET"
	}

	req, err := http.NewRequest(useMethod, toCheck.CheckURL, strings.NewReader(toCheck.Body))
	if err != nil {
		log.Error("Could not create request: ", err)
		return
	}
	for header_name, header_value := range toCheck.Headers {
		req.Header.Set(header_name, header_value)
	}
	req.Header.Set("Connection", "close")

	response, err := HostCheckerClient.Do(req)

	t2 := time.Now()

	millisec := float64(t2.UnixNano()-t1.UnixNano()) * 0.000001

	report := HostHealthReport{
		HostData: toCheck,
		Latency:  millisec,
	}

	if err != nil {
		report.IsTCPError = true
		h.errorChan <- report
		return
	}

	report.ResponseCode = response.StatusCode

	if response.StatusCode != 200 {
		h.errorChan <- report
		return
	}

	// host is healthy, report it
	h.okChan <- report
}

func (h *HostUptimeChecker) Init(workers, triggerLimit, timeout int, hostList map[string]HostData, failureCallback func(HostHealthReport), upCallback func(HostHealthReport), pingCallback func(HostHealthReport)) {
	h.sampleCache = cache.New(30*time.Second, 5*time.Second)
	h.stopPollingChan = make(chan bool)
	h.errorChan = make(chan HostHealthReport)
	h.okChan = make(chan HostHealthReport)
	h.HostList = hostList
	h.unHealthyList = make(map[string]bool)
	h.failureCallback = failureCallback
	h.upCallback = upCallback
	h.pingCallback = pingCallback

	h.workerPoolSize = workers
	if workers == 0 {
		h.workerPoolSize = defaultWorkerPoolSize
	}

	h.sampleTriggerLimit = triggerLimit
	if triggerLimit == 0 {
		h.sampleTriggerLimit = defaultSampletTriggerLimit
	}

	h.checkTimeout = timeout
	if timeout == 0 {
		h.checkTimeout = defaultTimeout
	}

	log.Debug("[HOST CHECKER] Config:TriggerLimit: ", h.sampleTriggerLimit)
	log.Debug("[HOST CHECKER] Config:Timeout: ~", h.checkTimeout)
	log.Debug("[HOST CHECKER] Config:WorkerPool: ", h.workerPoolSize)

	var err error
	h.pool, err = tunny.CreatePool(h.workerPoolSize, func(hostData interface{}) interface{} {
		input, _ := hostData.(HostData)
		h.CheckHost(input)
		return nil
	}).Open()

	log.Debug("[HOST CHECKER] Init complete")

	if err != nil {
		log.Errorf("[HOST CHECKER POOL] Error: %v\n", err)
	}
}

func (h *HostUptimeChecker) Start() {
	// Start the loop that checks for bum hosts
	h.stopLoop = false
	log.Debug("[HOST CHECKER] Starting...")
	go h.HostCheckLoop()
	log.Debug("[HOST CHECKER] Check loop started...")
	go h.HostReporter()
	log.Debug("[HOST CHECKER] Host reporter started...")
}

func (h *HostUptimeChecker) Stop() {
	h.stopLoop = true
	h.stopPollingChan <- true
	log.Info("[HOST CHECKER] Stopping poller")
	h.pool.Close()
}

func (h *HostUptimeChecker) ResetList(hostList map[string]HostData) {
	h.resetListMu.Lock()
	h.doResetList = true
	h.newList = hostList
	h.resetListMu.Unlock()
	log.Debug("[HOST CHECKER] Checker reset queued!")
}
<commit_msg>bug: uptime tests respect proxy_ssl_insecure_skip_verify<commit_after>package main

import (
	"crypto/tls"
	"math/rand"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/jeffail/tunny"
	cache "github.com/pmylund/go-cache"

	"github.com/TykTechnologies/tyk/config"
)

const (
	defaultTimeout             = 10
	defaultWorkerPoolSize      = 50
	defaultSampletTriggerLimit = 3
)

var (
	HostCheckerClient = &http.Client{
		Timeout: 500 * time.Millisecond,
	}

	hostCheckTicker = make(chan struct{})
)

type HostData struct {
	CheckURL string
	Method   string
	Headers  map[string]string
	Body     string
	MetaData map[string]string
}

type HostHealthReport struct {
	HostData
	ResponseCode int
	Latency      float64
	IsTCPError   bool
}

type HostUptimeChecker struct {
	failureCallback    func(HostHealthReport)
	upCallback         func(HostHealthReport)
	pingCallback       func(HostHealthReport)
	workerPoolSize     int
	sampleTriggerLimit int
	checkTimeout       int
	HostList           map[string]HostData
	unHealthyList      map[string]bool
	pool               *tunny.WorkPool

	errorChan       chan HostHealthReport
	okChan          chan HostHealthReport
	stopPollingChan chan bool
	sampleCache     *cache.Cache
	stopLoop        bool

	resetListMu sync.Mutex
	doResetList bool
	newList     map[string]HostData
}

func (h *HostUptimeChecker) getStaggeredTime() time.Duration {
	if h.checkTimeout <= 5 {
		return time.Duration(h.checkTimeout) * time.Second
	}

	rand.Seed(time.Now().Unix())
	min := h.checkTimeout - 3
	max := h.checkTimeout + 3

	dur := rand.Intn(max-min) + min

	return time.Duration(dur) * time.Second
}

func (h *HostUptimeChecker) HostCheckLoop() {
	for !h.stopLoop {
		if runningTests {
			<-hostCheckTicker
		}
		h.resetListMu.Lock()
		if h.doResetList && h.newList != nil {
			h.HostList = h.newList
			h.newList = nil
			h.doResetList = false
			log.Debug("[HOST CHECKER] Host list reset")
		}
		h.resetListMu.Unlock()
		for _, host := range h.HostList {
			_, err := h.pool.SendWork(host)
			if err != nil {
				log.Errorf("[HOST CHECKER] could not send work, error: %v", err)
			}
		}

		if !runningTests {
			time.Sleep(h.getStaggeredTime())
		}
	}
	log.Info("[HOST CHECKER] Checker stopped")
}

func (h *HostUptimeChecker) HostReporter() {
	for {
		select {
		case okHost := <-h.okChan:
			// Clear host from unhealthylist if it exists
			if h.unHealthyList[okHost.CheckURL] {
				h.upCallback(okHost)
				delete(h.unHealthyList, okHost.CheckURL)
			}
			go h.pingCallback(okHost)

		case failedHost := <-h.errorChan:
			newVal := 1
			if count, found := h.sampleCache.Get(failedHost.CheckURL); found {
				newVal = count.(int) + 1
			}

			h.sampleCache.Set(failedHost.CheckURL, newVal, cache.DefaultExpiration)

			if newVal >= h.sampleTriggerLimit {
				log.Debug("[HOST CHECKER] [HOST WARNING]: ", failedHost.CheckURL)
				// Reset the count
				h.sampleCache.Set(failedHost.CheckURL, 1, cache.DefaultExpiration)
				// track it
				h.unHealthyList[failedHost.CheckURL] = true
				// Call the custom callback hook
				go h.failureCallback(failedHost)
			}
			go h.pingCallback(failedHost)

		case <-h.stopPollingChan:
			log.Debug("[HOST CHECKER] Received kill signal")
			return
		}
	}
}

func (h *HostUptimeChecker) CheckHost(toCheck HostData) {
	log.Debug("[HOST CHECKER] Checking: ", toCheck.CheckURL)

	t1 := time.Now()

	useMethod := toCheck.Method
	if toCheck.Method == "" {
		useMethod = http.MethodGet
	}

	req, err := http.NewRequest(useMethod, toCheck.CheckURL, strings.NewReader(toCheck.Body))
	if err != nil {
		log.Error("Could not create request: ", err)
		return
	}
	for headerName, headerValue := range toCheck.Headers {
		req.Header.Set(headerName, headerValue)
	}
	req.Header.Set("Connection", "close")

	HostCheckerClient.Transport = &http.Transport{
		TLSClientConfig: &tls.Config{
			InsecureSkipVerify: config.Global.ProxySSLInsecureSkipVerify,
		},
	}

	response, err := HostCheckerClient.Do(req)

	t2 := time.Now()

	millisec := float64(t2.UnixNano()-t1.UnixNano()) * 0.000001

	report := HostHealthReport{
		HostData: toCheck,
		Latency:  millisec,
	}

	if err != nil {
		report.IsTCPError = true
		h.errorChan <- report
		return
	}

	report.ResponseCode = response.StatusCode

	if response.StatusCode != http.StatusOK {
		h.errorChan <- report
		return
	}

	// host is healthy, report it
	h.okChan <- report
}

func (h *HostUptimeChecker) Init(workers, triggerLimit, timeout int, hostList map[string]HostData, failureCallback func(HostHealthReport), upCallback func(HostHealthReport), pingCallback func(HostHealthReport)) {
	h.sampleCache = cache.New(30*time.Second, 5*time.Second)
	h.stopPollingChan = make(chan bool)
	h.errorChan = make(chan HostHealthReport)
	h.okChan = make(chan HostHealthReport)
	h.HostList = hostList
	h.unHealthyList = make(map[string]bool)
	h.failureCallback = failureCallback
	h.upCallback = upCallback
	h.pingCallback = pingCallback

	h.workerPoolSize = workers
	if workers == 0 {
		h.workerPoolSize = defaultWorkerPoolSize
	}

	h.sampleTriggerLimit = triggerLimit
	if triggerLimit == 0 {
		h.sampleTriggerLimit = defaultSampletTriggerLimit
	}

	h.checkTimeout = timeout
	if timeout == 0 {
		h.checkTimeout = defaultTimeout
	}

	log.Debug("[HOST CHECKER] Config:TriggerLimit: ", h.sampleTriggerLimit)
	log.Debug("[HOST CHECKER] Config:Timeout: ~", h.checkTimeout)
	log.Debug("[HOST CHECKER] Config:WorkerPool: ", h.workerPoolSize)

	var err error
	h.pool, err = tunny.CreatePool(h.workerPoolSize, func(hostData interface{}) interface{} {
		input, _ := hostData.(HostData)
		h.CheckHost(input)
		return nil
	}).Open()

	log.Debug("[HOST CHECKER] Init complete")

	if err != nil {
		log.Errorf("[HOST CHECKER POOL] Error: %v\n", err)
	}
}

func (h *HostUptimeChecker) Start() {
	// Start the loop that checks for bum hosts
	h.stopLoop = false
	log.Debug("[HOST CHECKER] Starting...")
	go h.HostCheckLoop()
	log.Debug("[HOST CHECKER] Check loop started...")
	go h.HostReporter()
	log.Debug("[HOST CHECKER] Host reporter started...")
}

func (h *HostUptimeChecker) Stop() {
	h.stopLoop = true
	h.stopPollingChan <- true
	log.Info("[HOST CHECKER] Stopping poller")
	h.pool.Close()
}

func (h *HostUptimeChecker) ResetList(hostList map[string]HostData) {
	h.resetListMu.Lock()
	h.doResetList = true
	h.newList = hostList
	h.resetListMu.Unlock()
	log.Debug("[HOST CHECKER] Checker reset queued!")
}
<|endoftext|>