<commit_before>package gleam

import (
	"crypto/tls"
	"log"
	"os"
	"syscall"
	"time"

	mqtt "github.com/eclipse/paho.mqtt.golang"
	"github.com/mikespook/golib/signal"
	"github.com/mikespook/schego"
)

type Gleam struct {
	lua        *luaEnv
	config     Config
	mqttClient mqtt.Client
	scheduler  *schego.Scheduler
}

func NewGleam(workdir string) *Gleam {
	return &Gleam{
		lua: newLuaEnv(workdir),
	}
}

func (g *Gleam) Init() error {
	if err := g.lua.Init(&g.config); err != nil {
		return err
	}

	g.initMQTT()
	g.initSchedule(&g.config)

	return g.lua.onEvent(AfterInitFunc, g.mqttClient)
}

func (g *Gleam) initSchedule(config *Config) {
	g.scheduler = schego.New(config.Schedule.Tick * time.Millisecond)
	g.scheduler.ErrorFunc = g.lua.onError
	for name, interval := range config.Schedule.Tasks {
		f := g.lua.newOnSchedule(name, &g.mqttClient)
		g.scheduler.Add(name, time.Now(), interval*time.Millisecond, schego.ForEver, f)
		fn := "Skip"
		if f != nil {
			fn = "Serve"
		}
		log.Printf("Schedule: %s (%d) => %s", name, interval, fn)
	}
}

func (g *Gleam) initMQTT() {
	opts := mqtt.NewClientOptions()
	for _, broker := range g.config.MQTT {
		opts.AddBroker(broker.Addr)
		log.Printf("Add Broker: %s@%s", broker.Username, broker.Addr)
		if broker.Username != "" {
			opts.SetUsername(broker.Username).SetPassword(broker.Password)
		}
	}
	opts.SetClientID(g.config.ClientId)
	log.Printf("ClientId: %s", g.config.ClientId)
	opts.SetDefaultPublishHandler(g.lua.newOnMessage(MessageFunc))
	opts.SetAutoReconnect(true)
	if g.config.NotVerifyTLS {
		opts.SetTLSConfig(&tls.Config{InsecureSkipVerify: true})
	}
	opts.SetOnConnectHandler(g.newOnConnect())
	opts.SetConnectionLostHandler(g.newOnLostConnect())
	g.mqttClient = mqtt.NewClient(opts)
	for {
		token := g.mqttClient.Connect()
		if !token.Wait() || token.Error() == nil {
			break
		}
		log.Printf("Conn Error: %s", token.Error())
		time.Sleep(5 * time.Second)
	}
}

func (g *Gleam) newOnConnect() mqtt.OnConnectHandler {
	return func(client mqtt.Client) {
		for name, task := range g.config.Tasks {
			for topic, qos := range task {
				f := g.lua.newOnMessage(name)
				if f == nil {
					name = "{default}"
				}
				log.Printf("Subscribe: %s (%d) => %s", topic, qos, name)
				if token := client.Subscribe(topic, qos, f); token.Wait() && token.Error() != nil {
					log.Printf("Subscribe Error: %s %s", topic, token.Error())
				}
			}
		}
	}
}

func (g *Gleam) newOnLostConnect() mqtt.ConnectionLostHandler {
	return func(client mqtt.Client, err error) {
		log.Printf("Conn Error: %s", err)
	}
}

func (g *Gleam) Serve() {
	go g.scheduler.Serve()

	sh := signal.New(nil)
	sh.Bind(os.Interrupt, func() uint {
		return signal.BreakExit
	})
	sh.Bind(syscall.SIGTERM, func() uint {
		return signal.BreakExit
	})
	sh.Wait()
}

func (g *Gleam) Final() error {
	if err := g.scheduler.Close(); err != nil {
		return err
	}

	for topic, _ := range g.config.Tasks {
		if token := g.mqttClient.Unsubscribe(topic); token.Wait() && token.Error() != nil {
			log.Printf("Unsubscribe error: %s => [%s]", topic, token.Error())
		}
		log.Printf("Unsubscribe: %s", topic)
	}

	if err := g.lua.onEvent(BeforeFinalizeFunc, g.mqttClient); err != nil {
		log.Printf("BeforeFinalize: %s", err)
	}
	if g.config.FinalTick != 0 {
		time.Sleep(g.config.FinalTick * time.Millisecond)
	}
	g.mqttClient.Disconnect(500)
	g.lua.Final()
	return nil
}
<commit_msg>fixed nil object method call<commit_after>package gleam

import (
	"crypto/tls"
	"log"
	"os"
	"syscall"
	"time"

	mqtt "github.com/eclipse/paho.mqtt.golang"
	"github.com/mikespook/golib/signal"
	"github.com/mikespook/schego"
)

type Gleam struct {
	lua        *luaEnv
	config     Config
	mqttClient mqtt.Client
	scheduler  *schego.Scheduler
}

func NewGleam(workdir string) *Gleam {
	return &Gleam{
		lua: newLuaEnv(workdir),
	}
}

func (g *Gleam) Init() error {
	if err := g.lua.Init(&g.config); err != nil {
		return err
	}

	g.initMQTT()
	g.initSchedule()

	return g.lua.onEvent(AfterInitFunc, g.mqttClient)
}

func (g *Gleam) initSchedule() {
	if g.config.Schedule.Tick == 0 {
		log.Printf("Scheduler not init")
		return
	}
	g.scheduler = schego.New(g.config.Schedule.Tick * time.Millisecond)
	g.scheduler.ErrorFunc = g.lua.onError
	for name, interval := range g.config.Schedule.Tasks {
		f := g.lua.newOnSchedule(name, &g.mqttClient)
		g.scheduler.Add(name, time.Now(), interval*time.Millisecond, schego.ForEver, f)
		fn := "Skip"
		if f != nil {
			fn = "Serve"
		}
		log.Printf("Schedule: %s (%d) => %s", name, interval, fn)
	}
}

func (g *Gleam) initMQTT() {
	if g.config.ClientId == "" {
		log.Printf("MQTT not init")
		return
	}

	opts := mqtt.NewClientOptions()
	for _, broker := range g.config.MQTT {
		opts.AddBroker(broker.Addr)
		log.Printf("Add Broker: %s@%s", broker.Username, broker.Addr)
		if broker.Username != "" {
			opts.SetUsername(broker.Username).SetPassword(broker.Password)
		}
	}
	opts.SetClientID(g.config.ClientId)
	log.Printf("ClientId: %s", g.config.ClientId)
	opts.SetDefaultPublishHandler(g.lua.newOnMessage(MessageFunc))
	opts.SetAutoReconnect(true)
	if g.config.NotVerifyTLS {
		opts.SetTLSConfig(&tls.Config{InsecureSkipVerify: true})
	}
	opts.SetOnConnectHandler(g.newOnConnect())
	opts.SetConnectionLostHandler(g.newOnLostConnect())
	g.mqttClient = mqtt.NewClient(opts)
	for {
		token := g.mqttClient.Connect()
		if !token.Wait() || token.Error() == nil {
			break
		}
		log.Printf("Conn Error: %s", token.Error())
		time.Sleep(5 * time.Second)
	}
}

func (g *Gleam) newOnConnect() mqtt.OnConnectHandler {
	return func(client mqtt.Client) {
		for name, task := range g.config.Tasks {
			for topic, qos := range task {
				f := g.lua.newOnMessage(name)
				if f == nil {
					name = "{default}"
				}
				log.Printf("Subscribe: %s (%d) => %s", topic, qos, name)
				if token := client.Subscribe(topic, qos, f); token.Wait() && token.Error() != nil {
					log.Printf("Subscribe Error: %s %s", topic, token.Error())
				}
			}
		}
	}
}

func (g *Gleam) newOnLostConnect() mqtt.ConnectionLostHandler {
	return func(client mqtt.Client, err error) {
		log.Printf("Conn Error: %s", err)
	}
}

func (g *Gleam) Serve() {
	if g.scheduler != nil {
		go g.scheduler.Serve()
	}

	sh := signal.New(nil)
	sh.Bind(os.Interrupt, func() uint {
		return signal.BreakExit
	})
	sh.Bind(syscall.SIGTERM, func() uint {
		return signal.BreakExit
	})
	sh.Wait()
}

func (g *Gleam) Final() error {
	if g.scheduler != nil {
		if err := g.scheduler.Close(); err != nil {
			return err
		}
	}

	for topic, _ := range g.config.Tasks {
		if token := g.mqttClient.Unsubscribe(topic); token.Wait() && token.Error() != nil {
			log.Printf("Unsubscribe error: %s => [%s]", topic, token.Error())
		}
		log.Printf("Unsubscribe: %s", topic)
	}

	if err := g.lua.onEvent(BeforeFinalizeFunc, g.mqttClient); err != nil {
		log.Printf("BeforeFinalize: %s", err)
	}
	if g.config.FinalTick != 0 {
		time.Sleep(g.config.FinalTick * time.Millisecond)
	}
	if g.mqttClient != nil {
		g.mqttClient.Disconnect(500)
	}
	g.lua.Final()
	return nil
}
<|endoftext|>


<commit_before>package mongo

import (
	l "logger"

	"errors"
	"strconv"
	"time"

	"github.com/Sirupsen/logrus"
	"labix.org/v2/mgo/bson"
)

func PostLoginPageData(loginInfo *PostLogin) (*LoginStatus, error) {

	l.PutInfo(l.I_M_PostPage, loginInfo, nil)

	_, ok := users[loginInfo.UserId]
	if ok {
		var teamName string
		for _, team := range teams {
			for _, userid := range team.UserIds {
				if userid == loginInfo.UserId {
					teamName = team.Name
				}
			}
		}

		l.Output(
			logrus.Fields{
				"User ID":   loginInfo.UserId,
				"User Name": users[loginInfo.UserId].Name,
			},
			"login success",
			l.Debug,
		)
		loginStatus := &LoginStatus{
			Status:   "success",
			UserId:   loginInfo.UserId,
			UserName: users[loginInfo.UserId].Name,
			Team:     teamName,
			Admin:    players[loginInfo.UserId].Admin,
		}
		return loginStatus, nil
	} else {
		l.Output(
			logrus.Fields{
				"User ID": loginInfo.UserId,
			},
			"login faild",
			l.Debug,
		)
		return &LoginStatus{Status: "failed"}, nil
	}
}

func PostApplyScoreData(teamName string, ApplyScore *PostApplyScore) (*Status, error) {
	l.PutInfo(l.I_M_PostPage, teamName, ApplyScore)

	//更新情報をGlobal変数に格納する
	defer SetPlayerCol(ApplyScore.UserIds)

	AUserIdInTheTeam := teams[teamName].UserIds[0]
	if players[AUserIdInTheTeam].Apply != 0 {
		l.Output(
			logrus.Fields{
				"User Apply": players[AUserIdInTheTeam].Apply,
			},
			"Apply score is already registered",
			l.Debug,
		)
		return &Status{"already registered"}, nil
	}
	for playerIndex, userId := range ApplyScore.UserIds {

		findQuery := bson.M{"userid": userId}
		setQuery := bson.M{"$set": bson.M{"apply": ApplyScore.Apply[playerIndex]}}
		if err = UpdateMongoData("player", findQuery, setQuery); err != nil {
			l.PutErr(err, l.Trace(), l.E_M_Update, ApplyScore.Apply[playerIndex])
			return &Status{"failed"}, err
		}
	}

	return &Status{"success"}, nil
}

func PostScoreViewSheetPageData(teamName string, definedTeam *PostDefinedTeam) (*Status, error) {
	l.PutInfo(l.I_M_PostPage, teamName, definedTeam)

	//更新情報をGlobal変数に格納する
	defer SetTeamCol(teamName)

	findQuery := bson.M{"name": teamName}
	setQuery := bson.M{"$set": bson.M{"defined": true}}
	if err = UpdateMongoData("team", findQuery, setQuery); err != nil {
		l.PutErr(err, l.Trace(), l.E_M_Update, teamName)
		return &Status{"failed"}, err
	}

	return &Status{"success"}, nil
}

func PostScoreEntrySheetPageData(teamName string, holeString string, teamScore *PostTeamScore) (*RequestTakePictureStatus, error) {
	l.PutInfo(l.I_M_PostPage, teamName, teamScore)

	userIds := teams[teamName].UserIds
	//更新情報をGlobal変数に格納する
	defer SetPlayerCol(userIds)

	if len(holeString) == 0 {
		l.PutErr(nil, l.Trace(), l.E_Nil, teamName)
		return &RequestTakePictureStatus{Status: "failed"}, errors.New("hole is not string")
	}

	holeNum, _ := strconv.Atoi(holeString)
	holeIndex := holeNum - 1
	holeIndexString := strconv.Itoa(holeIndex)

	if teamScore.Excnt != excnt[teamName][holeNum] {
		return &RequestTakePictureStatus{Status: "other updated"}, nil
	} else {
		excnt[teamName][holeNum]++
	}

	for playerIndex, userId := range teamScore.UserIds {
		total, putt := teamScore.Total[playerIndex], teamScore.Putt[playerIndex]

		findQuery := bson.M{"userid": userId}
		setQuery := bson.M{
			"$set": bson.M{
				"score." + holeIndexString + ".total": total,
				"score." + holeIndexString + ".putt":  putt,
			},
		}
		if err = UpdateMongoData("player", findQuery, setQuery); err != nil {
			l.PutErr(err, l.Trace(), l.E_M_Update, userId)
			return &RequestTakePictureStatus{Status: "failed update score"}, err
		}
	}
	//	Thread登録
	if err := RegisterThreadOfScore(holeString, teamScore); err != nil {
		l.PutErr(err, l.Trace(), l.E_M_RegisterThread, teamScore)
		return nil, err
	}

	//	チーム内に写真リクエストがあるか確認する
	requestTakePictureStatus, err := RequestTakePicture(userIds)
	if err != nil {
		l.PutErr(err, l.Trace(), l.E_M_SearchPhotoTask, userIds)
		return nil, err
	}

	return requestTakePictureStatus, nil
}

func UpsertNewTimeLine(thread *Thread) error {
	l.PutInfo(l.I_M_PostPage, thread, nil)

	//更新情報をGlobal変数に格納する
	defer SetAllThreadCol()

	defaultColor := "#c0c0c0"

	if len(thread.ThreadId) != 0 {
		l.PutErr(nil, l.Trace(), l.E_WrongData, thread)
		return errors.New("thread id exists")
	}

	db, session := mongoInit()
	threadCol := db.C("thread")
	defer session.Close()

	thread.ThreadId = make20lengthHashString()
	thread.CreatedAt = time.Now().Format(datetimeFormat)
	thread.ColorCode = defaultColor
	if err = threadCol.Insert(thread); err != nil {
		l.PutErr(err, l.Trace(), l.E_M_Insert, thread)
		return err
	}

	return nil
}

func UpdateExistingTimeLine(thread *Thread) (*Thread, error) {
	l.PutInfo(l.I_M_PostPage, thread, nil)

	targetThreadId := thread.ThreadId
	//更新情報をGlobal変数に格納する
	defer SetAllThreadCol()

	colorFeeling := make(map[string]string)
	colorFeeling["default"] = "#c0c0c0"
	colorFeeling["angry"] = "#ff7f7f"
	colorFeeling["great"] = "#ffff7f"
	colorFeeling["sad"] = "#7fbfff"
	colorFeeling["vexing"] = "#7fff7f"

	if len(targetThreadId) == 0 {
		l.PutErr(nil, l.Trace(), l.E_Nil, thread)
		return nil, errors.New("thread id do not exist")
	}

	if len(thread.ColorCode) == 0 {
		l.PutErr(err, l.Trace(), l.E_Nil, thread.ColorCode)
		return nil, errors.New("current colorCode do not contain in posted thread")
	}
	if len(thread.Reactions) > 1 {
		l.PutErr(err, l.Trace(), l.E_TooManyData, thread.Reactions)
		return nil, errors.New("reactions is not 1")
	}

	currentFeeling := ""
	currentColor := threads[targetThreadId].ColorCode
	postedFeeling := getFeelingFromAWSUrl(thread.Reactions[0].Content)
	postedColor := colorFeeling[postedFeeling]

	for feeling, code := range colorFeeling {
		if currentColor == code {
			currentFeeling = feeling
		}
	}

	thread.Reactions[0].DateTime = time.Now().Format(datetimeFormat)
	findQuery := bson.M{"threadid": targetThreadId}
	pushQuery := bson.M{"$push": bson.M{"reactions": thread.Reactions[0]}}
	if err = UpdateMongoData("thread", findQuery, pushQuery); err != nil {
		l.PutErr(err, l.Trace(), l.E_M_Update, thread.Reactions[0])
		return nil, err
	}

	//投稿された感情と、現在の感情に相違がある場合
	if currentFeeling != postedFeeling {
		var setColor string
		var currentFeelingCount, postedFeelingCount int

		if currentColor == colorFeeling["default"] {
			setColor = postedColor
		} else {

			//直前の更新を反映させる
			SetAllThreadCol()

			for _, r := range threads[targetThreadId].Reactions {
				switch getFeelingFromAWSUrl(r["content"].(string)) {
				case currentFeeling:
					currentFeelingCount++
				case postedFeeling:
					postedFeelingCount++
				}
			}

			if currentFeelingCount >= postedFeelingCount {
				setColor = currentColor
			} else {
				setColor = postedColor
			}
		}

		setQuery := bson.M{"$set": bson.M{"colorcode": setColor}}

		if err = UpdateMongoData("thread", findQuery, setQuery); err != nil {
			l.PutErr(err, l.Trace(), l.E_M_Update, setColor)
			return nil, err
		}
		thread.ColorCode = setColor
	}
	return thread, nil
}
<commit_msg>new thread push<commit_after>package mongo

import (
	l "logger"

	"errors"
	"strconv"
	"time"

	"github.com/Sirupsen/logrus"
	"labix.org/v2/mgo/bson"
)

func PostLoginPageData(loginInfo *PostLogin) (*LoginStatus, error) {

	l.PutInfo(l.I_M_PostPage, loginInfo, nil)

	_, ok := users[loginInfo.UserId]
	if ok {
		var teamName string
		for _, team := range teams {
			for _, userid := range team.UserIds {
				if userid == loginInfo.UserId {
					teamName = team.Name
				}
			}
		}

		l.Output(
			logrus.Fields{
				"User ID":   loginInfo.UserId,
				"User Name": users[loginInfo.UserId].Name,
			},
			"login success",
			l.Debug,
		)
		loginStatus := &LoginStatus{
			Status:   "success",
			UserId:   loginInfo.UserId,
			UserName: users[loginInfo.UserId].Name,
			Team:     teamName,
			Admin:    players[loginInfo.UserId].Admin,
		}
		return loginStatus, nil
	} else {
		l.Output(
			logrus.Fields{
				"User ID": loginInfo.UserId,
			},
			"login faild",
			l.Debug,
		)
		return &LoginStatus{Status: "failed"}, nil
	}
}

func PostApplyScoreData(teamName string, ApplyScore *PostApplyScore) (*Status, error) {
	l.PutInfo(l.I_M_PostPage, teamName, ApplyScore)

	//更新情報をGlobal変数に格納する
	defer SetPlayerCol(ApplyScore.UserIds)

	AUserIdInTheTeam := teams[teamName].UserIds[0]
	if players[AUserIdInTheTeam].Apply != 0 {
		l.Output(
			logrus.Fields{
				"User Apply": players[AUserIdInTheTeam].Apply,
			},
			"Apply score is already registered",
			l.Debug,
		)
		return &Status{"already registered"}, nil
	}
	for playerIndex, userId := range ApplyScore.UserIds {

		findQuery := bson.M{"userid": userId}
		setQuery := bson.M{"$set": bson.M{"apply": ApplyScore.Apply[playerIndex]}}
		if err = UpdateMongoData("player", findQuery, setQuery); err != nil {
			l.PutErr(err, l.Trace(), l.E_M_Update, ApplyScore.Apply[playerIndex])
			return &Status{"failed"}, err
		}
	}

	return &Status{"success"}, nil
}

func PostScoreViewSheetPageData(teamName string, definedTeam *PostDefinedTeam) (*Status, error) {
	l.PutInfo(l.I_M_PostPage, teamName, definedTeam)

	//更新情報をGlobal変数に格納する
	defer SetTeamCol(teamName)

	findQuery := bson.M{"name": teamName}
	setQuery := bson.M{"$set": bson.M{"defined": true}}
	if err = UpdateMongoData("team", findQuery, setQuery); err != nil {
		l.PutErr(err, l.Trace(), l.E_M_Update, teamName)
		return &Status{"failed"}, err
	}

	return &Status{"success"}, nil
}

func PostScoreEntrySheetPageData(teamName string, holeString string, teamScore *PostTeamScore) (*RequestTakePictureStatus, error) {
	l.PutInfo(l.I_M_PostPage, teamName, teamScore)

	userIds := teams[teamName].UserIds
	//更新情報をGlobal変数に格納する
	defer SetPlayerCol(userIds)

	if len(holeString) == 0 {
		l.PutErr(nil, l.Trace(), l.E_Nil, teamName)
		return &RequestTakePictureStatus{Status: "failed"}, errors.New("hole is not string")
	}

	holeNum, _ := strconv.Atoi(holeString)
	holeIndex := holeNum - 1
	holeIndexString := strconv.Itoa(holeIndex)

	if teamScore.Excnt != excnt[teamName][holeNum] {
		return &RequestTakePictureStatus{Status: "other updated"}, nil
	} else {
		excnt[teamName][holeNum]++
	}

	for playerIndex, userId := range teamScore.UserIds {
		total, putt := teamScore.Total[playerIndex], teamScore.Putt[playerIndex]

		findQuery := bson.M{"userid": userId}
		setQuery := bson.M{
			"$set": bson.M{
				"score." + holeIndexString + ".total": total,
				"score." + holeIndexString + ".putt":  putt,
			},
		}
		if err = UpdateMongoData("player", findQuery, setQuery); err != nil {
			l.PutErr(err, l.Trace(), l.E_M_Update, userId)
			return &RequestTakePictureStatus{Status: "failed update score"}, err
		}
	}
	//	Thread登録
	if err := RegisterThreadOfScore(holeString, teamScore); err != nil {
		l.PutErr(err, l.Trace(), l.E_M_RegisterThread, teamScore)
		return nil, err
	}

	//	チーム内に写真リクエストがあるか確認する
	requestTakePictureStatus, err := RequestTakePicture(userIds)
	if err != nil {
		l.PutErr(err, l.Trace(), l.E_M_SearchPhotoTask, userIds)
		return nil, err
	}

	return requestTakePictureStatus, nil
}

func UpsertNewTimeLine(thread *Thread) error {
	l.PutInfo(l.I_M_PostPage, thread, nil)

	//更新情報をGlobal変数に格納する
	defer SetAllThreadCol()

	defaultColor := "#c0c0c0"

	if len(thread.ThreadId) != 0 {
		l.PutErr(nil, l.Trace(), l.E_WrongData, thread)
		return errors.New("thread id exists")
	}

	db, session := mongoInit()
	threadCol := db.C("thread")
	defer session.Close()

	thread.ThreadId = make20lengthHashString()
	thread.CreatedAt = time.Now().Format(datetimeFormat)
	thread.ColorCode = defaultColor
	if err = threadCol.Insert(thread); err != nil {
		l.PutErr(err, l.Trace(), l.E_M_Insert, thread)
		return err
	}

	ThreadChan <- thread

	return nil
}

func UpdateExistingTimeLine(thread *Thread) (*Thread, error) {
	l.PutInfo(l.I_M_PostPage, thread, nil)

	targetThreadId := thread.ThreadId
	//更新情報をGlobal変数に格納する
	defer SetAllThreadCol()

	colorFeeling := make(map[string]string)
	colorFeeling["default"] = "#c0c0c0"
	colorFeeling["angry"] = "#ff7f7f"
	colorFeeling["great"] = "#ffff7f"
	colorFeeling["sad"] = "#7fbfff"
	colorFeeling["vexing"] = "#7fff7f"

	if len(targetThreadId) == 0 {
		l.PutErr(nil, l.Trace(), l.E_Nil, thread)
		return nil, errors.New("thread id do not exist")
	}

	if len(thread.ColorCode) == 0 {
		l.PutErr(err, l.Trace(), l.E_Nil, thread.ColorCode)
		return nil, errors.New("current colorCode do not contain in posted thread")
	}
	if len(thread.Reactions) > 1 {
		l.PutErr(err, l.Trace(), l.E_TooManyData, thread.Reactions)
		return nil, errors.New("reactions is not 1")
	}

	currentFeeling := ""
	currentColor := threads[targetThreadId].ColorCode
	postedFeeling := getFeelingFromAWSUrl(thread.Reactions[0].Content)
	postedColor := colorFeeling[postedFeeling]

	for feeling, code := range colorFeeling {
		if currentColor == code {
			currentFeeling = feeling
		}
	}

	thread.Reactions[0].DateTime = time.Now().Format(datetimeFormat)
	findQuery := bson.M{"threadid": targetThreadId}
	pushQuery := bson.M{"$push": bson.M{"reactions": thread.Reactions[0]}}
	if err = UpdateMongoData("thread", findQuery, pushQuery); err != nil {
		l.PutErr(err, l.Trace(), l.E_M_Update, thread.Reactions[0])
		return nil, err
	}

	//投稿された感情と、現在の感情に相違がある場合
	if currentFeeling != postedFeeling {
		var setColor string
		var currentFeelingCount, postedFeelingCount int

		if currentColor == colorFeeling["default"] {
			setColor = postedColor
		} else {

			//直前の更新を反映させる
			SetAllThreadCol()

			for _, r := range threads[targetThreadId].Reactions {
				switch getFeelingFromAWSUrl(r["content"].(string)) {
				case currentFeeling:
					currentFeelingCount++
				case postedFeeling:
					postedFeelingCount++
				}
			}

			if currentFeelingCount >= postedFeelingCount {
				setColor = currentColor
			} else {
				setColor = postedColor
			}
		}

		setQuery := bson.M{"$set": bson.M{"colorcode": setColor}}

		if err = UpdateMongoData("thread", findQuery, setQuery); err != nil {
			l.PutErr(err, l.Trace(), l.E_M_Update, setColor)
			return nil, err
		}
		thread.ColorCode = setColor
	}
	return thread, nil
}
<|endoftext|>


<commit_before>package metrics

import (
	"fmt"
	"math"
	"strings"
	"sync"
	"time"
)

// InmemSink provides a MetricSink that does in-memory aggregation
// without sending metrics over a network. It can be embedded within
// an application to provide profiling information.
type InmemSink struct {
	// How long is each aggregation interval
	interval time.Duration

	// Retain controls how many metrics interval we keep
	retain time.Duration

	// maxIntervals is the maximum length of intervals.
	// It is retain / interval.
	maxIntervals int

	// intervals is a slice of the retained intervals
	intervals    []*IntervalMetrics
	intervalLock sync.RWMutex

	rateDenom float64
}

// IntervalMetrics stores the aggregated metrics
// for a specific interval
type IntervalMetrics struct {
	sync.RWMutex

	// The start time of the interval
	Interval time.Time
r
	// Gauges maps the key to the last set value
	Gauges map[string]float32

	// Points maps the string to the list of emitted values
	// from EmitKey
	Points map[string][]float32

	// Counters maps the string key to a sum of the counter
	// values
	Counters map[string]*AggregateSample

	// Samples maps the key to an AggregateSample,
	// which has the rolled up view of a sample
	Samples map[string]*AggregateSample
}

// NewIntervalMetrics creates a new IntervalMetrics for a given interval
func NewIntervalMetrics(intv time.Time) *IntervalMetrics {
	return &IntervalMetrics{
		Interval: intv,
		Gauges:   make(map[string]float32),
		Points:   make(map[string][]float32),
		Counters: make(map[string]*AggregateSample),
		Samples:  make(map[string]*AggregateSample),
	}
}

// AggregateSample is used to hold aggregate metrics
// about a sample
type AggregateSample struct {
	Count       int       // The count of emitted pairs
	Rate	        float64   // The count of emitted pairs per time unit (usually 1 second)
	Sum         float64   // The sum of values
	SumSq       float64   // The sum of squared values
	Min         float64   // Minimum value
	Max         float64   // Maximum value
	LastUpdated time.Time // When value was last updated
}

// Computes a Stddev of the values
func (a *AggregateSample) Stddev() float64 {
	num := (float64(a.Count) * a.SumSq) - math.Pow(a.Sum, 2)
	div := float64(a.Count * (a.Count - 1))
	if div == 0 {
		return 0
	}
	return math.Sqrt(num / div)
}

// Computes a mean of the values
func (a *AggregateSample) Mean() float64 {
	if a.Count == 0 {
		return 0
	}
	return a.Sum / float64(a.Count)
}

// Ingest is used to update a sample
func (a *AggregateSample) Ingest(v float64, rateDenom float64) {
	a.Count++
	a.Sum += v
	a.SumSq += (v * v)
	if v < a.Min || a.Count == 1 {
		a.Min = v
	}
	if v > a.Max || a.Count == 1 {
		a.Max = v
	}
	a.Rate = float64(a.Count)/rateDenom
	a.LastUpdated = time.Now()
}

func (a *AggregateSample) String() string {
	if a.Count == 0 {
		return "Count: 0"
	} else if a.Stddev() == 0 {
		return fmt.Sprintf("Count: %d Sum: %0.3f LastUpdated: %s", a.Count, a.Sum, a.LastUpdated)
	} else {
		return fmt.Sprintf("Count: %d Min: %0.3f Mean: %0.3f Max: %0.3f Stddev: %0.3f Sum: %0.3f LastUpdated: %s",
			a.Count, a.Min, a.Mean(), a.Max, a.Stddev(), a.Sum, a.LastUpdated)
	}
}

// NewInmemSink is used to construct a new in-memory sink.
// Uses an aggregation interval and maximum retention period.
func NewInmemSink(interval, retain time.Duration) *InmemSink {
	rateTimeUnit := time.Second
	i := &InmemSink{
		interval:     interval,
		retain:       retain,
		maxIntervals: int(retain / interval),
		rateDenom: float64(interval.Nanoseconds()) / float64(rateTimeUnit.Nanoseconds()),
	}
	i.intervals = make([]*IntervalMetrics, 0, i.maxIntervals)
	return i
}

func (i *InmemSink) SetGauge(key []string, val float32) {
	k := i.flattenKey(key)
	intv := i.getInterval()

	intv.Lock()
	defer intv.Unlock()
	intv.Gauges[k] = val
}

func (i *InmemSink) EmitKey(key []string, val float32) {
	k := i.flattenKey(key)
	intv := i.getInterval()

	intv.Lock()
	defer intv.Unlock()
	vals := intv.Points[k]
	intv.Points[k] = append(vals, val)
}

func (i *InmemSink) IncrCounter(key []string, val float32) {
	k := i.flattenKey(key)
	intv := i.getInterval()

	intv.Lock()
	defer intv.Unlock()

	agg := intv.Counters[k]
	if agg == nil {
		agg = &AggregateSample{}
		intv.Counters[k] = agg
	}
	agg.Ingest(float64(val), i.rateDenom)
}

func (i *InmemSink) AddSample(key []string, val float32) {
	k := i.flattenKey(key)
	intv := i.getInterval()

	intv.Lock()
	defer intv.Unlock()

	agg := intv.Samples[k]
	if agg == nil {
		agg = &AggregateSample{}
		intv.Samples[k] = agg
	}
	agg.Ingest(float64(val), i.rateDenom)
}

// Data is used to retrieve all the aggregated metrics
// Intervals may be in use, and a read lock should be acquired
func (i *InmemSink) Data() []*IntervalMetrics {
	// Get the current interval, forces creation
	i.getInterval()

	i.intervalLock.RLock()
	defer i.intervalLock.RUnlock()

	intervals := make([]*IntervalMetrics, len(i.intervals))
	copy(intervals, i.intervals)
	return intervals
}

func (i *InmemSink) getExistingInterval(intv time.Time) *IntervalMetrics {
	i.intervalLock.RLock()
	defer i.intervalLock.RUnlock()

	n := len(i.intervals)
	if n > 0 && i.intervals[n-1].Interval == intv {
		return i.intervals[n-1]
	}
	return nil
}

func (i *InmemSink) createInterval(intv time.Time) *IntervalMetrics {
	i.intervalLock.Lock()
	defer i.intervalLock.Unlock()

	// Check for an existing interval
	n := len(i.intervals)
	if n > 0 && i.intervals[n-1].Interval == intv {
		return i.intervals[n-1]
	}

	// Add the current interval
	current := NewIntervalMetrics(intv)
	i.intervals = append(i.intervals, current)
	n++

	// Truncate the intervals if they are too long
	if n >= i.maxIntervals {
		copy(i.intervals[0:], i.intervals[n-i.maxIntervals:])
		i.intervals = i.intervals[:i.maxIntervals]
	}
	return current
}

// getInterval returns the current interval to write to
func (i *InmemSink) getInterval() *IntervalMetrics {
	intv := time.Now().Truncate(i.interval)
	if m := i.getExistingInterval(intv); m != nil {
		return m
	}
	return i.createInterval(intv)
}

// Flattens the key for formatting, removes spaces
func (i *InmemSink) flattenKey(parts []string) string {
	joined := strings.Join(parts, ".")
	return strings.Replace(joined, " ", "_", -1)
}
<commit_msg>Fix stupid typo<commit_after>package metrics

import (
	"fmt"
	"math"
	"strings"
	"sync"
	"time"
)

// InmemSink provides a MetricSink that does in-memory aggregation
// without sending metrics over a network. It can be embedded within
// an application to provide profiling information.
type InmemSink struct {
	// How long is each aggregation interval
	interval time.Duration

	// Retain controls how many metrics interval we keep
	retain time.Duration

	// maxIntervals is the maximum length of intervals.
	// It is retain / interval.
	maxIntervals int

	// intervals is a slice of the retained intervals
	intervals    []*IntervalMetrics
	intervalLock sync.RWMutex

	rateDenom float64
}

// IntervalMetrics stores the aggregated metrics
// for a specific interval
type IntervalMetrics struct {
	sync.RWMutex

	// The start time of the interval
	Interval time.Time

	// Gauges maps the key to the last set value
	Gauges map[string]float32

	// Points maps the string to the list of emitted values
	// from EmitKey
	Points map[string][]float32

	// Counters maps the string key to a sum of the counter
	// values
	Counters map[string]*AggregateSample

	// Samples maps the key to an AggregateSample,
	// which has the rolled up view of a sample
	Samples map[string]*AggregateSample
}

// NewIntervalMetrics creates a new IntervalMetrics for a given interval
func NewIntervalMetrics(intv time.Time) *IntervalMetrics {
	return &IntervalMetrics{
		Interval: intv,
		Gauges:   make(map[string]float32),
		Points:   make(map[string][]float32),
		Counters: make(map[string]*AggregateSample),
		Samples:  make(map[string]*AggregateSample),
	}
}

// AggregateSample is used to hold aggregate metrics
// about a sample
type AggregateSample struct {
	Count       int       // The count of emitted pairs
	Rate	        float64   // The count of emitted pairs per time unit (usually 1 second)
	Sum         float64   // The sum of values
	SumSq       float64   // The sum of squared values
	Min         float64   // Minimum value
	Max         float64   // Maximum value
	LastUpdated time.Time // When value was last updated
}

// Computes a Stddev of the values
func (a *AggregateSample) Stddev() float64 {
	num := (float64(a.Count) * a.SumSq) - math.Pow(a.Sum, 2)
	div := float64(a.Count * (a.Count - 1))
	if div == 0 {
		return 0
	}
	return math.Sqrt(num / div)
}

// Computes a mean of the values
func (a *AggregateSample) Mean() float64 {
	if a.Count == 0 {
		return 0
	}
	return a.Sum / float64(a.Count)
}

// Ingest is used to update a sample
func (a *AggregateSample) Ingest(v float64, rateDenom float64) {
	a.Count++
	a.Sum += v
	a.SumSq += (v * v)
	if v < a.Min || a.Count == 1 {
		a.Min = v
	}
	if v > a.Max || a.Count == 1 {
		a.Max = v
	}
	a.Rate = float64(a.Count)/rateDenom
	a.LastUpdated = time.Now()
}

func (a *AggregateSample) String() string {
	if a.Count == 0 {
		return "Count: 0"
	} else if a.Stddev() == 0 {
		return fmt.Sprintf("Count: %d Sum: %0.3f LastUpdated: %s", a.Count, a.Sum, a.LastUpdated)
	} else {
		return fmt.Sprintf("Count: %d Min: %0.3f Mean: %0.3f Max: %0.3f Stddev: %0.3f Sum: %0.3f LastUpdated: %s",
			a.Count, a.Min, a.Mean(), a.Max, a.Stddev(), a.Sum, a.LastUpdated)
	}
}

// NewInmemSink is used to construct a new in-memory sink.
// Uses an aggregation interval and maximum retention period.
func NewInmemSink(interval, retain time.Duration) *InmemSink {
	rateTimeUnit := time.Second
	i := &InmemSink{
		interval:     interval,
		retain:       retain,
		maxIntervals: int(retain / interval),
		rateDenom: float64(interval.Nanoseconds()) / float64(rateTimeUnit.Nanoseconds()),
	}
	i.intervals = make([]*IntervalMetrics, 0, i.maxIntervals)
	return i
}

func (i *InmemSink) SetGauge(key []string, val float32) {
	k := i.flattenKey(key)
	intv := i.getInterval()

	intv.Lock()
	defer intv.Unlock()
	intv.Gauges[k] = val
}

func (i *InmemSink) EmitKey(key []string, val float32) {
	k := i.flattenKey(key)
	intv := i.getInterval()

	intv.Lock()
	defer intv.Unlock()
	vals := intv.Points[k]
	intv.Points[k] = append(vals, val)
}

func (i *InmemSink) IncrCounter(key []string, val float32) {
	k := i.flattenKey(key)
	intv := i.getInterval()

	intv.Lock()
	defer intv.Unlock()

	agg := intv.Counters[k]
	if agg == nil {
		agg = &AggregateSample{}
		intv.Counters[k] = agg
	}
	agg.Ingest(float64(val), i.rateDenom)
}

func (i *InmemSink) AddSample(key []string, val float32) {
	k := i.flattenKey(key)
	intv := i.getInterval()

	intv.Lock()
	defer intv.Unlock()

	agg := intv.Samples[k]
	if agg == nil {
		agg = &AggregateSample{}
		intv.Samples[k] = agg
	}
	agg.Ingest(float64(val), i.rateDenom)
}

// Data is used to retrieve all the aggregated metrics
// Intervals may be in use, and a read lock should be acquired
func (i *InmemSink) Data() []*IntervalMetrics {
	// Get the current interval, forces creation
	i.getInterval()

	i.intervalLock.RLock()
	defer i.intervalLock.RUnlock()

	intervals := make([]*IntervalMetrics, len(i.intervals))
	copy(intervals, i.intervals)
	return intervals
}

func (i *InmemSink) getExistingInterval(intv time.Time) *IntervalMetrics {
	i.intervalLock.RLock()
	defer i.intervalLock.RUnlock()

	n := len(i.intervals)
	if n > 0 && i.intervals[n-1].Interval == intv {
		return i.intervals[n-1]
	}
	return nil
}

func (i *InmemSink) createInterval(intv time.Time) *IntervalMetrics {
	i.intervalLock.Lock()
	defer i.intervalLock.Unlock()

	// Check for an existing interval
	n := len(i.intervals)
	if n > 0 && i.intervals[n-1].Interval == intv {
		return i.intervals[n-1]
	}

	// Add the current interval
	current := NewIntervalMetrics(intv)
	i.intervals = append(i.intervals, current)
	n++

	// Truncate the intervals if they are too long
	if n >= i.maxIntervals {
		copy(i.intervals[0:], i.intervals[n-i.maxIntervals:])
		i.intervals = i.intervals[:i.maxIntervals]
	}
	return current
}

// getInterval returns the current interval to write to
func (i *InmemSink) getInterval() *IntervalMetrics {
	intv := time.Now().Truncate(i.interval)
	if m := i.getExistingInterval(intv); m != nil {
		return m
	}
	return i.createInterval(intv)
}

// Flattens the key for formatting, removes spaces
func (i *InmemSink) flattenKey(parts []string) string {
	joined := strings.Join(parts, ".")
	return strings.Replace(joined, " ", "_", -1)
}
<|endoftext|>


<commit_before>
package main

import (
  "fmt"
  "strconv"
  "unicode"
  "utf8"
)

const (
  kSplit = iota         // proceed down out & out1
  kAltBegin             // begin of alt section, i.e. '('
  kAltEnd               // end of alt section, i.e. ')'
  kRune                 // if match rune, proceed down out
  kCall                 // if matcher passes, proceed down out
  kMatch                // success state!
)

/**
 * Single instruction in regexp.
 */
type instr struct {
  idx int               // index of this instr
  mode byte             // mode (as above)
  out *instr            // next instr to process
  out1 *instr           // alt next instr (for kSplit)
  rune int              // rune to match (kRune)
  matcher func(rune int) bool   // matcher method (for kCall)
  alt int               // identifier of alt branch (for kAlt{Begin,End})
}

/**
 * String-representation of an individual instruction.
 */
func (i *instr) str() string {
  str := fmt.Sprintf("{%d", i.idx)
  out := ""
  if i.out != nil {
    out += fmt.Sprintf(" out=%d", i.out.idx)
  }
  switch i.mode {
  case kSplit:
    str += " kSplit"
    if i.out1 != nil {
      out += fmt.Sprintf(" out1=%d", i.out1.idx)
    }
  case kAltBegin:
    str += fmt.Sprintf(" kAltBegin alt=%d", i.alt)
  case kAltEnd:
    str += fmt.Sprintf(" kAltEnd alt=%d", i.alt)
  case kRune:
    str += fmt.Sprintf(" kRune rune=%c", i.rune)
  case kCall:
    str += " kCall meth=?"
  case kMatch:
    str += " kMatch"
  }
  return str + out + "}"
}

type parser struct {
  src string
  ch int
  pos int
  prog []*instr
  inst int
  altpos int
}

/*
 * Generic matcher for consuming instr instances (i.e. kRune/kCall). Does not
 * match anything else.
 */
func (s *instr) match(rune int) bool {
  if s.mode == kRune {
    return s.rune == rune || s.rune == -1
  } else if s.mode == kCall {
    return s.matcher(rune)
  }
  return false
}

/**
 * Generate a new pre-indexed instr.
 */
func (p *parser) instr() *instr {
  if p.inst == len(p.prog) {
    panic("overflow instr buffer")
  }
  i := &instr{p.inst, kSplit, nil, nil, -1, nil, -1}
  p.prog[p.inst] = i
  p.inst += 1
  return i
}

/**
 * Store/return the next character in parser. -1 indicates EOF.
 */
func (p *parser) nextc() int {
  if p.pos >= len(p.src) {
    p.ch = -1
  } else {
		c, w := utf8.DecodeRuneInString(p.src[p.pos:])
		p.ch = c
		p.pos += w
  }
  return p.ch
}

/**
 * Connect from -> to.
 */
func (p *parser) out(from *instr, to *instr) {
  if from.out == nil {
    from.out = to
  } else if from.mode == kSplit && from.out1 == nil {
    from.out1 = to
  } else {
    panic("can't out")
  }
}

func (p *parser) alt() (start *instr, end *instr) {
  altpos := p.altpos
  p.altpos += 1

  if p.ch != '(' {
    panic("alt must start with '('")
  }

  end = p.instr() // shared end state for alt
  end.mode = kAltEnd
  end.alt = altpos

  p.nextc()
  b_start, b_end := p.regexp()
  start = b_start
  p.out(b_end, end)

  for p.ch == '|' {
    split := p.instr()
    p.out(split, b_start)

    p.nextc()
    b_start, b_end = p.regexp()
    p.out(split, b_start)
    p.out(b_end, end)

    start = split
  }

  if p.ch != ')' {
    panic("alt must end with ')'")
  }

  alt_begin := p.instr()
  alt_begin.mode = kAltBegin
  alt_begin.alt = altpos
  p.out(alt_begin, start)
  return alt_begin, end
}

func (p *parser) term() (start *instr, end *instr) {
  start = p.instr()
  end = start

  switch p.ch {
  case -1:
    panic("EOF in term")
  case '*', '+', '{', '?':
    panic("unexpected expansion char")
  case ')', '}', ']':
    panic("unexpected close element")
  case '(':
    start, end = p.alt()
  case '[':
    panic("not yet supported: [")
  case '$':
    panic("not yet supported: end of string")
  case '^':
    panic("not yet supported: start of string")
  case '.':
    start.mode = kRune
  case '\\':
    next := p.nextc()
    start.mode = kRune
    switch next {
    case 'n':
      start.rune = '\n'
    case 't':
      start.rune = '\t'
    default:
      // TODO: limit this to punctuation
      start.rune = next
    }
  default:
    start.mode = kRune
    start.rune = p.ch
  }
  p.nextc()
  return start, end
}

func (p *parser) closure() (start *instr, end *instr) {
  start, end = p.term()

  switch p.ch {
  case '?', '*':
    p_start := start
    p_end := end
    start = p.instr()
    end = p.instr()
    p.out(start, p_start)
    p.out(start, end)
    p.out(p_end, end)
    if p.ch == '*' {
      p.out(end, start)
    }
    p.nextc()
  case '+':
    p_end := end
    end = p.instr()
    p.out(p_end, end)
    p.out(end, start)
    p.nextc()
  case '{':
    count_str := ""
    p.nextc()
    for unicode.IsDigit(p.ch) {
      count_str += fmt.Sprintf("%c", p.ch)
      p.nextc()
    }
    if len(count_str) == 0 {
      panic("{ must be followed by digit")
    }
    if p.ch == '}' {
      // fixed expansion
      count, _ := strconv.Atoi(count_str)
      panic(fmt.Sprintf("can't yet expand to: %d", count))
    } else if p.ch == ',' {
      panic("can't handle anything but {n}")
    } else {
      panic("unexpected char in {}")
    }
  }
  return start, end
}

/**
 * Match a regexp (defined as [closure]*) from parser, until either: EOF, |, or
 * ) is encountered.
 */
func (p *parser) regexp() (start *instr, end *instr) {
  start = p.instr()
  curr := start

  for {
    if p.ch == -1 || p.ch == '|' || p.ch == ')' {
      break
    }
    s, e := p.closure()
    p.out(curr, s)
    curr = e
  }

  end = p.instr()
  p.out(curr, end)
  return start, end
}

/**
 * Cleanup the given program. Assumes the given input is a flat slice containing
 * no nil instructions. Will not clean up the first instruction, as it is always
 * the canonical entry point for the regexp.
 *
 * Returns a similarly flat slice containing no nil instructions, however the
 * slice may potentially be smaller.
 */
func cleanup(prog []*instr) []*instr {
  // TODO: Clear kSplit recursion. In some cases, kSplit paths may recurse back
  // on themselves. We can remove this and convert it to a single-instr kSplit.

  // Iterate through the program, and remove single-instr kSplits.
  // NB: Don't parse the first instr, it will always be single.
  for i := 1; i < len(prog); i++ {
    pi := prog[i]
    if pi.mode == kSplit && (pi.out1 == nil || pi.out == pi.out1) {
      for j := 0; j < len(prog); j++ {
        if prog[j] == nil {
          continue
        }
        pj := prog[j]
        if pj.out == pi {
          pj.out = pi.out
        }
        if pj.out1 == pi {
          pj.out1 = pi.out
        }
      }
      prog[i] = nil
    }
  }

  // We may now have nil gaps: shift everything up.
  last := 0
  for i := 0; i < len(prog); i++ {
    if prog[i] != nil {
      last = i
    } else {
      // find next non-nil, move here
      var found int
      for found = i; found < len(prog); found++ {
        if prog[found] != nil {
          break
        }
      }
      if found == len(prog) {
        break // no more entries
      }

      // move found to i
      prog[i] = prog[found]
      prog[i].idx = i
      prog[found] = nil
      last = i
    }
  }

  return prog[0:last+1]
}

/**
 * Generates a simple straight-forward NFA.
 */
func Parse(src string) (r *sregexp) {
  p := parser{src, -1, 0, make([]*instr, 128), 0, 0}
  begin := p.instr()
  match := p.instr()
  match.mode = kMatch

  p.nextc()
  start, end := p.regexp()

  if p.nextc() != -1 {
    panic("could not consume all of regexp!")
  }

  p.out(begin, start)
  p.out(end, match)

  result := p.prog[0:end.idx+1]
  result = cleanup(result)

  return &sregexp{result, p.altpos}
}
<commit_msg>fixes to parser<commit_after>
package main

import (
  "fmt"
  "strconv"
  "unicode"
  "utf8"
)

const (
  kSplit = iota         // proceed down out & out1
  kAltBegin             // begin of alt section, i.e. '('
  kAltEnd               // end of alt section, i.e. ')'
  kRune                 // if match rune, proceed down out
  kCall                 // if matcher passes, proceed down out
  kMatch                // success state!
)

/**
 * Single instruction in regexp.
 */
type instr struct {
  idx int               // index of this instr
  mode byte             // mode (as above)
  out *instr            // next instr to process
  out1 *instr           // alt next instr (for kSplit)
  rune int              // rune to match (kRune)
  matcher func(rune int) bool   // matcher method (for kCall)
  alt int               // identifier of alt branch (for kAlt{Begin,End})
}

/**
 * String-representation of an individual instruction.
 */
func (i *instr) str() string {
  str := fmt.Sprintf("{%d", i.idx)
  out := ""
  if i.out != nil {
    out += fmt.Sprintf(" out=%d", i.out.idx)
  }
  switch i.mode {
  case kSplit:
    str += " kSplit"
    if i.out1 != nil {
      out += fmt.Sprintf(" out1=%d", i.out1.idx)
    }
  case kAltBegin:
    str += fmt.Sprintf(" kAltBegin alt=%d", i.alt)
  case kAltEnd:
    str += fmt.Sprintf(" kAltEnd alt=%d", i.alt)
  case kRune:
    str += fmt.Sprintf(" kRune rune=%c", i.rune)
  case kCall:
    str += " kCall meth=?"
  case kMatch:
    str += " kMatch"
  }
  return str + out + "}"
}

type parser struct {
  src string
  ch int
  pos int
  prog []*instr
  inst int
  altpos int
}

/*
 * Generic matcher for consuming instr instances (i.e. kRune/kCall). Does not
 * match anything else.
 */
func (s *instr) match(rune int) bool {
  if s.mode == kRune {
    return s.rune == rune || s.rune == -1
  } else if s.mode == kCall {
    return s.matcher(rune)
  }
  return false
}

/**
 * Generate a new pre-indexed instr.
 */
func (p *parser) instr() *instr {
  if p.inst == len(p.prog) {
    panic("overflow instr buffer")
  }
  i := &instr{p.inst, kSplit, nil, nil, -1, nil, -1}
  p.prog[p.inst] = i
  p.inst += 1
  return i
}

/**
 * Store/return the next character in parser. -1 indicates EOF.
 */
func (p *parser) nextc() int {
  if p.pos >= len(p.src) {
    p.ch = -1
  } else {
		c, w := utf8.DecodeRuneInString(p.src[p.pos:])
		p.ch = c
		p.pos += w
  }
  return p.ch
}

/**
 * Connect from -> to.
 */
func (p *parser) out(from *instr, to *instr) {
  if from.out == nil {
    from.out = to
  } else if from.mode == kSplit && from.out1 == nil {
    from.out1 = to
  } else {
    panic("can't out")
  }
}

func (p *parser) alt() (start *instr, end *instr) {
  altpos := p.altpos
  p.altpos += 1

  if p.ch != '(' {
    panic("alt must start with '('")
  }

  end = p.instr() // shared end state for alt
  end.mode = kAltEnd
  end.alt = altpos

  p.nextc()
  b_start, b_end := p.regexp()
  start = b_start
  p.out(b_end, end)

  for p.ch == '|' {
    start = p.instr()
    p.out(start, b_start)

    p.nextc()
    b_start, b_end = p.regexp()
    p.out(start, b_start)
    p.out(b_end, end)
    b_start = start
  }

  if p.ch != ')' {
    panic("alt must end with ')'")
  }

  alt_begin := p.instr()
  alt_begin.mode = kAltBegin
  alt_begin.alt = altpos
  p.out(alt_begin, start)
  return alt_begin, end
}

func (p *parser) term() (start *instr, end *instr) {
  start = p.instr()
  end = start

  switch p.ch {
  case -1:
    panic("EOF in term")
  case '*', '+', '{', '?':
    panic("unexpected expansion char")
  case ')', '}', ']':
    panic("unexpected close element")
  case '(':
    start, end = p.alt()
  case '[':
    panic("not yet supported: [")
  case '$':
    panic("not yet supported: end of string")
  case '^':
    panic("not yet supported: start of string")
  case '.':
    start.mode = kRune
  case '\\':
    next := p.nextc()
    start.mode = kRune
    switch next {
    case 'n':
      start.rune = '\n'
    case 't':
      start.rune = '\t'
    default:
      // TODO: limit this to punctuation
      start.rune = next
    }
  default:
    start.mode = kRune
    start.rune = p.ch
  }
  p.nextc()
  return start, end
}

func (p *parser) closure() (start *instr, end *instr) {
  start, end = p.term()

  switch p.ch {
  case '?', '*':
    p_start := start
    p_end := end
    start = p.instr()
    end = p.instr()
    p.out(start, p_start)
    p.out(start, end)
    if p.ch == '?' {
      p.out(p_end, end)
    } else {
      p.out(p_end, start)
    }
    p.nextc()
  case '+':
    p_end := end
    end = p.instr()
    p.out(p_end, end)
    p.out(end, start)
    p.nextc()
  case '{':
    count_str := ""
    p.nextc()
    for unicode.IsDigit(p.ch) {
      count_str += fmt.Sprintf("%c", p.ch)
      p.nextc()
    }
    if len(count_str) == 0 {
      panic("{ must be followed by digit")
    }
    if p.ch == '}' {
      // fixed expansion
      count, _ := strconv.Atoi(count_str)
      panic(fmt.Sprintf("can't yet expand to: %d", count))
    } else if p.ch == ',' {
      panic("can't handle anything but {n}")
    } else {
      panic("unexpected char in {}")
    }
  }
  return start, end
}

/**
 * Match a regexp (defined as [closure]*) from parser, until either: EOF, |, or
 * ) is encountered.
 */
func (p *parser) regexp() (start *instr, end *instr) {
  start = p.instr()
  curr := start

  for {
    if p.ch == -1 || p.ch == '|' || p.ch == ')' {
      break
    }
    s, e := p.closure()
    p.out(curr, s)
    curr = e
  }

  end = p.instr()
  p.out(curr, end)
  return start, end
}

/**
 * Cleanup the given program. Assumes the given input is a flat slice containing
 * no nil instructions. Will not clean up the first instruction, as it is always
 * the canonical entry point for the regexp.
 *
 * Returns a similarly flat slice containing no nil instructions, however the
 * slice may potentially be smaller.
 */
func cleanup(prog []*instr) []*instr {
  // Detect kSplit recursion. We can remove this and convert it to a single path.
  states := NewStateSet(len(prog), len(prog))
  for i := 1; i < len(prog); i++ {
    states.Clear()
    pi := prog[i]
    var fn func(ci *instr) bool
    fn = func(ci *instr) bool {
      if ci != nil && ci.mode == kSplit {
        if states.Put(ci.idx) {
          // NOTE: I'm not sure if this will ever happen. Panic for now. If we're
          // confident this won't happen, we could move the panic to runtime.
          panic("regexp should never loop")
          return true
        }
        if fn(ci.out) {
          ci.out = nil
        }
        if fn(ci.out1) {
          ci.out1 = nil
        }
        return true
      }
      return false
    }
    fn(pi)
  }

  // Iterate through the program, and remove single-instr kSplits.
  // NB: Don't parse the first instr, it will always be single.
  for i := 1; i < len(prog); i++ {
    pi := prog[i]
    if pi.mode == kSplit && (pi.out1 == nil || pi.out == pi.out1) {
      for j := 0; j < len(prog); j++ {
        if prog[j] == nil {
          continue
        }
        pj := prog[j]
        if pj.out == pi {
          pj.out = pi.out
        }
        if pj.out1 == pi {
          pj.out1 = pi.out
        }
      }
      prog[i] = nil
    }
  }

  // We may now have nil gaps: shift everything up.
  last := 0
  for i := 0; i < len(prog); i++ {
    if prog[i] != nil {
      last = i
    } else {
      // find next non-nil, move here
      var found int
      for found = i; found < len(prog); found++ {
        if prog[found] != nil {
          break
        }
      }
      if found == len(prog) {
        break // no more entries
      }

      // move found to i
      prog[i] = prog[found]
      prog[i].idx = i
      prog[found] = nil
      last = i
    }
  }

  return prog[0:last+1]
}

/**
 * Generates a simple straight-forward NFA.
 */
func Parse(src string) (r *sregexp) {
  p := parser{src, -1, 0, make([]*instr, 128), 0, 0}
  begin := p.instr()
  match := p.instr()
  match.mode = kMatch

  p.nextc()
  start, end := p.regexp()

  if p.nextc() != -1 {
    panic("could not consume all of regexp!")
  }

  p.out(begin, start)
  p.out(end, match)

  result := p.prog[0:end.idx+1]
  result = cleanup(result)

  return &sregexp{result, p.altpos}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/jessevdk/go-flags"
	"os"
)

type Options struct {
	/* verbosity level */
	Verbose []bool `short:"v" long:"verbose" description:"verbose output"`
}

var options Options
var parser = flags.NewParser(&options, flags.Default)

func main() {
	if _, err := parser.Parse(); err != nil {
		os.Exit(1)
	}
}
<commit_msg>all flag and command descriptions must be in caps<commit_after>package main

import (
	"github.com/jessevdk/go-flags"
	"os"
)

type Options struct {
	/* verbosity level */
	Verbose []bool `short:"v" long:"verbose" description:"Verbose output"`
}

var options Options
var parser = flags.NewParser(&options, flags.Default)

func main() {
	if _, err := parser.Parse(); err != nil {
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package goreq

import (
	"bufio"
	"bytes"
	"compress/flate"
	"compress/gzip"
	"compress/zlib"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"reflect"
	"strings"
	"time"
)

type Request struct {
	headers           []headerTuple
	Method            string
	Uri               string
	Body              interface{}
	QueryString       interface{}
	Timeout           time.Duration
	ContentType       string
	Accept            string
	Host              string
	UserAgent         string
	Insecure          bool
	MaxRedirects      int
	RedirectHeaders   bool
	Proxy             string
	Compression       *compression
	BasicAuthUsername string
	BasicAuthPassword string
}

type compression struct {
	writer          func(buffer io.Writer) (io.WriteCloser, error)
	reader          func(buffer io.Reader) (io.ReadCloser, error)
	ContentEncoding string
}

type Response struct {
	StatusCode    int
	ContentLength int64
	Body          *Body
	Header        http.Header
}

type headerTuple struct {
	name  string
	value string
}

type Body struct {
	reader           io.ReadCloser
	compressedReader io.ReadCloser
}

type Error struct {
	timeout bool
	Err     error
}

func (e *Error) Timeout() bool {
	return e.timeout
}

func (e *Error) Error() string {
	return e.Err.Error()
}

func (b *Body) Read(p []byte) (int, error) {
	if b.compressedReader != nil {
		return b.compressedReader.Read(p)
	}
	return b.reader.Read(p)
}

func (b *Body) Close() error {
	err := b.reader.Close()
	if b.compressedReader != nil {
		return b.compressedReader.Close()
	}
	return err
}

func (b *Body) FromJsonTo(o interface{}) error {
	if body, err := ioutil.ReadAll(b); err != nil {
		return err
	} else if err := json.Unmarshal(body, o); err != nil {
		return err
	}

	return nil
}

func (b *Body) ToString() (string, error) {
	body, err := ioutil.ReadAll(b)
	if err != nil {
		return "", err
	}
	return string(body), nil
}

func Gzip() *compression {
	reader := func(buffer io.Reader) (io.ReadCloser, error) {
		return gzip.NewReader(buffer)
	}
	writer := func(buffer io.Writer) (io.WriteCloser, error) {
		return gzip.NewWriter(buffer), nil
	}
	return &compression{writer: writer, reader: reader, ContentEncoding: "gzip"}
}

func Deflate() *compression {
	reader := func(buffer io.Reader) (io.ReadCloser, error) {
		return flate.NewReader(buffer), nil
	}
	writer := func(buffer io.Writer) (io.WriteCloser, error) {
		return flate.NewWriter(buffer, -1)
	}
	return &compression{writer: writer, reader: reader, ContentEncoding: "deflate"}
}

func Zlib() *compression {
	reader := func(buffer io.Reader) (io.ReadCloser, error) {
		return zlib.NewReader(buffer)
	}
	writer := func(buffer io.Writer) (io.WriteCloser, error) {
		return zlib.NewWriter(buffer), nil
	}
	return &compression{writer: writer, reader: reader, ContentEncoding: "deflate"}
}

func paramParse(query interface{}) (string, error) {
	switch query.(type) {
	case url.Values:
		return query.(url.Values).Encode(), nil
	case *url.Values:
		return query.(*url.Values).Encode(), nil
	default:
		var (
			v = &url.Values{}
			s = reflect.ValueOf(query)
			t = reflect.TypeOf(query)
		)
		for t.Kind() == reflect.Ptr || t.Kind() == reflect.Interface {
			s = s.Elem()
			t = s.Type()
		}
		for i := 0; i < t.NumField(); i++ {
			if len(t.Field(i).PkgPath) > 0 {
				continue
			}
			v.Add(strings.ToLower(t.Field(i).Name), fmt.Sprintf("%v", s.Field(i).Interface()))
		}
		return v.Encode(), nil
	}
}

func prepareRequestBody(b interface{}) (io.Reader, error) {
	switch b.(type) {
	case string:
		// treat is as text
		return strings.NewReader(b.(string)), nil
	case io.Reader:
		// treat is as text
		return b.(io.Reader), nil
	case []byte:
		//treat as byte array
		return bytes.NewReader(b.([]byte)), nil
	case nil:
		return nil, nil
	default:
		// try to jsonify it
		j, err := json.Marshal(b)
		if err == nil {
			return bytes.NewReader(j), nil
		}
		return nil, err
	}
}

var defaultDialer = &net.Dialer{Timeout: 1000 * time.Millisecond}
var defaultTransport = &http.Transport{Dial: defaultDialer.Dial, Proxy: http.ProxyFromEnvironment}
var defaultClient = &http.Client{Transport: defaultTransport}

var proxyTransport *http.Transport
var proxyClient *http.Client

func SetConnectTimeout(duration time.Duration) {
	defaultDialer.Timeout = duration
}

func (r *Request) AddHeader(name string, value string) {
	if r.headers == nil {
		r.headers = []headerTuple{}
	}
	r.headers = append(r.headers, headerTuple{name: name, value: value})
}

func (r Request) Do() (*Response, error) {
	var req *http.Request
	var er error
	var transport = defaultTransport
	var client = defaultClient
	var redirectFailed bool

	r.Method = valueOrDefault(r.Method, "GET")

	if r.Proxy != "" {
		proxyUrl, err := url.Parse(r.Proxy)
		if err != nil {
			// proxy address is in a wrong format
			return nil, &Error{Err: err}
		}
		if proxyTransport == nil {
			proxyTransport = &http.Transport{Dial: defaultDialer.Dial, Proxy: http.ProxyURL(proxyUrl)}
			proxyClient = &http.Client{Transport: proxyTransport}
		} else {
			proxyTransport.Proxy = http.ProxyURL(proxyUrl)
		}
		transport = proxyTransport
		client = proxyClient
	}

	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
		if len(via) > r.MaxRedirects {
			redirectFailed = true
			return errors.New("Error redirecting. MaxRedirects reached")
		}

		//By default Golang will not redirect request headers
		// https://code.google.com/p/go/issues/detail?id=4800&q=request%20header
		if r.RedirectHeaders {
			for key, val := range via[0].Header {
				req.Header[key] = val
			}
		}
		return nil
	}

	if r.Insecure {
		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
	} else if transport.TLSClientConfig != nil {
		// the default TLS client (when transport.TLSClientConfig==nil) is
		// already set to verify, so do nothing in that case
		transport.TLSClientConfig.InsecureSkipVerify = false
	}

	b, e := prepareRequestBody(r.Body)
	if e != nil {
		// there was a problem marshaling the body
		return nil, &Error{Err: e}
	}

	if r.QueryString != nil {
		param, e := paramParse(r.QueryString)
		if e != nil {
			return nil, &Error{Err: e}
		}
		r.Uri = r.Uri + "?" + param
	}

	var bodyReader io.Reader
	if b != nil && r.Compression != nil {
		buffer := bytes.NewBuffer([]byte{})
		readBuffer := bufio.NewReader(b)
		writer, err := r.Compression.writer(buffer)
		if err != nil {
			return nil, &Error{Err: err}
		}
		_, e = readBuffer.WriteTo(writer)
		writer.Close()
		if e != nil {
			return nil, &Error{Err: e}
		}
		bodyReader = buffer
	} else {
		bodyReader = b
	}
	req, er = http.NewRequest(r.Method, r.Uri, bodyReader)

	if er != nil {
		// we couldn't parse the URL.
		return nil, &Error{Err: er}
	}

	// add headers to the request
	req.Host = r.Host
	req.Header.Add("User-Agent", r.UserAgent)
	req.Header.Add("Content-Type", r.ContentType)
	req.Header.Add("Accept", r.Accept)
	if r.Compression != nil {
		req.Header.Add("Content-Encoding", r.Compression.ContentEncoding)
		req.Header.Add("Accept-Encoding", r.Compression.ContentEncoding)
	}
	if r.headers != nil {
		for _, header := range r.headers {
			req.Header.Add(header.name, header.value)
		}
	}

	//use basic auth if required
	if r.BasicAuthUsername != "" {
		req.SetBasicAuth(r.BasicAuthUsername, r.BasicAuthPassword)
	}

	timeout := false
	var timer *time.Timer
	if r.Timeout > 0 {
		timer = time.AfterFunc(r.Timeout, func() {
			transport.CancelRequest(req)
			timeout = true
		})
	}

	res, err := client.Do(req)
	if timer != nil {
		timer.Stop()
	}

	if err != nil {
		if !timeout {
			switch err := err.(type) {
			case *net.OpError:
				timeout = err.Timeout()
			case *url.Error:
				if op, ok := err.Err.(*net.OpError); ok {
					timeout = op.Timeout()
				}
			}
		}

		var response *Response
		//If redirect fails we still want to return response data
		if redirectFailed {
			response = &Response{StatusCode: res.StatusCode, ContentLength: res.ContentLength, Header: res.Header, Body: &Body{reader: res.Body}}
		}

		return response, &Error{timeout: timeout, Err: err}
	}

	if r.Compression != nil && strings.Contains(res.Header.Get("Content-Encoding"), r.Compression.ContentEncoding) {
		compressedReader, err := r.Compression.reader(res.Body)
		if err != nil {
			return nil, &Error{Err: err}
		}
		return &Response{StatusCode: res.StatusCode, ContentLength: res.ContentLength, Header: res.Header, Body: &Body{reader: res.Body, compressedReader: compressedReader}}, nil
	} else {
		return &Response{StatusCode: res.StatusCode, ContentLength: res.ContentLength, Header: res.Header, Body: &Body{reader: res.Body}}, nil
	}
}

// Return value if nonempty, def otherwise.
func valueOrDefault(value, def string) string {
	if value != "" {
		return value
	}
	return def
}
<commit_msg>Add Error when QueryString not supported<commit_after>package goreq

import (
	"bufio"
	"bytes"
	"compress/flate"
	"compress/gzip"
	"compress/zlib"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"reflect"
	"strings"
	"time"
)

type Request struct {
	headers           []headerTuple
	Method            string
	Uri               string
	Body              interface{}
	QueryString       interface{}
	Timeout           time.Duration
	ContentType       string
	Accept            string
	Host              string
	UserAgent         string
	Insecure          bool
	MaxRedirects      int
	RedirectHeaders   bool
	Proxy             string
	Compression       *compression
	BasicAuthUsername string
	BasicAuthPassword string
}

type compression struct {
	writer          func(buffer io.Writer) (io.WriteCloser, error)
	reader          func(buffer io.Reader) (io.ReadCloser, error)
	ContentEncoding string
}

type Response struct {
	StatusCode    int
	ContentLength int64
	Body          *Body
	Header        http.Header
}

type headerTuple struct {
	name  string
	value string
}

type Body struct {
	reader           io.ReadCloser
	compressedReader io.ReadCloser
}

type Error struct {
	timeout bool
	Err     error
}

func (e *Error) Timeout() bool {
	return e.timeout
}

func (e *Error) Error() string {
	return e.Err.Error()
}

func (b *Body) Read(p []byte) (int, error) {
	if b.compressedReader != nil {
		return b.compressedReader.Read(p)
	}
	return b.reader.Read(p)
}

func (b *Body) Close() error {
	err := b.reader.Close()
	if b.compressedReader != nil {
		return b.compressedReader.Close()
	}
	return err
}

func (b *Body) FromJsonTo(o interface{}) error {
	if body, err := ioutil.ReadAll(b); err != nil {
		return err
	} else if err := json.Unmarshal(body, o); err != nil {
		return err
	}

	return nil
}

func (b *Body) ToString() (string, error) {
	body, err := ioutil.ReadAll(b)
	if err != nil {
		return "", err
	}
	return string(body), nil
}

func Gzip() *compression {
	reader := func(buffer io.Reader) (io.ReadCloser, error) {
		return gzip.NewReader(buffer)
	}
	writer := func(buffer io.Writer) (io.WriteCloser, error) {
		return gzip.NewWriter(buffer), nil
	}
	return &compression{writer: writer, reader: reader, ContentEncoding: "gzip"}
}

func Deflate() *compression {
	reader := func(buffer io.Reader) (io.ReadCloser, error) {
		return flate.NewReader(buffer), nil
	}
	writer := func(buffer io.Writer) (io.WriteCloser, error) {
		return flate.NewWriter(buffer, -1)
	}
	return &compression{writer: writer, reader: reader, ContentEncoding: "deflate"}
}

func Zlib() *compression {
	reader := func(buffer io.Reader) (io.ReadCloser, error) {
		return zlib.NewReader(buffer)
	}
	writer := func(buffer io.Writer) (io.WriteCloser, error) {
		return zlib.NewWriter(buffer), nil
	}
	return &compression{writer: writer, reader: reader, ContentEncoding: "deflate"}
}

func paramParse(query interface{}) (string, error) {
	switch query.(type) {
	case url.Values:
		return query.(url.Values).Encode(), nil
	case *url.Values:
		return query.(*url.Values).Encode(), nil
	default:
		var (
			v = &url.Values{}
			s = reflect.ValueOf(query)
			t = reflect.TypeOf(query)
		)
		for t.Kind() == reflect.Ptr || t.Kind() == reflect.Interface {
			s = s.Elem()
			t = s.Type()
		}
		if t.Kind() == reflect.Struct {
			for i := 0; i < t.NumField(); i++ {
				if len(t.Field(i).PkgPath) > 0 {
					continue
				}
				v.Add(strings.ToLower(t.Field(i).Name), fmt.Sprintf("%v", s.Field(i).Interface()))
			}
			return v.Encode(), nil
		} else {
			return "", errors.New("Can not parse QueryString.")
		}
	}
}

func prepareRequestBody(b interface{}) (io.Reader, error) {
	switch b.(type) {
	case string:
		// treat is as text
		return strings.NewReader(b.(string)), nil
	case io.Reader:
		// treat is as text
		return b.(io.Reader), nil
	case []byte:
		//treat as byte array
		return bytes.NewReader(b.([]byte)), nil
	case nil:
		return nil, nil
	default:
		// try to jsonify it
		j, err := json.Marshal(b)
		if err == nil {
			return bytes.NewReader(j), nil
		}
		return nil, err
	}
}

var defaultDialer = &net.Dialer{Timeout: 1000 * time.Millisecond}
var defaultTransport = &http.Transport{Dial: defaultDialer.Dial, Proxy: http.ProxyFromEnvironment}
var defaultClient = &http.Client{Transport: defaultTransport}

var proxyTransport *http.Transport
var proxyClient *http.Client

func SetConnectTimeout(duration time.Duration) {
	defaultDialer.Timeout = duration
}

func (r *Request) AddHeader(name string, value string) {
	if r.headers == nil {
		r.headers = []headerTuple{}
	}
	r.headers = append(r.headers, headerTuple{name: name, value: value})
}

func (r Request) Do() (*Response, error) {
	var req *http.Request
	var er error
	var transport = defaultTransport
	var client = defaultClient
	var redirectFailed bool

	r.Method = valueOrDefault(r.Method, "GET")

	if r.Proxy != "" {
		proxyUrl, err := url.Parse(r.Proxy)
		if err != nil {
			// proxy address is in a wrong format
			return nil, &Error{Err: err}
		}
		if proxyTransport == nil {
			proxyTransport = &http.Transport{Dial: defaultDialer.Dial, Proxy: http.ProxyURL(proxyUrl)}
			proxyClient = &http.Client{Transport: proxyTransport}
		} else {
			proxyTransport.Proxy = http.ProxyURL(proxyUrl)
		}
		transport = proxyTransport
		client = proxyClient
	}

	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
		if len(via) > r.MaxRedirects {
			redirectFailed = true
			return errors.New("Error redirecting. MaxRedirects reached")
		}

		//By default Golang will not redirect request headers
		// https://code.google.com/p/go/issues/detail?id=4800&q=request%20header
		if r.RedirectHeaders {
			for key, val := range via[0].Header {
				req.Header[key] = val
			}
		}
		return nil
	}

	if r.Insecure {
		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
	} else if transport.TLSClientConfig != nil {
		// the default TLS client (when transport.TLSClientConfig==nil) is
		// already set to verify, so do nothing in that case
		transport.TLSClientConfig.InsecureSkipVerify = false
	}

	b, e := prepareRequestBody(r.Body)
	if e != nil {
		// there was a problem marshaling the body
		return nil, &Error{Err: e}
	}

	if r.QueryString != nil {
		param, e := paramParse(r.QueryString)
		if e != nil {
			return nil, &Error{Err: e}
		}
		r.Uri = r.Uri + "?" + param
	}

	var bodyReader io.Reader
	if b != nil && r.Compression != nil {
		buffer := bytes.NewBuffer([]byte{})
		readBuffer := bufio.NewReader(b)
		writer, err := r.Compression.writer(buffer)
		if err != nil {
			return nil, &Error{Err: err}
		}
		_, e = readBuffer.WriteTo(writer)
		writer.Close()
		if e != nil {
			return nil, &Error{Err: e}
		}
		bodyReader = buffer
	} else {
		bodyReader = b
	}
	req, er = http.NewRequest(r.Method, r.Uri, bodyReader)

	if er != nil {
		// we couldn't parse the URL.
		return nil, &Error{Err: er}
	}

	// add headers to the request
	req.Host = r.Host
	req.Header.Add("User-Agent", r.UserAgent)
	req.Header.Add("Content-Type", r.ContentType)
	req.Header.Add("Accept", r.Accept)
	if r.Compression != nil {
		req.Header.Add("Content-Encoding", r.Compression.ContentEncoding)
		req.Header.Add("Accept-Encoding", r.Compression.ContentEncoding)
	}
	if r.headers != nil {
		for _, header := range r.headers {
			req.Header.Add(header.name, header.value)
		}
	}

	//use basic auth if required
	if r.BasicAuthUsername != "" {
		req.SetBasicAuth(r.BasicAuthUsername, r.BasicAuthPassword)
	}

	timeout := false
	var timer *time.Timer
	if r.Timeout > 0 {
		timer = time.AfterFunc(r.Timeout, func() {
			transport.CancelRequest(req)
			timeout = true
		})
	}

	res, err := client.Do(req)
	if timer != nil {
		timer.Stop()
	}

	if err != nil {
		if !timeout {
			switch err := err.(type) {
			case *net.OpError:
				timeout = err.Timeout()
			case *url.Error:
				if op, ok := err.Err.(*net.OpError); ok {
					timeout = op.Timeout()
				}
			}
		}

		var response *Response
		//If redirect fails we still want to return response data
		if redirectFailed {
			response = &Response{StatusCode: res.StatusCode, ContentLength: res.ContentLength, Header: res.Header, Body: &Body{reader: res.Body}}
		}

		return response, &Error{timeout: timeout, Err: err}
	}

	if r.Compression != nil && strings.Contains(res.Header.Get("Content-Encoding"), r.Compression.ContentEncoding) {
		compressedReader, err := r.Compression.reader(res.Body)
		if err != nil {
			return nil, &Error{Err: err}
		}
		return &Response{StatusCode: res.StatusCode, ContentLength: res.ContentLength, Header: res.Header, Body: &Body{reader: res.Body, compressedReader: compressedReader}}, nil
	} else {
		return &Response{StatusCode: res.StatusCode, ContentLength: res.ContentLength, Header: res.Header, Body: &Body{reader: res.Body}}, nil
	}
}

// Return value if nonempty, def otherwise.
func valueOrDefault(value, def string) string {
	if value != "" {
		return value
	}
	return def
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"regexp"
	"strings"
)

type Toc struct {
	// DocToc
	depth  []int
	spaces []string
	title  []string
	link   []string
}

func headerTitle() string {
	header := "\n**Table of Contents**"
	return header
}

func headerCredits() string {
	return "\n<!-- Table of Contents generated by [gotoc](https://github.com/Belekkk/gotoc) -->"
}

func createHeader(title string) string {
	contentHeader := fmt.Sprintf("%s %s \n", headerCredits(), title)
	return contentHeader
}

func removeSpecialCharacters(str string) string {
	r, _ := regexp.Compile("([a-zA-Z]+)")
	cleanedStr := r.FindAllString(str, -1)
	return strings.Join(cleanedStr, " ")
}

func formatLink(str string) string {
	link := strings.Replace(str, " ", "-", -1)
	link = strings.ToLower(link)
	link = strings.Join([]string{"#", link}, "")
	return link
}

func computeSpaces(depth int) string {
	spaces := strings.Repeat(" ", depth)
	return spaces
}

func main() {
	var startsWith bool
	var line string
	var doctoc = new(Toc)
	var fileContent []string

	filename := flag.String("file", "", "a string")
	maxDepth := flag.Int("depth", 3, "an int")
	title := flag.String("title", headerTitle(), "a string")
	noTitle := flag.Bool("notitle", false, "a bool")
	flag.Parse()

	file, err := os.OpenFile(*filename, os.O_RDWR|os.O_APPEND, 0600)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line = scanner.Text()
		startsWith = strings.HasPrefix(line, "#")
		if startsWith == true {
			headerCount := strings.Count(line, "#")

			if headerCount <= *maxDepth {
				content := line[headerCount+1:]
				doctoc.depth = append(doctoc.depth, headerCount)
				indentation := (headerCount - 1) * 2
				nSpaces := computeSpaces(indentation)
				cleanedContent := removeSpecialCharacters(content)
				doctoc.spaces = append(doctoc.spaces, nSpaces)
				doctoc.title = append(doctoc.title, content)
				doctoc.link = append(doctoc.link, formatLink(cleanedContent))
			}
		}
		fileContent = append(fileContent, line)
	}

	err = os.Remove(*filename)
	if err != nil {
		log.Fatal(err)
	}

	if *noTitle == true {
		*title = ""
	}

	newFile, err := os.OpenFile(*filename, os.O_CREATE|os.O_RDWR, 0600)
	if err != nil {
		log.Fatal(err)
	}
	defer newFile.Close()

	if _, err = newFile.WriteString(createHeader(*title)); err != nil {
		log.Fatal(err)
	}
	for i := range doctoc.depth {
		doctocContent := fmt.Sprintf("%s- [%s](%s) \n", doctoc.spaces[i], doctoc.title[i], doctoc.link[i])
		if _, err = newFile.WriteString(doctocContent); err != nil {
			log.Fatal(err)
		}
	}

	for i := range fileContent {
		originalContent := fmt.Sprintf("%s\n", fileContent[i])
		if _, err = newFile.WriteString(originalContent); err != nil {
			log.Fatal(err)
		}
	}

	if err := scanner.Err(); err != nil {
		log.Fatal(err)
	}

}
<commit_msg>Improve output<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"regexp"
	"strings"
)

type Toc struct {
	// DocToc
	depth  []int
	spaces []string
	title  []string
	link   []string
}

func headerTitle() string {
	header := "\n**Table of Contents**"
	return header
}

func headerCredits() string {
	return "\n<!-- Table of Contents generated by [gotoc](https://github.com/Belekkk/gotoc) -->"
}

func createHeader(title string) string {
	contentHeader := fmt.Sprintf("%s %s \n", headerCredits(), title)
	return contentHeader
}

func removeSpecialCharacters(str string) string {
	r, _ := regexp.Compile("([a-zA-Z]+)")
	cleanedStr := r.FindAllString(str, -1)
	return strings.Join(cleanedStr, " ")
}

func formatLink(str string) string {
	link := strings.Replace(str, " ", "-", -1)
	link = strings.ToLower(link)
	link = strings.Join([]string{"#", link}, "")
	return link
}

func computeSpaces(depth int) string {
	spaces := strings.Repeat(" ", depth)
	return spaces
}

func outputLine(n int) string {
	line := strings.Repeat("-", n)
	return line
}

func main() {
	var startsWith bool
	var line string
	var doctoc = new(Toc)
	var fileContent []string

	filename := flag.String("file", "", "a string")
	maxDepth := flag.Int("depth", 3, "an int")
	title := flag.String("title", headerTitle(), "a string")
	noTitle := flag.Bool("notitle", false, "a bool")
	flag.Parse()

	file, err := os.OpenFile(*filename, os.O_RDWR|os.O_APPEND, 0600)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line = scanner.Text()
		startsWith = strings.HasPrefix(line, "#")
		if startsWith == true {
			headerCount := strings.Count(line, "#")

			if headerCount <= *maxDepth {
				content := line[headerCount+1:]
				doctoc.depth = append(doctoc.depth, headerCount)
				indentation := (headerCount - 1) * 2
				nSpaces := computeSpaces(indentation)
				cleanedContent := removeSpecialCharacters(content)
				doctoc.spaces = append(doctoc.spaces, nSpaces)
				doctoc.title = append(doctoc.title, content)
				doctoc.link = append(doctoc.link, formatLink(cleanedContent))
			}
		}
		fileContent = append(fileContent, line)
	}

	err = os.Remove(*filename)
	if err != nil {
		log.Fatal(err)
	}

	if *noTitle == true {
		*title = ""
	}

	newFile, err := os.OpenFile(*filename, os.O_CREATE|os.O_RDWR, 0600)
	if err != nil {
		log.Fatal(err)
	}
	defer newFile.Close()

	if _, err = newFile.WriteString(createHeader(*title)); err != nil {
		log.Fatal(err)
	}
	os.Stderr.WriteString("Generating Table of Contents\n")
	os.Stderr.WriteString(fmt.Sprintf("%s\n", outputLine(28)))
	for i := range doctoc.depth {
		doctocContent := fmt.Sprintf("%s- [%s](%s) \n", doctoc.spaces[i], doctoc.title[i], doctoc.link[i])
		if _, err = newFile.WriteString(doctocContent); err != nil {
			log.Fatal(err)
		}
	}

	os.Stderr.WriteString(fmt.Sprintf("'%s' will be updated\n", *filename))
	for i := range fileContent {
		originalContent := fmt.Sprintf("%s\n", fileContent[i])
		if _, err = newFile.WriteString(originalContent); err != nil {
			log.Fatal(err)
		}
	}
	os.Stderr.WriteString("Done\n")

	if err := scanner.Err(); err != nil {
		log.Fatal(err)
	}

}
<|endoftext|>


<commit_before>// Copyright 2016 The kube-etcd-controller Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package e2e

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"testing"
	"time"

	"github.com/coreos/kube-etcd-controller/pkg/spec"
	"github.com/coreos/kube-etcd-controller/pkg/util/k8sutil"
	"github.com/coreos/kube-etcd-controller/test/e2e/framework"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	k8sclient "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/util/wait"
)

func TestCreateCluster(t *testing.T) {
	f := framework.Global
	testEtcd, err := createEtcdCluster(f, makeEtcdCluster("test-etcd-", 3))
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		if err := deleteEtcdCluster(f, testEtcd.Name); err != nil {
			t.Fatal(err)
		}
	}()

	if _, err := waitUntilSizeReached(f, testEtcd.Name, 3, 60); err != nil {
		t.Fatalf("failed to create 3 members etcd cluster: %v", err)
	}
}

func TestResizeCluster3to5(t *testing.T) {
	f := framework.Global
	testEtcd, err := createEtcdCluster(f, makeEtcdCluster("test-etcd-", 3))
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		if err := deleteEtcdCluster(f, testEtcd.Name); err != nil {
			t.Fatal(err)
		}
	}()

	if _, err := waitUntilSizeReached(f, testEtcd.Name, 3, 60); err != nil {
		t.Fatalf("failed to create 3 members etcd cluster: %v", err)
		return
	}
	fmt.Println("reached to 3 members cluster")

	testEtcd.Spec.Size = 5
	if err := updateEtcdCluster(f, testEtcd); err != nil {
		t.Fatal(err)
	}

	if _, err := waitUntilSizeReached(f, testEtcd.Name, 5, 60); err != nil {
		t.Fatalf("failed to resize to 5 members etcd cluster: %v", err)
	}
}

func TestResizeCluster5to3(t *testing.T) {
	f := framework.Global
	testEtcd, err := createEtcdCluster(f, makeEtcdCluster("test-etcd-", 5))
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		if err := deleteEtcdCluster(f, testEtcd.Name); err != nil {
			t.Fatal(err)
		}
	}()

	if _, err := waitUntilSizeReached(f, testEtcd.Name, 5, 90); err != nil {
		t.Fatalf("failed to create 5 members etcd cluster: %v", err)
		return
	}
	fmt.Println("reached to 5 members cluster")

	testEtcd.Spec.Size = 3
	if err := updateEtcdCluster(f, testEtcd); err != nil {
		t.Fatal(err)
	}

	if _, err := waitUntilSizeReached(f, testEtcd.Name, 3, 60); err != nil {
		t.Fatalf("failed to resize to 3 members etcd cluster: %v", err)
	}
}

func TestOneMemberRecovery(t *testing.T) {
	f := framework.Global
	testEtcd, err := createEtcdCluster(f, makeEtcdCluster("test-etcd-", 3))
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		if err := deleteEtcdCluster(f, testEtcd.Name); err != nil {
			t.Fatal(err)
		}
	}()

	names, err := waitUntilSizeReached(f, testEtcd.Name, 3, 60)
	if err != nil {
		t.Fatalf("failed to create 3 members etcd cluster: %v", err)
		return
	}
	fmt.Println("reached to 3 members cluster")

	if err := killMembers(f, names[0]); err != nil {
		t.Fatal(err)
	}
	if _, err := waitUntilSizeReached(f, testEtcd.Name, 3, 60); err != nil {
		t.Fatalf("failed to resize to 3 members etcd cluster: %v", err)
	}
}

func TestDisasterRecovery(t *testing.T) {
	f := framework.Global
	backupPolicy := &spec.BackupPolicy{
		SnapshotIntervalInSecond: 120,
		MaxSnapshot:              5,
		VolumeSizeInMB:           512,
		StorageType:              spec.BackupStorageTypePersistentVolume,
		CleanupBackupIfDeleted:   true,
	}
	origEtcd := makeEtcdCluster("test-etcd-", 3)
	origEtcd = etcdClusterWithBackup(origEtcd, backupPolicy)
	testEtcd, err := createEtcdCluster(f, origEtcd)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		if err := deleteEtcdCluster(f, testEtcd.Name); err != nil {
			t.Fatal(err)
		}
		// TODO: add checking of removal of backup pod
	}()

	names, err := waitUntilSizeReached(f, testEtcd.Name, 3, 60)
	if err != nil {
		t.Fatalf("failed to create 3 members etcd cluster: %v", err)
		return
	}
	fmt.Println("reached to 3 members cluster")
	if err := killMembers(f, names[0], names[1]); err != nil {
		t.Fatal(err)
	}
	if _, err := waitUntilSizeReached(f, testEtcd.Name, 3, 120); err != nil {
		t.Fatalf("failed to resize to 3 members etcd cluster: %v", err)
	}
	// TODO: add checking of data in etcd
}

func waitUntilSizeReached(f *framework.Framework, clusterName string, size, timeout int) ([]string, error) {
	return waitSizeReachedWithFilter(f, clusterName, size, timeout, func(*api.Pod) bool { return true })
}

func waitSizeReachedWithFilter(f *framework.Framework, clusterName string, size, timeout int, filterPod func(*api.Pod) bool) ([]string, error) {
	var names []string
	err := wait.Poll(5*time.Second, time.Duration(timeout)*time.Second, func() (done bool, err error) {
		podList, err := f.KubeClient.Pods(f.Namespace.Name).List(k8sutil.EtcdPodListOpt(clusterName))
		if err != nil {
			return false, err
		}
		names = nil
		for i := range podList.Items {
			pod := &podList.Items[i]
			if pod.Status.Phase == api.PodRunning {
				names = append(names, pod.Name)
			}
		}
		fmt.Printf("waiting size (%d), etcd pods: %v\n", size, names)
		if len(names) != size {
			return false, nil
		}
		// TODO: check etcd member membership
		return true, nil
	})
	if err != nil {
		return nil, err
	}
	return names, nil
}

func killMembers(f *framework.Framework, names ...string) error {
	for _, name := range names {
		err := f.KubeClient.Pods(f.Namespace.Name).Delete(name, nil)
		if err != nil {
			return err
		}
	}
	return nil
}

func makeEtcdCluster(genName string, size int) *spec.EtcdCluster {
	return &spec.EtcdCluster{
		TypeMeta: unversioned.TypeMeta{
			Kind:       "EtcdCluster",
			APIVersion: "coreos.com/v1",
		},
		ObjectMeta: api.ObjectMeta{
			GenerateName: genName,
		},
		Spec: spec.ClusterSpec{
			Size: size,
		},
	}
}

func etcdClusterWithBackup(ec *spec.EtcdCluster, backupPolicy *spec.BackupPolicy) *spec.EtcdCluster {
	ec.Spec.Backup = backupPolicy
	return ec
}
func etcdClusterWithVersion(ec *spec.EtcdCluster, version string) *spec.EtcdCluster {
	ec.Spec.Version = version
	return ec
}

func createEtcdCluster(f *framework.Framework, e *spec.EtcdCluster) (*spec.EtcdCluster, error) {
	b, err := json.Marshal(e)
	if err != nil {
		return nil, err
	}
	resp, err := f.KubeClient.Client.Post(
		fmt.Sprintf("%s/apis/coreos.com/v1/namespaces/%s/etcdclusters", f.MasterHost, f.Namespace.Name),
		"application/json", bytes.NewReader(b))
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusCreated {
		return nil, fmt.Errorf("unexpected status: %v", resp.Status)
	}
	decoder := json.NewDecoder(resp.Body)
	res := &spec.EtcdCluster{}
	if err := decoder.Decode(res); err != nil {
		return nil, err
	}
	fmt.Printf("created etcd cluster: %v\n", res.Name)
	return res, nil
}

func updateEtcdCluster(f *framework.Framework, e *spec.EtcdCluster) error {
	b, err := json.Marshal(e)
	if err != nil {
		return err
	}
	req, err := http.NewRequest("PUT",
		fmt.Sprintf("%s/apis/coreos.com/v1/namespaces/%s/etcdclusters/%s", f.MasterHost, f.Namespace.Name, e.Name),
		bytes.NewReader(b))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	resp, err := f.KubeClient.Client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("unexpected status: %v", resp.Status)
	}
	return nil
}

func deleteEtcdCluster(f *framework.Framework, name string) error {
	// TODO: save etcd logs.
	fmt.Printf("deleting etcd cluster: %v\n", name)

	buf := bytes.NewBuffer(nil)
	if err := getLogs(f.KubeClient, f.Namespace.Name, "kube-etcd-controller", "kube-etcd-controller", buf); err != nil {
		return err
	}
	fmt.Println("kube-etcd-controller logs ===")
	fmt.Println(buf.String())
	fmt.Println("kube-etcd-controller logs END ===")

	req, err := http.NewRequest("DELETE",
		fmt.Sprintf("%s/apis/coreos.com/v1/namespaces/%s/etcdclusters/%s", f.MasterHost, f.Namespace.Name, name), nil)
	if err != nil {
		return err
	}
	resp, err := f.KubeClient.Client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("unexpected status: %v", resp.Status)
	}
	return nil
}

func getLogs(kubecli *k8sclient.Client, ns, p, c string, out io.Writer) error {
	req := kubecli.RESTClient.Get().
		Namespace(ns).
		Resource("pods").
		Name(p).
		SubResource("log").
		Param("container", c).
		Param("tailLines", "20")

	readCloser, err := req.Stream()
	if err != nil {
		return err
	}
	defer readCloser.Close()

	_, err = io.Copy(out, readCloser)
	return err
}
<commit_msg>e2e: add etcd pods info and failed pod logs<commit_after>// Copyright 2016 The kube-etcd-controller Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package e2e

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"testing"
	"time"

	"github.com/coreos/kube-etcd-controller/pkg/spec"
	"github.com/coreos/kube-etcd-controller/pkg/util/k8sutil"
	"github.com/coreos/kube-etcd-controller/test/e2e/framework"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	k8sclient "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/util/wait"
)

func TestCreateCluster(t *testing.T) {
	f := framework.Global
	testEtcd, err := createEtcdCluster(f, makeEtcdCluster("test-etcd-", 3))
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		if err := deleteEtcdCluster(f, testEtcd.Name); err != nil {
			t.Fatal(err)
		}
	}()

	if _, err := waitUntilSizeReached(f, testEtcd.Name, 3, 60); err != nil {
		t.Fatalf("failed to create 3 members etcd cluster: %v", err)
	}
}

func TestResizeCluster3to5(t *testing.T) {
	f := framework.Global
	testEtcd, err := createEtcdCluster(f, makeEtcdCluster("test-etcd-", 3))
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		if err := deleteEtcdCluster(f, testEtcd.Name); err != nil {
			t.Fatal(err)
		}
	}()

	if _, err := waitUntilSizeReached(f, testEtcd.Name, 3, 60); err != nil {
		t.Fatalf("failed to create 3 members etcd cluster: %v", err)
		return
	}
	fmt.Println("reached to 3 members cluster")

	testEtcd.Spec.Size = 5
	if err := updateEtcdCluster(f, testEtcd); err != nil {
		t.Fatal(err)
	}

	if _, err := waitUntilSizeReached(f, testEtcd.Name, 5, 60); err != nil {
		t.Fatalf("failed to resize to 5 members etcd cluster: %v", err)
	}
}

func TestResizeCluster5to3(t *testing.T) {
	f := framework.Global
	testEtcd, err := createEtcdCluster(f, makeEtcdCluster("test-etcd-", 5))
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		if err := deleteEtcdCluster(f, testEtcd.Name); err != nil {
			t.Fatal(err)
		}
	}()

	if _, err := waitUntilSizeReached(f, testEtcd.Name, 5, 90); err != nil {
		t.Fatalf("failed to create 5 members etcd cluster: %v", err)
		return
	}
	fmt.Println("reached to 5 members cluster")

	testEtcd.Spec.Size = 3
	if err := updateEtcdCluster(f, testEtcd); err != nil {
		t.Fatal(err)
	}

	if _, err := waitUntilSizeReached(f, testEtcd.Name, 3, 60); err != nil {
		t.Fatalf("failed to resize to 3 members etcd cluster: %v", err)
	}
}

func TestOneMemberRecovery(t *testing.T) {
	f := framework.Global
	testEtcd, err := createEtcdCluster(f, makeEtcdCluster("test-etcd-", 3))
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		if err := deleteEtcdCluster(f, testEtcd.Name); err != nil {
			t.Fatal(err)
		}
	}()

	names, err := waitUntilSizeReached(f, testEtcd.Name, 3, 60)
	if err != nil {
		t.Fatalf("failed to create 3 members etcd cluster: %v", err)
		return
	}
	fmt.Println("reached to 3 members cluster")

	if err := killMembers(f, names[0]); err != nil {
		t.Fatal(err)
	}
	if _, err := waitUntilSizeReached(f, testEtcd.Name, 3, 60); err != nil {
		t.Fatalf("failed to resize to 3 members etcd cluster: %v", err)
	}
}

func TestDisasterRecovery(t *testing.T) {
	f := framework.Global
	backupPolicy := &spec.BackupPolicy{
		SnapshotIntervalInSecond: 120,
		MaxSnapshot:              5,
		VolumeSizeInMB:           512,
		StorageType:              spec.BackupStorageTypePersistentVolume,
		CleanupBackupIfDeleted:   true,
	}
	origEtcd := makeEtcdCluster("test-etcd-", 3)
	origEtcd = etcdClusterWithBackup(origEtcd, backupPolicy)
	testEtcd, err := createEtcdCluster(f, origEtcd)
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		if err := deleteEtcdCluster(f, testEtcd.Name); err != nil {
			t.Fatal(err)
		}
		// TODO: add checking of removal of backup pod
	}()

	names, err := waitUntilSizeReached(f, testEtcd.Name, 3, 60)
	if err != nil {
		t.Fatalf("failed to create 3 members etcd cluster: %v", err)
		return
	}
	fmt.Println("reached to 3 members cluster")
	if err := killMembers(f, names[0], names[1]); err != nil {
		t.Fatal(err)
	}
	if _, err := waitUntilSizeReached(f, testEtcd.Name, 3, 120); err != nil {
		t.Fatalf("failed to resize to 3 members etcd cluster: %v", err)
	}
	// TODO: add checking of data in etcd
}

func waitUntilSizeReached(f *framework.Framework, clusterName string, size, timeout int) ([]string, error) {
	return waitSizeReachedWithFilter(f, clusterName, size, timeout, func(*api.Pod) bool { return true })
}

func waitSizeReachedWithFilter(f *framework.Framework, clusterName string, size, timeout int, filterPod func(*api.Pod) bool) ([]string, error) {
	var names []string
	err := wait.Poll(5*time.Second, time.Duration(timeout)*time.Second, func() (done bool, err error) {
		podList, err := f.KubeClient.Pods(f.Namespace.Name).List(k8sutil.EtcdPodListOpt(clusterName))
		if err != nil {
			return false, err
		}
		names = nil
		for i := range podList.Items {
			pod := &podList.Items[i]
			if pod.Status.Phase == api.PodRunning {
				names = append(names, pod.Name)
			}
		}
		fmt.Printf("waiting size (%d), etcd pods: %v\n", size, names)
		if len(names) != size {
			return false, nil
		}
		// TODO: check etcd member membership
		return true, nil
	})
	if err != nil {
		return nil, err
	}
	return names, nil
}

func killMembers(f *framework.Framework, names ...string) error {
	for _, name := range names {
		err := f.KubeClient.Pods(f.Namespace.Name).Delete(name, nil)
		if err != nil {
			return err
		}
	}
	return nil
}

func makeEtcdCluster(genName string, size int) *spec.EtcdCluster {
	return &spec.EtcdCluster{
		TypeMeta: unversioned.TypeMeta{
			Kind:       "EtcdCluster",
			APIVersion: "coreos.com/v1",
		},
		ObjectMeta: api.ObjectMeta{
			GenerateName: genName,
		},
		Spec: spec.ClusterSpec{
			Size: size,
		},
	}
}

func etcdClusterWithBackup(ec *spec.EtcdCluster, backupPolicy *spec.BackupPolicy) *spec.EtcdCluster {
	ec.Spec.Backup = backupPolicy
	return ec
}
func etcdClusterWithVersion(ec *spec.EtcdCluster, version string) *spec.EtcdCluster {
	ec.Spec.Version = version
	return ec
}

func createEtcdCluster(f *framework.Framework, e *spec.EtcdCluster) (*spec.EtcdCluster, error) {
	b, err := json.Marshal(e)
	if err != nil {
		return nil, err
	}
	resp, err := f.KubeClient.Client.Post(
		fmt.Sprintf("%s/apis/coreos.com/v1/namespaces/%s/etcdclusters", f.MasterHost, f.Namespace.Name),
		"application/json", bytes.NewReader(b))
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusCreated {
		return nil, fmt.Errorf("unexpected status: %v", resp.Status)
	}
	decoder := json.NewDecoder(resp.Body)
	res := &spec.EtcdCluster{}
	if err := decoder.Decode(res); err != nil {
		return nil, err
	}
	fmt.Printf("created etcd cluster: %v\n", res.Name)
	return res, nil
}

func updateEtcdCluster(f *framework.Framework, e *spec.EtcdCluster) error {
	b, err := json.Marshal(e)
	if err != nil {
		return err
	}
	req, err := http.NewRequest("PUT",
		fmt.Sprintf("%s/apis/coreos.com/v1/namespaces/%s/etcdclusters/%s", f.MasterHost, f.Namespace.Name, e.Name),
		bytes.NewReader(b))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	resp, err := f.KubeClient.Client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("unexpected status: %v", resp.Status)
	}
	return nil
}

func deleteEtcdCluster(f *framework.Framework, name string) error {
	fmt.Printf("deleting etcd cluster: %v\n", name)
	podList, err := f.KubeClient.Pods(f.Namespace.Name).List(k8sutil.EtcdPodListOpt(name))
	if err != nil {
		return err
	}
	fmt.Println("etcd pods ======")
	for i := range podList.Items {
		pod := &podList.Items[i]
		fmt.Printf("pod (%v): status (%v)\n", pod.Name, pod.Status.Phase)
		buf := bytes.NewBuffer(nil)

		if pod.Status.Phase == api.PodFailed {
			if err := getLogs(f.KubeClient, f.Namespace.Name, pod.Name, "etcd", buf); err != nil {
				return err
			}
			fmt.Println(pod.Name, "logs ===")
			fmt.Println(buf.String())
			fmt.Println(pod.Name, "logs END ===")
		}
	}

	buf := bytes.NewBuffer(nil)
	if err := getLogs(f.KubeClient, f.Namespace.Name, "kube-etcd-controller", "kube-etcd-controller", buf); err != nil {
		return err
	}
	fmt.Println("kube-etcd-controller logs ===")
	fmt.Println(buf.String())
	fmt.Println("kube-etcd-controller logs END ===")

	req, err := http.NewRequest("DELETE",
		fmt.Sprintf("%s/apis/coreos.com/v1/namespaces/%s/etcdclusters/%s", f.MasterHost, f.Namespace.Name, name), nil)
	if err != nil {
		return err
	}
	resp, err := f.KubeClient.Client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("unexpected status: %v", resp.Status)
	}
	return nil
}

func getLogs(kubecli *k8sclient.Client, ns, p, c string, out io.Writer) error {
	req := kubecli.RESTClient.Get().
		Namespace(ns).
		Resource("pods").
		Name(p).
		SubResource("log").
		Param("container", c).
		Param("tailLines", "20")

	readCloser, err := req.Stream()
	if err != nil {
		return err
	}
	defer readCloser.Close()

	_, err = io.Copy(out, readCloser)
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
)

type IPAddress struct {
	IP string `json:"ip"`
}

func ipify(w http.ResponseWriter, r *http.Request) {
	host, _, err := net.SplitHostPort(r.RemoteAddr)
	fmt.Println(r.Header["X-Forwarded-For"])
	//host, _, err := net.SplitHostPort(r.Header["X-Forwarded-For"])
	if err != nil {
		log.Fatal("SplitHostPort:", err)
	}

	w.Header().Set("Content-Type", "application/json")

	jsonStr, _ := json.MarshalIndent(IPAddress{host}, "", "  ")
	fmt.Fprintf(w, string(jsonStr))
}

func main() {
	http.HandleFunc("/", ipify)

	err := http.ListenAndServe(":"+os.Getenv("PORT"), nil)
	if err != nil {
		panic(err)
	}
}
<commit_msg>Trying to grab the last element.<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
)

type IPAddress struct {
	IP string `json:"ip"`
}

func ipify(w http.ResponseWriter, r *http.Request) {
	host, _, err := net.SplitHostPort(r.RemoteAddr)
	fmt.Println(r.Header["X-Forwarded-For"][len(r.Header["X-Forwarded-For"])])
	//host, _, err := net.SplitHostPort(r.Header["X-Forwarded-For"])
	if err != nil {
		log.Fatal("SplitHostPort:", err)
	}

	w.Header().Set("Content-Type", "application/json")

	jsonStr, _ := json.MarshalIndent(IPAddress{host}, "", "  ")
	fmt.Fprintf(w, string(jsonStr))
}

func main() {
	http.HandleFunc("/", ipify)

	err := http.ListenAndServe(":"+os.Getenv("PORT"), nil)
	if err != nil {
		panic(err)
	}
}
<|endoftext|>