<commit_before>// Copyright 2013 The go-github AUTHORS. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package github provides a client for using the GitHub API.

Construct a new GitHub client, then use the various services on the client to
access different parts of the GitHub API. For example:

	client := github.NewClient(nil)

	// list all organizations for user "willnorris"
	orgs, _, err := client.Organizations.List("willnorris", nil)

Set optional parameters for an API method by passing an Options object.

	// list recently updated repositories for org "github"
	opt := &github.RepositoryListByOrgOptions{Sort: "updated"}
	repos, _, err := client.Repositories.ListByOrg("github", opt)

The services of a client divide the API into logical chunks and correspond to
the structure of the GitHub API documentation at
http://developer.github.com/v3/.

Authentication

The go-github library does not directly handle authentication. Instead, when
creating a new client, pass an http.Client that can handle authentication for
you. The easiest and recommended way to do this is using the goauth2 library,
but you can always use any other library that provides an http.Client. If you
have an OAuth2 access token (for example, a personal API token), you can use it
with the goauth2 using:

	import "code.google.com/p/goauth2/oauth"

	// simple OAuth transport if you already have an access token;
	// see goauth2 library for full usage
	t := &oauth.Transport{
		Token: &oauth.Token{AccessToken: "..."},
	}

	client := github.NewClient(t.Client())

	// list all repositories for the authenticated user
	repos, _, err := client.Repositories.List("", nil)

Note that when using an authenticated Client, all calls made by the client will
include the specified OAuth token. Therefore, authenticated clients should
almost never be shared between different users.

Rate Limiting

GitHub imposes a rate limit on all API clients.  Unauthenticated clients are
limited to 60 requests per hour, while authenticated clients can make up to
5,000 requests per hour.  To receive the higher rate limit when making calls
that are not issued on behalf of a user, use the
UnauthenticatedRateLimitedTransport.

The Rate field on a client tracks the rate limit information based on the most
recent API call.  This is updated on every call, but may be out of date if it's
been some time since the last API call and other clients have made subsequent
requests since then.  You can always call RateLimit() directly to get the most
up-to-date rate limit data for the client.

Learn more about GitHub rate limiting at
http://developer.github.com/v3/#rate-limiting.

Creating and Updating Resources

All structs for GitHub resources use pointer values for all non-repeated fields.
This allows distinguishing between unset fields and those set to a zero-value.
Helper functions have been provided to easily create these pointers for string,
bool, and int values.  For example:

	// create a new private repository named "foo"
	repo := &github.Repo{
		Name:    github.String("foo"),
		Private: github.Bool(true),
	}
	client.Repositories.Create("", repo)

Users who have worked with protocol buffers should find this pattern familiar.
*/
package github
<commit_msg>Correct type name in the doc example<commit_after>// Copyright 2013 The go-github AUTHORS. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package github provides a client for using the GitHub API.

Construct a new GitHub client, then use the various services on the client to
access different parts of the GitHub API. For example:

	client := github.NewClient(nil)

	// list all organizations for user "willnorris"
	orgs, _, err := client.Organizations.List("willnorris", nil)

Set optional parameters for an API method by passing an Options object.

	// list recently updated repositories for org "github"
	opt := &github.RepositoryListByOrgOptions{Sort: "updated"}
	repos, _, err := client.Repositories.ListByOrg("github", opt)

The services of a client divide the API into logical chunks and correspond to
the structure of the GitHub API documentation at
http://developer.github.com/v3/.

Authentication

The go-github library does not directly handle authentication. Instead, when
creating a new client, pass an http.Client that can handle authentication for
you. The easiest and recommended way to do this is using the goauth2 library,
but you can always use any other library that provides an http.Client. If you
have an OAuth2 access token (for example, a personal API token), you can use it
with the goauth2 using:

	import "code.google.com/p/goauth2/oauth"

	// simple OAuth transport if you already have an access token;
	// see goauth2 library for full usage
	t := &oauth.Transport{
		Token: &oauth.Token{AccessToken: "..."},
	}

	client := github.NewClient(t.Client())

	// list all repositories for the authenticated user
	repos, _, err := client.Repositories.List("", nil)

Note that when using an authenticated Client, all calls made by the client will
include the specified OAuth token. Therefore, authenticated clients should
almost never be shared between different users.

Rate Limiting

GitHub imposes a rate limit on all API clients.  Unauthenticated clients are
limited to 60 requests per hour, while authenticated clients can make up to
5,000 requests per hour.  To receive the higher rate limit when making calls
that are not issued on behalf of a user, use the
UnauthenticatedRateLimitedTransport.

The Rate field on a client tracks the rate limit information based on the most
recent API call.  This is updated on every call, but may be out of date if it's
been some time since the last API call and other clients have made subsequent
requests since then.  You can always call RateLimit() directly to get the most
up-to-date rate limit data for the client.

Learn more about GitHub rate limiting at
http://developer.github.com/v3/#rate-limiting.

Creating and Updating Resources

All structs for GitHub resources use pointer values for all non-repeated fields.
This allows distinguishing between unset fields and those set to a zero-value.
Helper functions have been provided to easily create these pointers for string,
bool, and int values.  For example:

	// create a new private repository named "foo"
	repo := &github.Repository{
		Name:    github.String("foo"),
		Private: github.Bool(true),
	}
	client.Repositories.Create("", repo)

Users who have worked with protocol buffers should find this pattern familiar.
*/
package github
<|endoftext|>


<commit_before>package controllers

import (
	"fmt"
	"html/template"
	"strconv"
	"strings"

	"github.com/mark-greene/go-blackjack/lib/blackjack"

	"github.com/astaxie/beego"
)

type Session struct {
	PlayerName  string
	PlayerBet   int
	PlayerPot   int
	Turn        string
	Deck        blackjack.Deck
	DealerCards []blackjack.Card
	PlayerCards []blackjack.Card
}

func CalculateTotal(s *Session, cards []blackjack.Card) int {
	total := 0
	aces := 0
	for _, card := range cards {
		if card.Rank == "Ace" {
			total += 11
			aces++
		} else {
			v, err := strconv.Atoi(card.Rank)
			if err != nil {
				total += 10
			} else {
				total += v
			}
		}
	}

	for i := 0; i < aces; i++ {
		if total <= blackjack.BLACKJACK {
			break
		}
		total -= 10
	}

	return total
}
func CardImage(card blackjack.Card) string {

	return fmt.Sprintf("/static/img/cards/%s_%s.jpg", strings.ToLower(card.Suit), strings.ToLower(card.Rank))
}

func winner(c *MainController, msg string) {
	s := c.GetSession("session").(*Session)
	c.Data["playAgain"] = true
	c.Data["showHitStayButton"] = false
	s.PlayerPot = s.PlayerPot + s.PlayerBet
	c.Data["winner"] = HTML("%s won! %s", s.PlayerName, msg)
}
func loser(c *MainController, msg string) {
	s := c.GetSession("session").(*Session)
	c.Data["playAgain"] = true
	c.Data["showHitStayButton"] = false
	s.PlayerPot = s.PlayerPot - s.PlayerBet
	c.Data["loser"] = HTML("%s loses %s", s.PlayerName, msg)
}
func tie(c *MainController, msg string) {
	c.Data["playAgain"] = true
	c.Data["showHitStayButton"] = false
	c.Data["winner"] = HTML("It's a tie! %s", msg)
}
func HTML(format string, a ...interface{}) template.HTML {
	return template.HTML(fmt.Sprintf(format, a...))
}

func init() {
	// Functionss available to Template
	beego.AddFuncMap("CalculateTotal", CalculateTotal)
	beego.AddFuncMap("CardImage", CardImage)
}

type MainController struct {
	beego.Controller
}

// Prepare runs after Init before request function execution.
func (c *MainController) Prepare() {
	c.Layout = "layout.tpl"
	s := c.GetSession("session")
	if s == nil {
		s = &Session{}
		c.SetSession("session", s)
	}
	c.Data["session"] = s
	c.Data["playAgain"] = false
	c.Data["showHitStayButton"] = true
	c.Data["showDealerHitButton"] = false
	c.Data["success"] = ""
	c.Data["error"] = ""
	c.Data["winner"] = ""
	c.Data["loser"] = ""
}

func (c *MainController) Get() {
	s := c.GetSession("session").(*Session)
	if s.PlayerName == "" {
		c.Redirect("/new_player", 302)
		return
	}

	c.TplName = "game.tpl"
}

func (c *MainController) NewPlayer() {
	s := c.GetSession("session").(*Session)
	if c.Ctx.Input.Method() == "POST" {
		s.PlayerName = c.GetString("playerName")
		if s.PlayerName != "" {
			c.Redirect("/bet", 302)
			return
		}
		c.Data["error"] = "Name is required"
	}

	s.PlayerPot = blackjack.INITIAL_POT

	c.TplName = "new_player.tpl"
}

func (c *MainController) Bet() {
	s := c.GetSession("session").(*Session)
	if c.Ctx.Input.Method() == "POST" {
		betAmount, _ := strconv.Atoi(c.GetString("betAmount"))
		if betAmount == 0 {
			c.Data["error"] = "Must place a bet"
		} else if betAmount > s.PlayerPot {
			c.Data["error"] = HTML("Bet must be less that %d.", s.PlayerPot)
		} else {
			s.PlayerBet = betAmount
			c.Redirect("/game", 302)
			return
		}
	}

	s.PlayerBet = 0
	if s.PlayerPot <= 0 {
		c.Redirect("/game/over", 302)
		return
	}

	c.TplName = "bet.tpl"
}

func (c *MainController) Game() {
	s := c.GetSession("session").(*Session)

	s.Turn = s.PlayerName

	deck := blackjack.NewBlackJackDeck()
	deck.Shuffle()
	s.PlayerCards = nil
	s.DealerCards = nil
	s.PlayerCards = append(s.PlayerCards, deck.Deal())
	s.DealerCards = append(s.DealerCards, deck.Deal())
	s.PlayerCards = append(s.PlayerCards, deck.Deal())
	s.DealerCards = append(s.DealerCards, deck.Deal())
	// s.PlayerCards = append(s.PlayerCards, blackjack.Card{Suit: "Spades", Rank: "Ace"})
	// s.PlayerCards = append(s.PlayerCards, blackjack.Card{Suit: "Spades", Rank: "Jack"})
	// s.DealerCards = append(s.DealerCards, blackjack.Card{Suit: "Clubs", Rank: "Ace"})
	// s.DealerCards = append(s.DealerCards, blackjack.Card{Suit: "Clubs", Rank: "Jack"})
	s.Deck = deck

	total := CalculateTotal(s, s.DealerCards)
	if total == blackjack.BLACKJACK {
		c.Redirect("/game/dealer/blackjack", 302)
	}
	c.TplName = "game.tpl"
}

func (c *MainController) PlayerHit() {
	s := c.GetSession("session").(*Session)

	s.PlayerCards = append(s.PlayerCards, s.Deck.Deal())
	total := CalculateTotal(s, s.PlayerCards)
	if total == blackjack.BLACKJACK {
		// Winner(s, "You hit Blackjack!")
		c.Redirect("/game/dealer", 302)
	} else if total > blackjack.BLACKJACK {
		loser(c, "You busted!")
	}

	c.Layout = ""
	c.TplName = "game.tpl"
}

func (c *MainController) PlayerStay() {
	s := c.GetSession("session").(*Session)
	total := CalculateTotal(s, s.PlayerCards)
	if total == blackjack.BLACKJACK {
		c.Redirect("/game/compare", 302)
	} else {
		c.Data["success"] = HTML("%s has chosen to stay.", s.PlayerName)
		c.Redirect("/game/dealer", 302)
	}
}

func (c *MainController) Dealer() {
	s := c.GetSession("session").(*Session)

	s.Turn = "dealer"
	total := CalculateTotal(s, s.DealerCards)
	if total == blackjack.BLACKJACK {
		c.Redirect("/game/compare", 302)
	} else if total > blackjack.BLACKJACK {
		winner(c, "Dealer busted!")
	} else if total >= 17 {
		c.Redirect("/game/compare", 302)
	} else {
		c.Data["showDealerHitButton"] = true
	}

	c.Layout = ""
	c.TplName = "game.tpl"
}

func (c *MainController) DealerBlackjack() {
	s := c.GetSession("session").(*Session)
	s.Turn = "dealer"
	c.Data["showHitStayButton"] = false
	playerTotal := CalculateTotal(s, s.PlayerCards)
	if playerTotal == blackjack.BLACKJACK {
		tie(c, "You and dealer hit Blackjack.")
	} else {
		loser(c, "Dealer hit Blackjack!")
	}

	c.TplName = "game.tpl"
}

func (c *MainController) DealerHit() {
	s := c.GetSession("session").(*Session)

	s.DealerCards = append(s.DealerCards, s.Deck.Deal())

	c.Redirect("/game/dealer", 302)
}

func (c *MainController) Compare() {
	s := c.GetSession("session").(*Session)

	c.Data["showHitStayButton"] = false

	playerTotal := CalculateTotal(s, s.PlayerCards)
	dealerTotal := CalculateTotal(s, s.DealerCards)

	beego.Debug("Compare ", playerTotal, dealerTotal)
	if playerTotal == blackjack.BLACKJACK {
		if dealerTotal < blackjack.BLACKJACK {
			winner(c, fmt.Sprintf("You hit Blackjack! (dealer had %d)", dealerTotal))
		} else if dealerTotal == blackjack.BLACKJACK {
			tie(c, "You and dealer hit Blackjack.")
		}
	} else if dealerTotal == blackjack.BLACKJACK {
		loser(c, "Dealer hit Blackjack.")
	} else if playerTotal > dealerTotal {
		winner(c, fmt.Sprintf("You stayed at %d and the dealer stayed at %d.", playerTotal, dealerTotal))
	} else if playerTotal < dealerTotal {
		loser(c, fmt.Sprintf("You stayed at %d and the dealer stayed at %d.", playerTotal, dealerTotal))
	} else {
		tie(c, fmt.Sprintf("You and the dealer stayed at %d.", playerTotal))
	}

	c.Layout = ""
	c.TplName = "game.tpl"
}

func (c *MainController) Over() {
	s := c.GetSession("session").(*Session)
	if s.PlayerPot <= 0 {
		c.Data["error"] = HTML("%s, You are broke!", s.PlayerName)
	}
	c.TplName = "game_over.tpl"
}
<commit_msg>tweek<commit_after>package controllers

import (
	"fmt"
	"html/template"
	"strconv"
	"strings"

	"github.com/mark-greene/go-blackjack/lib/blackjack"

	"github.com/astaxie/beego"
)

type Session struct {
	PlayerName  string
	PlayerBet   int
	PlayerPot   int
	Turn        string
	Deck        blackjack.Deck
	DealerCards []blackjack.Card
	PlayerCards []blackjack.Card
}

func CalculateTotal(s *Session, cards []blackjack.Card) int {
	total := 0
	aces := 0
	for _, card := range cards {
		if card.Rank == "Ace" {
			total += 11
			aces++
		} else {
			v, err := strconv.Atoi(card.Rank)
			if err != nil {
				total += 10
			} else {
				total += v
			}
		}
	}

	for i := 0; i < aces; i++ {
		if total <= blackjack.BLACKJACK {
			break
		}
		total -= 10
	}

	return total
}
func CardImage(card blackjack.Card) string {

	return fmt.Sprintf("/static/img/cards/%s_%s.jpg", strings.ToLower(card.Suit), strings.ToLower(card.Rank))
}

func init() {
	// Functionss available to Template
	beego.AddFuncMap("CalculateTotal", CalculateTotal)
	beego.AddFuncMap("CardImage", CardImage)
}

type MainController struct {
	beego.Controller
}

func winner(c *MainController, msg string) {
	s := c.GetSession("session").(*Session)
	c.Data["playAgain"] = true
	c.Data["showHitStayButton"] = false
	s.PlayerPot = s.PlayerPot + s.PlayerBet
	c.Data["winner"] = HTML("%s won! %s", s.PlayerName, msg)
}
func loser(c *MainController, msg string) {
	s := c.GetSession("session").(*Session)
	c.Data["playAgain"] = true
	c.Data["showHitStayButton"] = false
	s.PlayerPot = s.PlayerPot - s.PlayerBet
	c.Data["loser"] = HTML("%s lost. %s", s.PlayerName, msg)
}
func tie(c *MainController, msg string) {
	c.Data["playAgain"] = true
	c.Data["showHitStayButton"] = false
	c.Data["winner"] = HTML("It's a tie! %s", msg)
}
func HTML(format string, a ...interface{}) template.HTML {
	return template.HTML(fmt.Sprintf(format, a...))
}

// Prepare runs after Init before request function execution.
func (c *MainController) Prepare() {
	c.Layout = "layout.tpl"
	s := c.GetSession("session")
	if s == nil {
		s = &Session{}
		c.SetSession("session", s)
	}
	c.Data["session"] = s
	c.Data["playAgain"] = false
	c.Data["showHitStayButton"] = true
	c.Data["showDealerHitButton"] = false
	c.Data["success"] = ""
	c.Data["error"] = ""
	c.Data["winner"] = ""
	c.Data["loser"] = ""
}

func (c *MainController) Get() {
	s := c.GetSession("session").(*Session)
	if s.PlayerName == "" {
		c.Redirect("/new_player", 302)
		return
	}

	c.TplName = "game.tpl"
}

func (c *MainController) NewPlayer() {
	s := c.GetSession("session").(*Session)
	if c.Ctx.Input.Method() == "POST" {
		s.PlayerName = c.GetString("playerName")
		if s.PlayerName != "" {
			c.Redirect("/bet", 302)
			return
		}
		c.Data["error"] = "Name is required"
	}

	s.PlayerPot = blackjack.INITIAL_POT

	c.TplName = "new_player.tpl"
}

func (c *MainController) Bet() {
	s := c.GetSession("session").(*Session)
	if c.Ctx.Input.Method() == "POST" {
		betAmount, _ := strconv.Atoi(c.GetString("betAmount"))
		if betAmount == 0 {
			c.Data["error"] = "Must place a bet"
		} else if betAmount > s.PlayerPot {
			c.Data["error"] = HTML("Bet must be less that %d.", s.PlayerPot)
		} else {
			s.PlayerBet = betAmount
			c.Redirect("/game", 302)
			return
		}
	}

	s.PlayerBet = 0
	if s.PlayerPot <= 0 {
		c.Redirect("/game/over", 302)
		return
	}

	c.TplName = "bet.tpl"
}

func (c *MainController) Game() {
	s := c.GetSession("session").(*Session)

	s.Turn = s.PlayerName

	deck := blackjack.NewBlackJackDeck()
	deck.Shuffle()
	s.PlayerCards = nil
	s.DealerCards = nil
	s.PlayerCards = append(s.PlayerCards, deck.Deal())
	s.DealerCards = append(s.DealerCards, deck.Deal())
	s.PlayerCards = append(s.PlayerCards, deck.Deal())
	s.DealerCards = append(s.DealerCards, deck.Deal())
	// s.PlayerCards = append(s.PlayerCards, blackjack.Card{Suit: "Spades", Rank: "Ace"})
	// s.PlayerCards = append(s.PlayerCards, blackjack.Card{Suit: "Spades", Rank: "Jack"})
	// s.DealerCards = append(s.DealerCards, blackjack.Card{Suit: "Clubs", Rank: "Ace"})
	// s.DealerCards = append(s.DealerCards, blackjack.Card{Suit: "Clubs", Rank: "Jack"})
	s.Deck = deck

	total := CalculateTotal(s, s.DealerCards)
	if total == blackjack.BLACKJACK {
		c.Redirect("/game/dealer/blackjack", 302)
	}
	c.TplName = "game.tpl"
}

func (c *MainController) PlayerHit() {
	s := c.GetSession("session").(*Session)

	s.PlayerCards = append(s.PlayerCards, s.Deck.Deal())
	total := CalculateTotal(s, s.PlayerCards)
	if total == blackjack.BLACKJACK {
		// Winner(s, "You hit Blackjack!")
		c.Redirect("/game/dealer", 302)
	} else if total > blackjack.BLACKJACK {
		loser(c, "You busted!")
	}

	c.Layout = ""
	c.TplName = "game.tpl"
}

func (c *MainController) PlayerStay() {
	s := c.GetSession("session").(*Session)
	total := CalculateTotal(s, s.PlayerCards)
	if total == blackjack.BLACKJACK {
		c.Redirect("/game/compare", 302)
	} else {
		c.Data["success"] = HTML("%s has chosen to stay.", s.PlayerName)
		c.Redirect("/game/dealer", 302)
	}
}

func (c *MainController) Dealer() {
	s := c.GetSession("session").(*Session)

	s.Turn = "dealer"
	total := CalculateTotal(s, s.DealerCards)
	if total == blackjack.BLACKJACK {
		c.Redirect("/game/compare", 302)
	} else if total > blackjack.BLACKJACK {
		winner(c, "Dealer busted!")
	} else if total >= 17 {
		c.Redirect("/game/compare", 302)
	} else {
		c.Data["showDealerHitButton"] = true
	}

	c.Layout = ""
	c.TplName = "game.tpl"
}

func (c *MainController) DealerBlackjack() {
	s := c.GetSession("session").(*Session)
	s.Turn = "dealer"
	c.Data["showHitStayButton"] = false
	playerTotal := CalculateTotal(s, s.PlayerCards)
	if playerTotal == blackjack.BLACKJACK {
		tie(c, "You and dealer hit Blackjack.")
	} else {
		loser(c, "Dealer hit Blackjack!")
	}

	c.TplName = "game.tpl"
}

func (c *MainController) DealerHit() {
	s := c.GetSession("session").(*Session)

	s.DealerCards = append(s.DealerCards, s.Deck.Deal())

	c.Redirect("/game/dealer", 302)
}

func (c *MainController) Compare() {
	s := c.GetSession("session").(*Session)

	c.Data["showHitStayButton"] = false

	playerTotal := CalculateTotal(s, s.PlayerCards)
	dealerTotal := CalculateTotal(s, s.DealerCards)

	beego.Debug("Compare ", playerTotal, dealerTotal)
	if playerTotal == blackjack.BLACKJACK {
		if dealerTotal < blackjack.BLACKJACK {
			winner(c, fmt.Sprintf("You hit Blackjack! (dealer has %d)", dealerTotal))
		} else if dealerTotal == blackjack.BLACKJACK {
			tie(c, "You and dealer hit Blackjack.")
		}
	} else if dealerTotal == blackjack.BLACKJACK {
		loser(c, "Dealer hit Blackjack.")
	} else if playerTotal > dealerTotal {
		winner(c, fmt.Sprintf("You stayed at %d and the dealer stayed at %d.", playerTotal, dealerTotal))
	} else if playerTotal < dealerTotal {
		loser(c, fmt.Sprintf("You stayed at %d and the dealer stayed at %d.", playerTotal, dealerTotal))
	} else {
		tie(c, fmt.Sprintf("You and the dealer stayed at %d.", playerTotal))
	}

	c.Layout = ""
	c.TplName = "game.tpl"
}

func (c *MainController) Over() {
	s := c.GetSession("session").(*Session)
	if s.PlayerPot <= 0 {
		c.Data["error"] = HTML("%s, You are broke!", s.PlayerName)
	}
	c.TplName = "game_over.tpl"
}
<|endoftext|>


<commit_before>package convey

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"github.com/stretchr/testify/assert"
	"io/ioutil"
	"testing"
)

var payloadTestData = []struct {
	encoding *base64.Encoding
	value    string
	expected Payload
}{
	{
		encoding: base64.StdEncoding,
		value:    "eyAicGFyYW1ldGVycyI6IFsgeyAibmFtZSI6ICJEZXZpY2UuRGV2aWNlSW5mby5XZWJwYS5YX0NPTUNBU1QtQ09NX0NJRCIsICJ2YWx1ZSI6ICIwIiwgImRhdGFUeXBlIjogMCB9LCB7ICJuYW1lIjogIkRldmljZS5EZXZpY2VJbmZvLldlYnBhLlhfQ09NQ0FTVC1DT01fQ01DIiwgInZhbHVlIjogIjI2OSIsICJkYXRhVHlwZSI6IDIgfSBdIH0K",
		expected: Payload{
			"parameters": []Payload{
				Payload{
					"name":     "Device.DeviceInfo.Webpa.X_COMCAST-COM_CID",
					"value":    "0",
					"dataType": 0,
				},
				Payload{
					"name":     "Device.DeviceInfo.Webpa.X_COMCAST-COM_CMC",
					"value":    "269",
					"dataType": 2,
				},
			},
		},
	},
}

func TestDecodeBase64(t *testing.T) {
	assertions := assert.New(t)
	for _, record := range payloadTestData {
		var actual Payload
		if err := actual.DecodeBase64(record.encoding, record.value); err != nil {
			t.Errorf("DecodeBase64 failed: %v", err)
		}

		expectedJson, err := json.Marshal(record.expected)
		if err != nil {
			t.Fatalf("Unable to marshal expected JSON: %v", err)
		}

		actualJson, err := json.Marshal(actual)
		if err != nil {
			t.Fatalf("Unable to marshal actual JSON: %v", err)
		}

		assertions.JSONEq(string(expectedJson), string(actualJson))
	}
}

func TestEncodeBase64(t *testing.T) {
	assertions := assert.New(t)
	for _, record := range payloadTestData {
		// perform the reverse test: use the expected as our actual JSON
		actualEncoded, err := record.expected.EncodeBase64(record.encoding)
		if err != nil {
			t.Fatalf("Unable to encode: %v", err)
		}

		// decode the actual value, to get JSON that was can compare against
		actualInput := bytes.NewBufferString(actualEncoded)
		actualDecoder := base64.NewDecoder(record.encoding, actualInput)
		actualJson, err := ioutil.ReadAll(actualDecoder)
		if err != nil {
			t.Fatalf("Unable to decode the output of EncodeBase64: %v", err)
		}

		expectedInput := bytes.NewBufferString(record.value)
		expectedDecoder := base64.NewDecoder(record.encoding, expectedInput)
		expectedJson, err := ioutil.ReadAll(expectedDecoder)
		if err != nil {
			t.Fatalf("Unable to decode expected value: %v", err)
		}

		assertions.JSONEq(string(expectedJson), string(actualJson))
	}
}

func TestParsePayload(t *testing.T) {
	assertions := assert.New(t)
	for _, record := range payloadTestData {
		actualPayload, err := ParsePayload(record.encoding, record.value)
		if err != nil {
			t.Fatalf("ParsePayload failed: %v", err)
		}

		actualJson, err := json.Marshal(actualPayload)
		if err != nil {
			t.Fatalf("Failed to marshal actual payload: %v", err)
		}

		expectedJson, err := json.Marshal(record.expected)
		if err != nil {
			t.Fatalf("Failed to marshal expected JSON: %v", err)
		}

		assertions.JSONEq(string(expectedJson), string(actualJson))
	}
}
<commit_msg>Expanded tests<commit_after>package convey

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"github.com/stretchr/testify/assert"
	"io/ioutil"
	"testing"
)

// payloadTestData can be generated with:
// https://play.golang.org/p/Dn_wIn1Y14
var payloadTestData = []struct {
	encoding *base64.Encoding
	value    string
	expected Payload
}{
	{
		encoding: base64.StdEncoding,
		value:    "eyAicGFyYW1ldGVycyI6IFsgeyAibmFtZSI6ICJEZXZpY2UuRGV2aWNlSW5mby5XZWJwYS5YX0NPTUNBU1QtQ09NX0NJRCIsICJ2YWx1ZSI6ICIwIiwgImRhdGFUeXBlIjogMCB9LCB7ICJuYW1lIjogIkRldmljZS5EZXZpY2VJbmZvLldlYnBhLlhfQ09NQ0FTVC1DT01fQ01DIiwgInZhbHVlIjogIjI2OSIsICJkYXRhVHlwZSI6IDIgfSBdIH0K",
		expected: Payload{
			"parameters": []Payload{
				Payload{
					"name":     "Device.DeviceInfo.Webpa.X_COMCAST-COM_CID",
					"value":    "0",
					"dataType": 0,
				},
				Payload{
					"name":     "Device.DeviceInfo.Webpa.X_COMCAST-COM_CMC",
					"value":    "269",
					"dataType": 2,
				},
			},
		},
	},
	{
		encoding: base64.RawURLEncoding,
		value:    "eyJwYXJhbWV0ZXJzIjpbeyJkYXRhVHlwZSI6MCwibmFtZSI6IkRldmljZS5EZXZpY2VJbmZvLldlYnBhLlhfQ09NQ0FTVC1DT01fQ0lEIiwidmFsdWUiOiIwIn0seyJkYXRhVHlwZSI6MiwibmFtZSI6IkRldmljZS5EZXZpY2VJbmZvLldlYnBhLlhfQ09NQ0FTVC1DT01fQ01DIiwidmFsdWUiOiIyNjkifV19",
		expected: Payload{
			"parameters": []Payload{
				Payload{
					"name":     "Device.DeviceInfo.Webpa.X_COMCAST-COM_CID",
					"value":    "0",
					"dataType": 0,
				},
				Payload{
					"name":     "Device.DeviceInfo.Webpa.X_COMCAST-COM_CMC",
					"value":    "269",
					"dataType": 2,
				},
			},
		},
	},
	{
		encoding: base64.URLEncoding,
		value:    "eyJwYXJhbWV0ZXJzIjpbeyJkYXRhVHlwZSI6MCwibmFtZSI6IkRldmljZS5EZXZpY2VJbmZvLldlYnBhLlhfQ09NQ0FTVC1DT01fQ0lEIiwidmFsdWUiOiIwIn0seyJkYXRhVHlwZSI6MiwibmFtZSI6IkRldmljZS5EZXZpY2VJbmZvLldlYnBhLlhfQ09NQ0FTVC1DT01fQ01DIiwidmFsdWUiOiIyNjkifV19",
		expected: Payload{
			"parameters": []Payload{
				Payload{
					"name":     "Device.DeviceInfo.Webpa.X_COMCAST-COM_CID",
					"value":    "0",
					"dataType": 0,
				},
				Payload{
					"name":     "Device.DeviceInfo.Webpa.X_COMCAST-COM_CMC",
					"value":    "269",
					"dataType": 2,
				},
			},
		},
	},
	{
		encoding: base64.RawStdEncoding,
		value:    "eyJwYXJhbWV0ZXJzIjpbeyJkYXRhVHlwZSI6MCwibmFtZSI6IkRldmljZS5EZXZpY2VJbmZvLldlYnBhLlhfQ09NQ0FTVC1DT01fQ0lEIiwidmFsdWUiOiIwIn0seyJkYXRhVHlwZSI6MiwibmFtZSI6IkRldmljZS5EZXZpY2VJbmZvLldlYnBhLlhfQ09NQ0FTVC1DT01fQ01DIiwidmFsdWUiOiIyNjkifV19",
		expected: Payload{
			"parameters": []Payload{
				Payload{
					"name":     "Device.DeviceInfo.Webpa.X_COMCAST-COM_CID",
					"value":    "0",
					"dataType": 0,
				},
				Payload{
					"name":     "Device.DeviceInfo.Webpa.X_COMCAST-COM_CMC",
					"value":    "269",
					"dataType": 2,
				},
			},
		},
	},
}

func TestDecodeBase64(t *testing.T) {
	assertions := assert.New(t)
	for _, record := range payloadTestData {
		var actual Payload
		if err := actual.DecodeBase64(record.encoding, record.value); err != nil {
			t.Errorf("DecodeBase64 failed: %v", err)
		}

		expectedJson, err := json.Marshal(record.expected)
		if err != nil {
			t.Fatalf("Unable to marshal expected JSON: %v", err)
		}

		actualJson, err := json.Marshal(actual)
		if err != nil {
			t.Fatalf("Unable to marshal actual JSON: %v", err)
		}

		assertions.JSONEq(string(expectedJson), string(actualJson))
	}
}

func TestEncodeBase64(t *testing.T) {
	assertions := assert.New(t)
	for _, record := range payloadTestData {
		// perform the reverse test: use the expected as our actual JSON
		actualEncoded, err := record.expected.EncodeBase64(record.encoding)
		if err != nil {
			t.Fatalf("Unable to encode: %v", err)
		}

		// decode the actual value, to get JSON that was can compare against
		actualInput := bytes.NewBufferString(actualEncoded)
		actualDecoder := base64.NewDecoder(record.encoding, actualInput)
		actualJson, err := ioutil.ReadAll(actualDecoder)
		if err != nil {
			t.Fatalf("Unable to decode the output of EncodeBase64: %v", err)
		}

		expectedInput := bytes.NewBufferString(record.value)
		expectedDecoder := base64.NewDecoder(record.encoding, expectedInput)
		expectedJson, err := ioutil.ReadAll(expectedDecoder)
		if err != nil {
			t.Fatalf("Unable to decode expected value: %v", err)
		}

		assertions.JSONEq(string(expectedJson), string(actualJson))
	}
}

func TestParsePayload(t *testing.T) {
	assertions := assert.New(t)
	for _, record := range payloadTestData {
		actualPayload, err := ParsePayload(record.encoding, record.value)
		if err != nil {
			t.Fatalf("ParsePayload failed: %v", err)
		}

		actualJson, err := json.Marshal(actualPayload)
		if err != nil {
			t.Fatalf("Failed to marshal actual payload: %v", err)
		}

		expectedJson, err := json.Marshal(record.expected)
		if err != nil {
			t.Fatalf("Failed to marshal expected JSON: %v", err)
		}

		assertions.JSONEq(string(expectedJson), string(actualJson))
	}
}
<|endoftext|>


<commit_before>package obj

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/aws/aws-sdk-go/service/storagegateway"
	"github.com/cenkalti/backoff"
)

type amazonClient struct {
	bucket   string
	s3       *s3.S3
	uploader *s3manager.Uploader
}

func newAmazonClient(bucket string, id string, secret string, token string, region string) (*amazonClient, error) {
	session := session.New(&aws.Config{
		Credentials: credentials.NewStaticCredentials(id, secret, token),
		Region:      aws.String(region),
	})
	return &amazonClient{
		bucket:   bucket,
		s3:       s3.New(session),
		uploader: s3manager.NewUploader(session),
	}, nil
}

func (c *amazonClient) Writer(name string) (io.WriteCloser, error) {
	return newBackoffWriteCloser(c, newWriter(c, name)), nil
}

func (c *amazonClient) Walk(name string, fn func(name string) error) error {
	var fnErr error
	if err := c.s3.ListObjectsPages(
		&s3.ListObjectsInput{
			Bucket: aws.String(c.bucket),
			Prefix: aws.String(name),
		},
		func(listObjectsOutput *s3.ListObjectsOutput, lastPage bool) bool {
			for _, object := range listObjectsOutput.Contents {
				if err := fn(*object.Key); err != nil {
					fnErr = err
					return false
				}
			}
			return true
		},
	); err != nil {
		return err
	}
	return fnErr
}

func isRetryableGetError(err error) bool {
	if strings.Contains(err.Error(), "dial tcp: i/o timeout") {
		return true
	}
	return isNetRetryable(err)
}

func (c *amazonClient) Reader(name string, offset uint64, size uint64) (io.ReadCloser, error) {
	byteRange := byteRange(offset, size)
	if byteRange != "" {
		byteRange = fmt.Sprintf("bytes=%s", byteRange)
	}

	var resp *http.Response
	var connErr error
	url := fmt.Sprintf("http://d2z5sy3mh7px6z.cloudfront.net/%v", name)

	backoff.RetryNotify(func() error {
		resp, connErr = http.Get(url)
		//	defer resp.Body.Close()
		fmt.Printf("got resp for url (%v), err: %v,\n\n%v\n", url, resp, connErr)
		fmt.Printf("Got http error: %v\n", connErr)
		if connErr != nil && isRetryableGetError(connErr) {
			fmt.Printf("this is a retryable error (%v)\n", connErr)
			return connErr
		}
		return nil
	}, backoff.NewExponentialBackOff(), func(err error, d time.Duration) {
		log.Infof("Error connecting to (%v); retrying in %s: %#v", url, d, err)
	})

	if resp.StatusCode != 200 {
		fmt.Printf("HTTP error code %v", resp.StatusCode)
		return nil, fmt.Errorf("HTTP error code %v", resp.StatusCode)
	}
	// TODO: send the offset header as part of the request
	n, err := io.CopyN(ioutil.Discard, resp.Body, int64(offset))
	fmt.Printf("slurped n bytes: %v off of get file %v to accomodate offset\n", n, url)
	if err != nil {
		return nil, err
	}
	//return newBackoffReadCloser(c, getObjectOutput.Body), nil
	//	return resp.Body, nil
	return newBackoffReadCloserForRealsies(url, c, resp.Body), nil
}

func (c *amazonClient) Delete(name string) error {
	_, err := c.s3.DeleteObject(&s3.DeleteObjectInput{
		Bucket: aws.String(c.bucket),
		Key:    aws.String(name),
	})
	return err
}

func (c *amazonClient) Exists(name string) bool {
	_, err := c.s3.HeadObject(&s3.HeadObjectInput{
		Bucket: aws.String(c.bucket),
		Key:    aws.String(name),
	})
	return err == nil
}

func (c *amazonClient) isRetryable(err error) (retVal bool) {
	fmt.Printf("is err (%v) retryable?\n", err)
	defer func() {
		fmt.Printf("err (%v) retryable? %v\n", retVal, err)
	}()
	if strings.Contains(err.Error(), "unexpected EOF") {
		return false
	}

	awsErr, ok := err.(awserr.Error)
	if !ok {
		return false
	}
	for _, c := range []string{
		storagegateway.ErrorCodeServiceUnavailable,
		storagegateway.ErrorCodeInternalError,
		storagegateway.ErrorCodeGatewayInternalError,
	} {
		if c == awsErr.Code() {
			return true
		}
	}
	return false
}

func (c *amazonClient) IsIgnorable(err error) bool {
	return false
}

func (c *amazonClient) IsNotExist(err error) bool {
	fmt.Printf("IsNotExist? error: %v\n", err)
	// cloudfront returns forbidden error for nonexisting data
	if strings.Contains(err.Error(), "error code 403") {
		fmt.Printf("its a 403, dne")
		return true
	}
	if strings.Contains(err.Error(), "error code 404") {
		fmt.Printf("its a 404, dne")
		return true
	}
	awsErr, ok := err.(awserr.Error)
	if !ok {
		return false
	}
	if awsErr.Code() == "NoSuchKey" {
		return true
	}
	return false
}

type amazonWriter struct {
	errChan chan error
	pipe    *io.PipeWriter
}

func newWriter(client *amazonClient, name string) *amazonWriter {
	reader, writer := io.Pipe()
	w := &amazonWriter{
		errChan: make(chan error),
		pipe:    writer,
	}
	go func() {
		_, err := client.uploader.Upload(&s3manager.UploadInput{
			Body:            reader,
			Bucket:          aws.String(client.bucket),
			Key:             aws.String(name),
			ContentEncoding: aws.String("application/octet-stream"),
		})
		w.errChan <- err
	}()
	return w
}

func (w *amazonWriter) Write(p []byte) (int, error) {
	return w.pipe.Write(p)
}

func (w *amazonWriter) Close() error {
	if err := w.pipe.Close(); err != nil {
		return err
	}
	return <-w.errChan
}
<commit_msg>Add a beacon to make sure this code is useful<commit_after>package obj

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/aws/aws-sdk-go/service/storagegateway"
	"github.com/cenkalti/backoff"
)

type amazonClient struct {
	bucket   string
	s3       *s3.S3
	uploader *s3manager.Uploader
}

func newAmazonClient(bucket string, id string, secret string, token string, region string) (*amazonClient, error) {
	session := session.New(&aws.Config{
		Credentials: credentials.NewStaticCredentials(id, secret, token),
		Region:      aws.String(region),
	})
	return &amazonClient{
		bucket:   bucket,
		s3:       s3.New(session),
		uploader: s3manager.NewUploader(session),
	}, nil
}

func (c *amazonClient) Writer(name string) (io.WriteCloser, error) {
	return newBackoffWriteCloser(c, newWriter(c, name)), nil
}

func (c *amazonClient) Walk(name string, fn func(name string) error) error {
	var fnErr error
	if err := c.s3.ListObjectsPages(
		&s3.ListObjectsInput{
			Bucket: aws.String(c.bucket),
			Prefix: aws.String(name),
		},
		func(listObjectsOutput *s3.ListObjectsOutput, lastPage bool) bool {
			for _, object := range listObjectsOutput.Contents {
				if err := fn(*object.Key); err != nil {
					fnErr = err
					return false
				}
			}
			return true
		},
	); err != nil {
		return err
	}
	return fnErr
}

func isRetryableGetError(err error) bool {
	if strings.Contains(err.Error(), "dial tcp: i/o timeout") {
		fmt.Printf("SAW A DIAL TCP TIMEOUT ERROR\n")
		return true
	}
	return isNetRetryable(err)
}

func (c *amazonClient) Reader(name string, offset uint64, size uint64) (io.ReadCloser, error) {
	byteRange := byteRange(offset, size)
	if byteRange != "" {
		byteRange = fmt.Sprintf("bytes=%s", byteRange)
	}

	var resp *http.Response
	var connErr error
	url := fmt.Sprintf("http://d2z5sy3mh7px6z.cloudfront.net/%v", name)

	backoff.RetryNotify(func() error {
		resp, connErr = http.Get(url)
		//	defer resp.Body.Close()
		fmt.Printf("got resp for url (%v), err: %v,\n\n%v\n", url, resp, connErr)
		fmt.Printf("Got http error: %v\n", connErr)
		if connErr != nil && isRetryableGetError(connErr) {
			fmt.Printf("this is a retryable error (%v)\n", connErr)
			return connErr
		}
		return nil
	}, backoff.NewExponentialBackOff(), func(err error, d time.Duration) {
		log.Infof("Error connecting to (%v); retrying in %s: %#v", url, d, err)
	})

	if resp.StatusCode != 200 {
		fmt.Printf("HTTP error code %v", resp.StatusCode)
		return nil, fmt.Errorf("HTTP error code %v", resp.StatusCode)
	}
	// TODO: send the offset header as part of the request
	n, err := io.CopyN(ioutil.Discard, resp.Body, int64(offset))
	fmt.Printf("slurped n bytes: %v off of get file %v to accomodate offset\n", n, url)
	if err != nil {
		return nil, err
	}
	//return newBackoffReadCloser(c, getObjectOutput.Body), nil
	//	return resp.Body, nil
	return newBackoffReadCloserForRealsies(url, c, resp.Body), nil
}

func (c *amazonClient) Delete(name string) error {
	_, err := c.s3.DeleteObject(&s3.DeleteObjectInput{
		Bucket: aws.String(c.bucket),
		Key:    aws.String(name),
	})
	return err
}

func (c *amazonClient) Exists(name string) bool {
	_, err := c.s3.HeadObject(&s3.HeadObjectInput{
		Bucket: aws.String(c.bucket),
		Key:    aws.String(name),
	})
	return err == nil
}

func (c *amazonClient) isRetryable(err error) (retVal bool) {
	fmt.Printf("is err (%v) retryable?\n", err)
	defer func() {
		fmt.Printf("err (%v) retryable? %v\n", retVal, err)
	}()
	if strings.Contains(err.Error(), "unexpected EOF") {
		return false
	}

	awsErr, ok := err.(awserr.Error)
	if !ok {
		return false
	}
	for _, c := range []string{
		storagegateway.ErrorCodeServiceUnavailable,
		storagegateway.ErrorCodeInternalError,
		storagegateway.ErrorCodeGatewayInternalError,
	} {
		if c == awsErr.Code() {
			return true
		}
	}
	return false
}

func (c *amazonClient) IsIgnorable(err error) bool {
	return false
}

func (c *amazonClient) IsNotExist(err error) bool {
	fmt.Printf("IsNotExist? error: %v\n", err)
	// cloudfront returns forbidden error for nonexisting data
	if strings.Contains(err.Error(), "error code 403") {
		fmt.Printf("its a 403, dne")
		return true
	}
	if strings.Contains(err.Error(), "error code 404") {
		fmt.Printf("its a 404, dne")
		return true
	}
	awsErr, ok := err.(awserr.Error)
	if !ok {
		return false
	}
	if awsErr.Code() == "NoSuchKey" {
		return true
	}
	return false
}

type amazonWriter struct {
	errChan chan error
	pipe    *io.PipeWriter
}

func newWriter(client *amazonClient, name string) *amazonWriter {
	reader, writer := io.Pipe()
	w := &amazonWriter{
		errChan: make(chan error),
		pipe:    writer,
	}
	go func() {
		_, err := client.uploader.Upload(&s3manager.UploadInput{
			Body:            reader,
			Bucket:          aws.String(client.bucket),
			Key:             aws.String(name),
			ContentEncoding: aws.String("application/octet-stream"),
		})
		w.errChan <- err
	}()
	return w
}

func (w *amazonWriter) Write(p []byte) (int, error) {
	return w.pipe.Write(p)
}

func (w *amazonWriter) Close() error {
	if err := w.pipe.Close(); err != nil {
		return err
	}
	return <-w.errChan
}
<|endoftext|>


<commit_before>package worker

import (
	"fmt"
	"io"
	"sort"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pps"
)

// DatumIterator is an interface which allows you to iterate through the datums
// for a job. A datum iterator keeps track of which datum it is on, which can be Reset()
// The intended use is by using this pattern `for di.Next() { ... datum := di.Datum() ... }`
// Note that since you start the loop by a call to Next(), the datum iterator's location starts at -1
type DatumIterator interface {
	Reset()
	Len() int
	Next() bool
	Datum() []*Input
	DatumN(int) []*Input
}

type pfsDatumIterator struct {
	inputs   []*Input
	location int
}

func newPFSDatumIterator(pachClient *client.APIClient, input *pps.PFSInput) (DatumIterator, error) {
	result := &pfsDatumIterator{}
	// make sure it gets initialized properly
	result.Reset()
	if input.Commit == "" {
		// this can happen if a pipeline with multiple inputs has been triggered
		// before all commits have inputs
		return result, nil
	}
	fs, err := pachClient.GlobFileStream(pachClient.Ctx(), &pfs.GlobFileRequest{
		Commit:  client.NewCommit(input.Repo, input.Commit),
		Pattern: input.Glob,
	})
	if err != nil {
		return nil, err
	}
	for {
		fileInfo, err := fs.Recv()
		if err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}
		result.inputs = append(result.inputs, &Input{
			FileInfo:   fileInfo,
			Name:       input.Name,
			Lazy:       input.Lazy,
			Branch:     input.Branch,
			EmptyFiles: input.EmptyFiles,
		})
	}
	// We sort the inputs so that the order is deterministic. Note that it's
	// not possible for 2 inputs to have the same path so this is guaranteed to
	// produce a deterministic order.
	sort.Slice(result.inputs, func(i, j int) bool {
		// We sort by descending size first because it can boost performance to
		// process the biggest datums first.
		if result.inputs[i].FileInfo.SizeBytes != result.inputs[j].FileInfo.SizeBytes {
			return result.inputs[i].FileInfo.SizeBytes > result.inputs[j].FileInfo.SizeBytes
		}
		return result.inputs[i].FileInfo.File.Path < result.inputs[j].FileInfo.File.Path
	})
	return result, nil
}

func (d *pfsDatumIterator) Reset() {
	d.location = -1
}

func (d *pfsDatumIterator) Len() int {
	return len(d.inputs)
}

func (d *pfsDatumIterator) Datum() []*Input {
	return []*Input{d.inputs[d.location]}
}

func (d *pfsDatumIterator) DatumN(n int) []*Input {
	if n > d.location {
		d.Reset()
	}
	for d.location != n {
		d.Next()
	}
	return d.Datum()
}

func (d *pfsDatumIterator) Next() bool {
	d.location++
	return d.location < len(d.inputs)
}

type unionDatumIterator struct {
	iterators []DatumIterator
	unionIdx  int
	location  int
}

func newUnionDatumIterator(pachClient *client.APIClient, union []*pps.Input) (DatumIterator, error) {
	result := &unionDatumIterator{}
	defer result.Reset()
	for _, input := range union {
		datumIterator, err := NewDatumIterator(pachClient, input)
		if err != nil {
			return nil, err
		}
		result.iterators = append(result.iterators, datumIterator)
	}
	return result, nil
}

func (d *unionDatumIterator) Reset() {
	for _, input := range d.iterators {
		input.Reset()
	}
	d.unionIdx = 0
	d.location = -1
}

func (d *unionDatumIterator) Len() int {
	result := 0
	for _, datumIterator := range d.iterators {
		result += datumIterator.Len()
	}
	return result
}

func (d *unionDatumIterator) Next() bool {
	d.location++
	if d.unionIdx >= len(d.iterators) {
		return false
	}
	if !d.iterators[d.unionIdx].Next() {
		d.unionIdx++
		return d.Next()
	}
	return true
}

func (d *unionDatumIterator) Datum() []*Input {
	return d.iterators[d.unionIdx].Datum()
}

func (d *unionDatumIterator) DatumN(n int) []*Input {
	if n > d.location {
		d.Reset()
	}
	for d.location != n {
		d.Next()
	}
	return d.Datum()
}

type crossDatumIterator struct {
	iterators []DatumIterator
	started   bool
	location  int
}

func newCrossDatumIterator(pachClient *client.APIClient, cross []*pps.Input) (DatumIterator, error) {
	result := &crossDatumIterator{}
	defer result.Reset()
	for _, iterator := range cross {
		datumIterator, err := NewDatumIterator(pachClient, iterator)
		if err != nil {
			return nil, err
		}
		result.iterators = append(result.iterators, datumIterator)
	}
	result.location = -1
	return result, nil
}

func (d *crossDatumIterator) Reset() {
	inhabited := len(d.iterators) > 0
	for _, iterators := range d.iterators {
		iterators.Reset()
		if !iterators.Next() {
			inhabited = false
		}
	}
	if !inhabited {
		d.iterators = nil
	}
	d.started = !inhabited
}

func (d *crossDatumIterator) Len() int {
	if len(d.iterators) == 0 {
		return 0
	}
	result := d.iterators[0].Len()
	for i := 1; i < len(d.iterators); i++ {
		result *= d.iterators[i].Len()
	}
	return result
}

func (d *crossDatumIterator) Next() bool {
	d.location++
	if !d.started {
		d.started = true
		return true
	}
	for _, input := range d.iterators {
		// if we're at the end of the "row"
		if !input.Next() {
			// we reset the "row"
			input.Reset()
			// and start it back up
			input.Next()
			// after resetting this "row", start iterating through the next "row"
		} else {
			return true
		}
	}
	return false
}

func (d *crossDatumIterator) Datum() []*Input {
	var result []*Input
	for _, datumIterator := range d.iterators {
		result = append(result, datumIterator.Datum()...)
	}
	sortInputs(result)
	return result
}

func (d *crossDatumIterator) DatumN(n int) []*Input {
	if n > d.location {
		d.Reset()
	}
	for d.location != n {
		d.Next()
	}
	return d.Datum()
}

type joinDatumIterator struct {
	inputs   []DatumIterator
	started  bool
	location int
}

func newJoinDatumIterator(pachClient *client.APIClient, join []*pps.Input) (DatumIterator, error) {
	result := &joinDatumIterator{}
	for _, input := range join {
		datumIterator, err := NewDatumIterator(pachClient, input)
		if err != nil {
			return nil, err
		}
		// start the iterator, and make sure it isn't empty
		if !datumIterator.Next() {
			return &joinDatumIterator{}, nil
		}
		result.inputs = append(result.inputs, datumIterator)
	}
	result.location = -1
	return result, nil
}

func (d *joinDatumIterator) Reset() {
	for _, input := range d.inputs {
		input.Reset()
		input.Next()
	}
	d.started = false
}

func (d *joinDatumIterator) Len() int {
	if len(d.inputs) == 0 {
		return 0
	}
	result := d.inputs[0].Len()
	for i := 1; i < len(d.inputs); i++ {
		result *= d.inputs[i].Len()
	}
	return result
}

func (d *joinDatumIterator) Next() bool {
	d.location++
	if !d.started {
		d.started = true
		return true
	}
	for _, input := range d.inputs {
		// if we're at the end of the "row"
		if !input.Next() {
			// we reset the "row"
			input.Reset()
			// and start back it up
			input.Next()
			// after resetting this "row", start iterating through the next "row"
		} else {
			return true
		}
	}
	return false
}

func (d *joinDatumIterator) Datum() []*Input {
	var result []*Input
	for _, datumIterator := range d.inputs {
		result = append(result, datumIterator.Datum()...)
	}
	sortInputs(result)
	return result
}

func (d *joinDatumIterator) DatumN(n int) []*Input {
	if n > d.location {
		d.Reset()
	}
	for d.location != n {
		d.Next()
	}
	return d.Datum()
}

type gitDatumIterator struct {
	inputs   []*Input
	location int
}

func newGitDatumIterator(pachClient *client.APIClient, input *pps.GitInput) (DatumIterator, error) {
	result := &gitDatumIterator{}
	defer result.Reset()
	if input.Commit == "" {
		// this can happen if a pipeline with multiple inputs has been triggered
		// before all commits have inputs
		return result, nil
	}
	fileInfo, err := pachClient.InspectFile(input.Name, input.Commit, "/commit.json")
	if err != nil {
		return nil, err
	}
	result.inputs = append(
		result.inputs,
		&Input{
			FileInfo: fileInfo,
			Name:     input.Name,
			Branch:   input.Branch,
			GitURL:   input.URL,
		},
	)
	return result, nil
}

func (d *gitDatumIterator) Reset() {
	d.location = -1
}

func (d *gitDatumIterator) Len() int {
	return len(d.inputs)
}

func (d *gitDatumIterator) Datum() []*Input {
	return []*Input{d.inputs[d.location]}
}

func (d *gitDatumIterator) Next() bool {
	d.location++
	return d.location < len(d.inputs)
}

func (d *gitDatumIterator) DatumN(n int) []*Input {
	if n > d.location {
		d.Reset()
	}
	for d.location != n {
		d.Next()
	}
	return d.Datum()
}

func newCronDatumIterator(pachClient *client.APIClient, input *pps.CronInput) (DatumIterator, error) {
	return newPFSDatumIterator(pachClient, &pps.PFSInput{
		Name:   input.Name,
		Repo:   input.Repo,
		Branch: "master",
		Commit: input.Commit,
		Glob:   "/*",
	})
}

// NewDatumIterator creates a datumIterator for an input.
func NewDatumIterator(pachClient *client.APIClient, input *pps.Input) (DatumIterator, error) {
	switch {
	case input.Pfs != nil:
		return newPFSDatumIterator(pachClient, input.Pfs)
	case input.Union != nil:
		return newUnionDatumIterator(pachClient, input.Union)
	case input.Cross != nil:
		return newCrossDatumIterator(pachClient, input.Cross)
	case input.Cron != nil:
		return newCronDatumIterator(pachClient, input.Cron)
	case input.Git != nil:
		return newGitDatumIterator(pachClient, input.Git)
	}
	return nil, fmt.Errorf("unrecognized input type")
}

func sortInputs(inputs []*Input) {
	sort.Slice(inputs, func(i, j int) bool {
		return inputs[i].Name < inputs[j].Name
	})
}
<commit_msg>first stab at implementing join datum iterators<commit_after>package worker

import (
	"fmt"
	"io"
	"sort"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pps"
)

// DatumIterator is an interface which allows you to iterate through the datums
// for a job. A datum iterator keeps track of which datum it is on, which can be Reset()
// The intended use is by using this pattern `for di.Next() { ... datum := di.Datum() ... }`
// Note that since you start the loop by a call to Next(), the datum iterator's location starts at -1
type DatumIterator interface {
	Reset()
	Len() int
	Next() bool
	Datum() []*Input
	DatumN(int) []*Input
}

type pfsDatumIterator struct {
	inputs   []*Input
	location int
}

func newPFSDatumIterator(pachClient *client.APIClient, input *pps.PFSInput) (DatumIterator, error) {
	result := &pfsDatumIterator{}
	// make sure it gets initialized properly
	result.Reset()
	if input.Commit == "" {
		// this can happen if a pipeline with multiple inputs has been triggered
		// before all commits have inputs
		return result, nil
	}
	fs, err := pachClient.GlobFileStream(pachClient.Ctx(), &pfs.GlobFileRequest{
		Commit:  client.NewCommit(input.Repo, input.Commit),
		Pattern: input.Glob,
	})
	if err != nil {
		return nil, err
	}
	for {
		fileInfo, err := fs.Recv()
		if err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}
		result.inputs = append(result.inputs, &Input{
			FileInfo:   fileInfo,
			Name:       input.Name,
			Lazy:       input.Lazy,
			Branch:     input.Branch,
			EmptyFiles: input.EmptyFiles,
		})
	}
	// We sort the inputs so that the order is deterministic. Note that it's
	// not possible for 2 inputs to have the same path so this is guaranteed to
	// produce a deterministic order.
	sort.Slice(result.inputs, func(i, j int) bool {
		// We sort by descending size first because it can boost performance to
		// process the biggest datums first.
		if result.inputs[i].FileInfo.SizeBytes != result.inputs[j].FileInfo.SizeBytes {
			return result.inputs[i].FileInfo.SizeBytes > result.inputs[j].FileInfo.SizeBytes
		}
		return result.inputs[i].FileInfo.File.Path < result.inputs[j].FileInfo.File.Path
	})
	return result, nil
}

func (d *pfsDatumIterator) Reset() {
	d.location = -1
}

func (d *pfsDatumIterator) Len() int {
	return len(d.inputs)
}

func (d *pfsDatumIterator) Datum() []*Input {
	return []*Input{d.inputs[d.location]}
}

func (d *pfsDatumIterator) DatumN(n int) []*Input {
	if n > d.location {
		d.Reset()
	}
	for d.location != n {
		d.Next()
	}
	return d.Datum()
}

func (d *pfsDatumIterator) Next() bool {
	d.location++
	return d.location < len(d.inputs)
}

type unionDatumIterator struct {
	iterators []DatumIterator
	unionIdx  int
	location  int
}

func newUnionDatumIterator(pachClient *client.APIClient, union []*pps.Input) (DatumIterator, error) {
	result := &unionDatumIterator{}
	defer result.Reset()
	for _, input := range union {
		datumIterator, err := NewDatumIterator(pachClient, input)
		if err != nil {
			return nil, err
		}
		result.iterators = append(result.iterators, datumIterator)
	}
	return result, nil
}

func (d *unionDatumIterator) Reset() {
	for _, input := range d.iterators {
		input.Reset()
	}
	d.unionIdx = 0
	d.location = -1
}

func (d *unionDatumIterator) Len() int {
	result := 0
	for _, datumIterator := range d.iterators {
		result += datumIterator.Len()
	}
	return result
}

func (d *unionDatumIterator) Next() bool {
	d.location++
	if d.unionIdx >= len(d.iterators) {
		return false
	}
	if !d.iterators[d.unionIdx].Next() {
		d.unionIdx++
		return d.Next()
	}
	return true
}

func (d *unionDatumIterator) Datum() []*Input {
	return d.iterators[d.unionIdx].Datum()
}

func (d *unionDatumIterator) DatumN(n int) []*Input {
	if n > d.location {
		d.Reset()
	}
	for d.location != n {
		d.Next()
	}
	return d.Datum()
}

type crossDatumIterator struct {
	iterators []DatumIterator
	started   bool
	location  int
}

func newCrossDatumIterator(pachClient *client.APIClient, cross []*pps.Input) (DatumIterator, error) {
	result := &crossDatumIterator{}
	defer result.Reset()
	for _, iterator := range cross {
		datumIterator, err := NewDatumIterator(pachClient, iterator)
		if err != nil {
			return nil, err
		}
		result.iterators = append(result.iterators, datumIterator)
	}
	result.location = -1
	return result, nil
}

func (d *crossDatumIterator) Reset() {
	inhabited := len(d.iterators) > 0
	for _, iterators := range d.iterators {
		iterators.Reset()
		if !iterators.Next() {
			inhabited = false
		}
	}
	if !inhabited {
		d.iterators = nil
	}
	d.started = !inhabited
}

func (d *crossDatumIterator) Len() int {
	if len(d.iterators) == 0 {
		return 0
	}
	result := d.iterators[0].Len()
	for i := 1; i < len(d.iterators); i++ {
		result *= d.iterators[i].Len()
	}
	return result
}

func (d *crossDatumIterator) Next() bool {
	d.location++
	if !d.started {
		d.started = true
		return true
	}
	for _, input := range d.iterators {
		// if we're at the end of the "row"
		if !input.Next() {
			// we reset the "row"
			input.Reset()
			// and start it back up
			input.Next()
			// after resetting this "row", start iterating through the next "row"
		} else {
			return true
		}
	}
	return false
}

func (d *crossDatumIterator) Datum() []*Input {
	var result []*Input
	for _, datumIterator := range d.iterators {
		result = append(result, datumIterator.Datum()...)
	}
	sortInputs(result)
	return result
}

func (d *crossDatumIterator) DatumN(n int) []*Input {
	if n > d.location {
		d.Reset()
	}
	for d.location != n {
		d.Next()
	}
	return d.Datum()
}

type joinDatumIterator struct {
	datums   [][]*Input
	location int
}

func newJoinDatumIterator(pachClient *client.APIClient, join []*pps.Input) (DatumIterator, error) {
	cross, err := newCrossDatumIterator(pachClient, join)
	if err != nil {
		return nil, err
	}
	result := &joinDatumIterator{}
	for cross.Next() {
		tuple := cross.Datum()
		count := make(map[string]int)
		for _, input := range tuple {
			count[input.MatchOn]++
		}
		if len(count) == 1 {
			result.datums = append(result.datums, tuple)
		}
	}
	result.location = -1
	return result, nil
}

func (d *joinDatumIterator) Reset() {
	d.location = -1
}

func (d *joinDatumIterator) Len() int {
	return len(d.datums)
}

func (d *joinDatumIterator) Next() bool {
	d.location++
	return d.location < len(d.datums)
}

func (d *joinDatumIterator) Datum() []*Input {
	var result []*Input
	for _, datum := range d.datums[d.location] {
		result = append(result, datum)
	}
	sortInputs(result)
	return result
}

func (d *joinDatumIterator) DatumN(n int) []*Input {
	if n > d.location {
		d.Reset()
	}
	for d.location != n {
		d.Next()
	}
	return d.Datum()
}

type gitDatumIterator struct {
	inputs   []*Input
	location int
}

func newGitDatumIterator(pachClient *client.APIClient, input *pps.GitInput) (DatumIterator, error) {
	result := &gitDatumIterator{}
	defer result.Reset()
	if input.Commit == "" {
		// this can happen if a pipeline with multiple inputs has been triggered
		// before all commits have inputs
		return result, nil
	}
	fileInfo, err := pachClient.InspectFile(input.Name, input.Commit, "/commit.json")
	if err != nil {
		return nil, err
	}
	result.inputs = append(
		result.inputs,
		&Input{
			FileInfo: fileInfo,
			Name:     input.Name,
			Branch:   input.Branch,
			GitURL:   input.URL,
		},
	)
	return result, nil
}

func (d *gitDatumIterator) Reset() {
	d.location = -1
}

func (d *gitDatumIterator) Len() int {
	return len(d.inputs)
}

func (d *gitDatumIterator) Datum() []*Input {
	return []*Input{d.inputs[d.location]}
}

func (d *gitDatumIterator) Next() bool {
	d.location++
	return d.location < len(d.inputs)
}

func (d *gitDatumIterator) DatumN(n int) []*Input {
	if n > d.location {
		d.Reset()
	}
	for d.location != n {
		d.Next()
	}
	return d.Datum()
}

func newCronDatumIterator(pachClient *client.APIClient, input *pps.CronInput) (DatumIterator, error) {
	return newPFSDatumIterator(pachClient, &pps.PFSInput{
		Name:   input.Name,
		Repo:   input.Repo,
		Branch: "master",
		Commit: input.Commit,
		Glob:   "/*",
	})
}

// NewDatumIterator creates a datumIterator for an input.
func NewDatumIterator(pachClient *client.APIClient, input *pps.Input) (DatumIterator, error) {
	switch {
	case input.Pfs != nil:
		return newPFSDatumIterator(pachClient, input.Pfs)
	case input.Union != nil:
		return newUnionDatumIterator(pachClient, input.Union)
	case input.Cross != nil:
		return newCrossDatumIterator(pachClient, input.Cross)
	case input.Cron != nil:
		return newCronDatumIterator(pachClient, input.Cron)
	case input.Git != nil:
		return newGitDatumIterator(pachClient, input.Git)
	}
	return nil, fmt.Errorf("unrecognized input type")
}

func sortInputs(inputs []*Input) {
	sort.Slice(inputs, func(i, j int) bool {
		return inputs[i].Name < inputs[j].Name
	})
}
<|endoftext|>


<commit_before>package logger

import (
	"fmt"
	"github.com/DanielRenne/GoCore/core/extensions"
	"github.com/DanielRenne/GoCore/core/serverSettings"
	"github.com/DanielRenne/GoCore/core/utils"
	"github.com/fatih/color"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"log"
	"math/rand"
	"sync"
	"sync/atomic"
	"time"
)

type Color int

var VerboseBornAndDeadGophers bool
var TotalSystemGoRoutines int32
var RunningGophers []string
var GopherTimeRunning map[string]time.Time
var gopherMutex sync.RWMutex

const (
	RED     = 1
	GREEN   = 2
	YELLOW  = 3
	BLUE    = 4
	MAGENTA = 5
	CYAN    = 6
	WHITE   = 7
)

func init() {
	//VerboseBornAndDeadGophers= true
	rand.Seed(time.Now().UnixNano())
	RunningGophers = utils.Array()
	GopherTimeRunning = make(map[string]time.Time, 0)
	go func() {
		time.Sleep(time.Minute * 1)
		for {
			serverSettings.WebConfigMutex.RLock()
			if serverSettings.WebConfig.Application.LogGophers {
				ViewRunningGophers()
			}
			serverSettings.WebConfigMutex.RUnlock()
			time.Sleep(time.Minute * 1)
		}
	}()
}

func Message(message string, c Color) {
	switch c {
	case 1:
		color.Red(message)
	case 2:
		color.Green(message)
	case 3:
		color.Yellow(message)
	case 4:
		color.Blue(message)
	case 5:
		color.Magenta(message)
	case 6:
		color.Cyan(message)
	case 7:
		color.White(message)
	}
}

func deferGoRoutine(routineDesc string, goRoutineIdStarted int32, id string) {
	if VerboseBornAndDeadGophers {
		log.Println(TimeTrack(time.Now(), time.Now().String()+" "+id+" finished ["+routineDesc+"] died ;.-("))
	}
	atomic.AddInt32(&TotalSystemGoRoutines, -1)
	gopherMutex.Lock()
	RunningGophers = utils.ArrayRemove(RunningGophers, id+"-> ("+routineDesc+")")
	gopherMutex.Unlock()

}

func getGopherGender() string {
	if rand.Intn(2) == 1 {
		return "Mrs."
	} else {
		return "Mr."
	}
}

func ViewRunningGophers() {
	gopherMutex.RLock()
	if len(RunningGophers) > 0 {
		log.Println(`
           ,_---~~~~~----._
    _,,_,*^____      _____ -g--"*,
   / __/ /'     ^.  /      \ ^@q  f
  [  @f | @))    |  | @))   l  0 _/
   \ /   \~____ / __ \_____/    \
    |           _l__l_           I
    }          [______]           I
    |            | | |            |
    ]             ~ ~             |
    |                            |
     |                           |
at ` + time.Now().String() + " " + extensions.IntToString(len(RunningGophers)) + ` Gophers workin up in here!
`)
		for i, gopher := range RunningGophers {
			val, ok := GopherTimeRunning[gopher]
			var timeRunning string
			if ok {
				timeRunning = " (" + time.Since(val).String() + " elapsed)"
			}
			log.Println("#" + extensions.IntToString(i) + ":" + gopher + timeRunning)
		}
	} else {
		log.Println(time.Now().String() + " no gophers in memory yay!")
	}
	gopherMutex.RUnlock()
}

func GoRoutineLogger(fn func(), routineDesc string) {
	if serverSettings.WebConfig.Application.LogGophers {
		id := getGopherGender() + utils.RandStringRunes(5)
		gopherMutex.Lock()
		descId := id + "-> (" + routineDesc + ")"
		GopherTimeRunning[descId] = time.Now()
		RunningGophers = append(RunningGophers, descId)
		gopherMutex.Unlock()
		atomic.AddInt32(&TotalSystemGoRoutines, 1)
		goRoutineIdStarted := atomic.LoadInt32(&TotalSystemGoRoutines)
		defer deferGoRoutine(routineDesc, goRoutineIdStarted, id)
		if VerboseBornAndDeadGophers {
			log.Println(time.Now().String() + " " + id + " is starting to [" + routineDesc + "]")
		}
	}
	fn()
}

func TimeTrack(start time.Time, name string) (log string) {
	elapsed := time.Since(start)
	//if elapsed.Seconds() > 1 {
	//}
	return fmt.Sprintf("<Timing>%s took %s</Timing>", name, elapsed)
}

func TimeTrackQuery(start time.Time, name string, collection *mgo.Collection, m bson.M, q *mgo.Query) (log string) {
	elapsed := time.Since(start)
	log += "<Timing>\n\n"
	log += fmt.Sprintf("%#v", collection) + "\n\n"
	log += fmt.Sprintf("%#v", m) + "\n\n"
	log += fmt.Sprintf("%#v", q) + "\n\n"
	log += fmt.Sprintf("%s took %s", name, elapsed) + "\n</Timing>\n"
	return
}
<commit_msg>increase speed of my gopher friends<commit_after>package logger

import (
	"fmt"
	"github.com/DanielRenne/GoCore/core/extensions"
	"github.com/DanielRenne/GoCore/core/serverSettings"
	"github.com/DanielRenne/GoCore/core/utils"
	"github.com/fatih/color"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"log"
	"math/rand"
	"sync"
	"sync/atomic"
	"time"
)

type Color int

var VerboseBornAndDeadGophers bool
var TotalSystemGoRoutines int32
var RunningGophers []string
var GopherTimeRunning map[string]time.Time
var gopherMutex sync.RWMutex

const (
	RED     = 1
	GREEN   = 2
	YELLOW  = 3
	BLUE    = 4
	MAGENTA = 5
	CYAN    = 6
	WHITE   = 7
)

func init() {
	//VerboseBornAndDeadGophers= true
	rand.Seed(time.Now().UnixNano())
	RunningGophers = utils.Array()
	GopherTimeRunning = make(map[string]time.Time, 0)
	go func() {
		time.Sleep(time.Second * 15)
		for {
			serverSettings.WebConfigMutex.RLock()
			if serverSettings.WebConfig.Application.LogGophers {
				ViewRunningGophers()
			}
			serverSettings.WebConfigMutex.RUnlock()
			time.Sleep(time.Second * 15)
		}
	}()
}

func Message(message string, c Color) {
	switch c {
	case 1:
		color.Red(message)
	case 2:
		color.Green(message)
	case 3:
		color.Yellow(message)
	case 4:
		color.Blue(message)
	case 5:
		color.Magenta(message)
	case 6:
		color.Cyan(message)
	case 7:
		color.White(message)
	}
}

func deferGoRoutine(routineDesc string, goRoutineIdStarted int32, id string) {
	if VerboseBornAndDeadGophers {
		log.Println(TimeTrack(time.Now(), time.Now().String()+" "+id+" finished ["+routineDesc+"] died ;.-("))
	}
	atomic.AddInt32(&TotalSystemGoRoutines, -1)
	gopherMutex.Lock()
	RunningGophers = utils.ArrayRemove(RunningGophers, id+"-> ("+routineDesc+")")
	gopherMutex.Unlock()

}

func getGopherGender() string {
	if rand.Intn(2) == 1 {
		return "Mrs."
	} else {
		return "Mr."
	}
}

func ViewRunningGophers() {
	gopherMutex.RLock()
	if len(RunningGophers) > 0 {
		log.Println(`
           ,_---~~~~~----._
    _,,_,*^____      _____ -g--"*,
   / __/ /'     ^.  /      \ ^@q  f
  [  @f | @))    |  | @))   l  0 _/
   \ /   \~____ / __ \_____/    \
    |           _l__l_           I
    }          [______]           I
    |            | | |            |
    ]             ~ ~             |
    |                            |
     |                           |
at ` + time.Now().String() + " " + extensions.IntToString(len(RunningGophers)) + ` Gophers workin up in here!
`)
		for i, gopher := range RunningGophers {
			val, ok := GopherTimeRunning[gopher]
			var timeRunning string
			if ok {
				timeRunning = " (" + time.Since(val).String() + " elapsed)"
			}
			log.Println("#" + extensions.IntToString(i) + ":" + gopher + timeRunning)
		}
	} else {
		log.Println(time.Now().String() + " no gophers in memory yay!")
	}
	gopherMutex.RUnlock()
}

func GoRoutineLogger(fn func(), routineDesc string) {
	if serverSettings.WebConfig.Application.LogGophers {
		id := getGopherGender() + utils.RandStringRunes(5)
		gopherMutex.Lock()
		descId := id + "-> (" + routineDesc + ")"
		GopherTimeRunning[descId] = time.Now()
		RunningGophers = append(RunningGophers, descId)
		gopherMutex.Unlock()
		atomic.AddInt32(&TotalSystemGoRoutines, 1)
		goRoutineIdStarted := atomic.LoadInt32(&TotalSystemGoRoutines)
		defer deferGoRoutine(routineDesc, goRoutineIdStarted, id)
		if VerboseBornAndDeadGophers {
			log.Println(time.Now().String() + " " + id + " is starting to [" + routineDesc + "]")
		}
	}
	fn()
}

func TimeTrack(start time.Time, name string) (log string) {
	elapsed := time.Since(start)
	//if elapsed.Seconds() > 1 {
	//}
	return fmt.Sprintf("<Timing>%s took %s</Timing>", name, elapsed)
}

func TimeTrackQuery(start time.Time, name string, collection *mgo.Collection, m bson.M, q *mgo.Query) (log string) {
	elapsed := time.Since(start)
	log += "<Timing>\n\n"
	log += fmt.Sprintf("%#v", collection) + "\n\n"
	log += fmt.Sprintf("%#v", m) + "\n\n"
	log += fmt.Sprintf("%#v", q) + "\n\n"
	log += fmt.Sprintf("%s took %s", name, elapsed) + "\n</Timing>\n"
	return
}
<|endoftext|>


<commit_before>package release

import (
	"crypto"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"runtime"

	"github.com/hashicorp/go-version"
	binaryUpdate "github.com/inconshreveable/go-update"

	"github.com/Shopify/themekit/src/colors"
)

var (
	builds = map[string]string{
		"darwin-amd64":  "theme",
		"darwin-386":    "theme",
		"linux-386":     "theme",
		"linux-amd64":   "theme",
		"freebsd-386":   "theme",
		"freebsd-amd64": "theme",
		"windows-386":   "theme.exe",
		"windows-amd64": "theme.exe",
	}
	// ThemeKitVersion is the version build of the library
	ThemeKitVersion, _ = version.NewVersion("1.0.1")
)

const (
	releasesS3URL = "https://shopify-themekit.s3.amazonaws.com/releases/all.json"
	latestS3URL   = "https://shopify-themekit.s3.amazonaws.com/releases/latest.json"
)

type release struct {
	Version   string     `json:"version"`
	Platforms []platform `json:"platforms"`
}

func (r release) isValid() bool {
	return len(r.Platforms) > 0
}

func (r release) isApplicable() bool {
	version, err := version.NewVersion(r.Version)
	if err != nil {
		return false
	}
	return ThemeKitVersion.LessThan(version) && version.Metadata() == "" && version.Prerelease() == ""
}

func (r release) getVersion() *version.Version {
	version, _ := version.NewVersion(r.Version)
	return version
}

func (r release) forCurrentPlatform() platform {
	platformKey := runtime.GOOS + "-" + runtime.GOARCH
	for _, p := range r.Platforms {
		if p.Name == platformKey {
			return p
		}
	}
	return platform{}
}

// IsUpdateAvailable will check if there is an update to the theme kit command
// and if there is one it will return true. Otherwise it will return false.
func IsUpdateAvailable() bool {
	return checkUpdateAvailable(latestS3URL)
}

// Install will take a semver string and parse it then check if that
// update is available and install it. If the string is 'latest' it will install
// the most current. If the string is latest and there is no update it will return an
// error. An error will also be returned if the requested version does not exist.
func Install(ver string) error {
	installer := func(p platform) error {
		return applyUpdate(p, "")
	}
	if ver == "latest" {
		return installLatest(latestS3URL, installer)
	}
	return installVersion(ver, releasesS3URL, installer)
}

// Update will update the details of a release or deploy a new release with the
// deploy feed
func Update(key, secret, ver string, force bool) error {
	return update(ver, releasesS3URL, filepath.Join("build", "dist"), force, newS3Uploader(key, secret))
}

// Remove will remove a themekit release from the deployed releases list. This will
// prevent any users from installing the version again. This can only be done will
// appropriate S3 priviledges
func Remove(key, secret, ver string) error {
	return remove(ver, releasesS3URL, newS3Uploader(key, secret))
}

func checkUpdateAvailable(latestURL string) bool {
	release, err := fetchLatest(latestURL)
	if err != nil {
		return false
	}
	return release.isApplicable()
}

func installLatest(latestURL string, install func(platform) error) error {
	release, err := fetchLatest(latestURL)
	if err != nil {
		return err
	}
	if !release.isApplicable() {
		return fmt.Errorf("no applicable update available")
	}
	return install(release.forCurrentPlatform())
}

func installVersion(ver, releasesURL string, install func(platform) error) error {
	if _, err := version.NewVersion(ver); err != nil {
		return err
	}

	releases, err := fetchReleases(releasesURL)
	if err != nil {
		return err
	}
	requestedRelease := releases.get(ver)
	if !requestedRelease.isValid() {
		return fmt.Errorf("version %s not found", ver)
	}
	return install(requestedRelease.forCurrentPlatform())
}

func applyUpdate(platformRelease platform, targetPath string) error {
	checksum, err := hex.DecodeString(platformRelease.Digest)
	if err != nil {
		return err
	}

	updateFile, err := http.Get(platformRelease.URL)
	if err != nil {
		return err
	}
	defer updateFile.Body.Close()

	err = binaryUpdate.Apply(updateFile.Body, binaryUpdate.Options{
		TargetPath: targetPath,
		Hash:       crypto.MD5,
		Checksum:   checksum,
	})

	if err != nil {
		if rerr := binaryUpdate.RollbackError(err); rerr != nil {
			return fmt.Errorf("Failed to rollback from bad update: %v", rerr)
		}
		return fmt.Errorf("Could not update and had to roll back. %v", err)
	}

	return nil
}

func update(ver, releasesURL, distDir string, force bool, u uploader) error {
	if _, err := version.NewVersion(ver); err != nil {
		return err
	}

	if !force {
		requestedVersion, _ := version.NewVersion(ver)
		if !requestedVersion.Equal(ThemeKitVersion) {
			return errors.New("deploy version does not match themekit version")
		}
	}

	_, err := os.Stat(distDir)
	if os.IsNotExist(err) {
		return errors.New("Dist folder does not exist. Run 'make dist' before attempting to create a new release")
	}

	releases, err := fetchReleases(releasesURL)
	if err != nil {
		return err
	}

	if requestedRelease := releases.get(ver); !force && requestedRelease.isValid() {
		return errors.New("version has already been deployed")
	}

	newRelease, err := buildRelease(ver, distDir, u)
	if err != nil {
		return err
	}

	return updateDeploy(releases.add(newRelease), u)
}

func remove(ver, releaseURL string, u uploader) error {
	if _, err := version.NewVersion(ver); err != nil {
		return err
	}

	releases, err := fetchReleases(releaseURL)
	if err != nil {
		return err
	}

	requestedRelease := releases.get(ver)
	if !requestedRelease.isValid() {
		return errors.New("version has not be deployed")
	}

	return updateDeploy(releases.del(ver), u)
}

func updateDeploy(releases releasesList, u uploader) error {
	colors.ColorStdOut.Printf("Updating releases")
	if err := u.JSON("releases/all.json", releases); err != nil {
		return err
	}
	return u.JSON("releases/latest.json", releases.get("latest"))
}

func fetchLatest(url string) (release, error) {
	var latest release
	resp, err := http.Get(url)
	if err != nil {
		return latest, err
	}
	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err == nil {
		err = json.Unmarshal(data, &latest)
	}
	return latest, err
}

func buildRelease(ver, distDir string, u uploader) (release, error) {
	colors.ColorStdOut.Printf("Building %s", colors.Green(ver))
	newRelease := release{Version: ver, Platforms: []platform{}}

	for platformName, binName := range builds {
		plat, err := buildPlatform(ver, platformName, distDir, binName, u)
		if err != nil {
			return newRelease, err
		}
		newRelease.Platforms = append(newRelease.Platforms, plat)
	}

	return newRelease, nil
}
<commit_msg>Verison update<commit_after>package release

import (
	"crypto"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"runtime"

	"github.com/hashicorp/go-version"
	binaryUpdate "github.com/inconshreveable/go-update"

	"github.com/Shopify/themekit/src/colors"
)

var (
	builds = map[string]string{
		"darwin-amd64":  "theme",
		"darwin-386":    "theme",
		"linux-386":     "theme",
		"linux-amd64":   "theme",
		"freebsd-386":   "theme",
		"freebsd-amd64": "theme",
		"windows-386":   "theme.exe",
		"windows-amd64": "theme.exe",
	}
	// ThemeKitVersion is the version build of the library
	ThemeKitVersion, _ = version.NewVersion("1.0.2")
)

const (
	releasesS3URL = "https://shopify-themekit.s3.amazonaws.com/releases/all.json"
	latestS3URL   = "https://shopify-themekit.s3.amazonaws.com/releases/latest.json"
)

type release struct {
	Version   string     `json:"version"`
	Platforms []platform `json:"platforms"`
}

func (r release) isValid() bool {
	return len(r.Platforms) > 0
}

func (r release) isApplicable() bool {
	version, err := version.NewVersion(r.Version)
	if err != nil {
		return false
	}
	return ThemeKitVersion.LessThan(version) && version.Metadata() == "" && version.Prerelease() == ""
}

func (r release) getVersion() *version.Version {
	version, _ := version.NewVersion(r.Version)
	return version
}

func (r release) forCurrentPlatform() platform {
	platformKey := runtime.GOOS + "-" + runtime.GOARCH
	for _, p := range r.Platforms {
		if p.Name == platformKey {
			return p
		}
	}
	return platform{}
}

// IsUpdateAvailable will check if there is an update to the theme kit command
// and if there is one it will return true. Otherwise it will return false.
func IsUpdateAvailable() bool {
	return checkUpdateAvailable(latestS3URL)
}

// Install will take a semver string and parse it then check if that
// update is available and install it. If the string is 'latest' it will install
// the most current. If the string is latest and there is no update it will return an
// error. An error will also be returned if the requested version does not exist.
func Install(ver string) error {
	installer := func(p platform) error {
		return applyUpdate(p, "")
	}
	if ver == "latest" {
		return installLatest(latestS3URL, installer)
	}
	return installVersion(ver, releasesS3URL, installer)
}

// Update will update the details of a release or deploy a new release with the
// deploy feed
func Update(key, secret, ver string, force bool) error {
	return update(ver, releasesS3URL, filepath.Join("build", "dist"), force, newS3Uploader(key, secret))
}

// Remove will remove a themekit release from the deployed releases list. This will
// prevent any users from installing the version again. This can only be done will
// appropriate S3 priviledges
func Remove(key, secret, ver string) error {
	return remove(ver, releasesS3URL, newS3Uploader(key, secret))
}

func checkUpdateAvailable(latestURL string) bool {
	release, err := fetchLatest(latestURL)
	if err != nil {
		return false
	}
	return release.isApplicable()
}

func installLatest(latestURL string, install func(platform) error) error {
	release, err := fetchLatest(latestURL)
	if err != nil {
		return err
	}
	if !release.isApplicable() {
		return fmt.Errorf("no applicable update available")
	}
	return install(release.forCurrentPlatform())
}

func installVersion(ver, releasesURL string, install func(platform) error) error {
	if _, err := version.NewVersion(ver); err != nil {
		return err
	}

	releases, err := fetchReleases(releasesURL)
	if err != nil {
		return err
	}
	requestedRelease := releases.get(ver)
	if !requestedRelease.isValid() {
		return fmt.Errorf("version %s not found", ver)
	}
	return install(requestedRelease.forCurrentPlatform())
}

func applyUpdate(platformRelease platform, targetPath string) error {
	checksum, err := hex.DecodeString(platformRelease.Digest)
	if err != nil {
		return err
	}

	updateFile, err := http.Get(platformRelease.URL)
	if err != nil {
		return err
	}
	defer updateFile.Body.Close()

	err = binaryUpdate.Apply(updateFile.Body, binaryUpdate.Options{
		TargetPath: targetPath,
		Hash:       crypto.MD5,
		Checksum:   checksum,
	})

	if err != nil {
		if rerr := binaryUpdate.RollbackError(err); rerr != nil {
			return fmt.Errorf("Failed to rollback from bad update: %v", rerr)
		}
		return fmt.Errorf("Could not update and had to roll back. %v", err)
	}

	return nil
}

func update(ver, releasesURL, distDir string, force bool, u uploader) error {
	if _, err := version.NewVersion(ver); err != nil {
		return err
	}

	if !force {
		requestedVersion, _ := version.NewVersion(ver)
		if !requestedVersion.Equal(ThemeKitVersion) {
			return errors.New("deploy version does not match themekit version")
		}
	}

	_, err := os.Stat(distDir)
	if os.IsNotExist(err) {
		return errors.New("Dist folder does not exist. Run 'make dist' before attempting to create a new release")
	}

	releases, err := fetchReleases(releasesURL)
	if err != nil {
		return err
	}

	if requestedRelease := releases.get(ver); !force && requestedRelease.isValid() {
		return errors.New("version has already been deployed")
	}

	newRelease, err := buildRelease(ver, distDir, u)
	if err != nil {
		return err
	}

	return updateDeploy(releases.add(newRelease), u)
}

func remove(ver, releaseURL string, u uploader) error {
	if _, err := version.NewVersion(ver); err != nil {
		return err
	}

	releases, err := fetchReleases(releaseURL)
	if err != nil {
		return err
	}

	requestedRelease := releases.get(ver)
	if !requestedRelease.isValid() {
		return errors.New("version has not be deployed")
	}

	return updateDeploy(releases.del(ver), u)
}

func updateDeploy(releases releasesList, u uploader) error {
	colors.ColorStdOut.Printf("Updating releases")
	if err := u.JSON("releases/all.json", releases); err != nil {
		return err
	}
	return u.JSON("releases/latest.json", releases.get("latest"))
}

func fetchLatest(url string) (release, error) {
	var latest release
	resp, err := http.Get(url)
	if err != nil {
		return latest, err
	}
	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err == nil {
		err = json.Unmarshal(data, &latest)
	}
	return latest, err
}

func buildRelease(ver, distDir string, u uploader) (release, error) {
	colors.ColorStdOut.Printf("Building %s", colors.Green(ver))
	newRelease := release{Version: ver, Platforms: []platform{}}

	for platformName, binName := range builds {
		plat, err := buildPlatform(ver, platformName, distDir, binName, u)
		if err != nil {
			return newRelease, err
		}
		newRelease.Platforms = append(newRelease.Platforms, plat)
	}

	return newRelease, nil
}
<|endoftext|>


<commit_before>package controllers

import (
  "seeme/models"
  "seeme/db"

  "io"
  "net/http"
  "encoding/json"
)

func PostNotesController(w http.ResponseWriter, r *http.Request) {
  if _, err := db.GetUser(r.FormValue("username")); err != nil {
    w.Write([]byte("User Not Found!"))
    return
  }

  r.ParseForm()
  var notes models.NotesJSON
  if err := json.NewDecoder(r.Body).Decode(&notes); err != nil && err != io.EOF {
    w.Write([]byte(err.Error()))
    return
  }

  if err := db.UpdateUserNotes(r.FormValue("username"), notes.Notes); err != nil {
    w.Write([]byte(err.Error()))
    return
  }

  w.Write([]byte("Notes Uploaded"))
}<commit_msg>changed spacing<commit_after>package controllers

import (
  "seeme/models"
  "seeme/db"
  "io"
  "net/http"
  "encoding/json"
)

func PostNotesController(w http.ResponseWriter, r *http.Request) {
  if _, err := db.GetUser(r.FormValue("username")); err != nil {
    w.Write([]byte("User Not Found!"))
    return
  }

  r.ParseForm()
  var notes models.NotesJSON
  if err := json.NewDecoder(r.Body).Decode(&notes); err != nil && err != io.EOF {
    w.Write([]byte(err.Error()))
    return
  }

  if err := db.UpdateUserNotes(r.FormValue("username"), notes.Notes); err != nil {
    w.Write([]byte(err.Error()))
    return
  }

  w.Write([]byte("Notes Uploaded"))
}<|endoftext|>


<commit_before>package snakecase

import "testing"

func TestConvert(t *testing.T) {
	s := "GbRocks"
	want := "Gb_Rocks"
	r := Convert(s)

	if want != r {
		t.Error("want %s got %s", want, r)
	}
}
<commit_msg>snakecase: fix tests<commit_after>package snakecase

import "testing"

func TestConvert(t *testing.T) {
	s := "GbRocks"
	want := "gb_rocks"
	r := Convert(s)

	if want != r {
		t.Error("want %s got %s", want, r)
	}
}
<|endoftext|>


<commit_before>package zlogger

import (
	"io"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/macroblock/zl/core/loglevel"
)

// ~d - date when a message occurred
// ~t - time when a message occurred
// ~l - loglevel
// ~m - module name
// ~s - log state
// ~e - error message
// ~x - text message
// example: "~d ~t (~n) ~l: ~x~e\n"
const defaultFormat = "~d ~t (~m) ~l~s~x~e\n"

// TStyler -
type TStyler func(key rune, params *TFormatParams) (string, bool)

// Supported stylers
var (
	DefaultStyler TStyler = defaultStyler
	AnsiStyler    TStyler = ansiStyler
)

// TLogger -
type TLogger struct {
	name         string
	writer       io.Writer
	styler       TStyler
	levelFilter  loglevel.TFilter
	moduleFilter []string
	format       string
}

// LevelFilter -
func (o *TLogger) LevelFilter() loglevel.TFilter {
	return o.levelFilter
}

// ModuleFilter -
func (o *TLogger) ModuleFilter() []string {
	return append([]string(nil), o.moduleFilter...)
}

// Format -
func (o *TLogger) Format() string {
	return o.format
}

// Writer -
func (o *TLogger) Writer() io.Writer {
	return o.writer
}

// Styler -
func (o *TLogger) Styler() TStyler {
	return o.styler
}

// CanHandle -
func (o *TLogger) CanHandle(moduleName string) bool {
	if len(o.moduleFilter) == 0 {
		return true
	}
	for _, name := range o.moduleFilter {
		if name == moduleName {
			return true
		}
	}
	return false
}

// CannotHandle -
func (o *TLogger) CannotHandle(moduleName string) bool {
	return !o.CanHandle(moduleName)
}

// TFormatParams -
type TFormatParams struct {
	Format     string
	Time       time.Time
	LogLevel   loglevel.TLevel
	Text       string
	Error      error
	HasError   bool
	State      loglevel.TFilter
	ModuleName string
}

// Formatter -
func (o *TLogger) Formatter(params TFormatParams) string {
	if len(params.Format) == 0 {
		return ""
	}
	ret := ""

	if str, ok := o.styler('~', &params); ok {
		params.Format = str
	}
	format := params.Format
	for len(format) > 0 {
		ch, offs := utf8.DecodeRuneInString(format)
		format = format[offs:]
		//if ch == utf8.RuneError {
		//}
		if ch != '~' {
			ret += string(ch)
			continue
		}
		if len(format) > 0 {
			nextCh, offs := utf8.DecodeRuneInString(format)
			//if ch == utf8.RuneError {
			//}
			if nextCh == '~' {
				ret += "~"
			} else if str, ok := o.styler(nextCh, &params); ok {
				ret += str
			} else {
				ret += string(ch)
				continue
			}
			format = format[offs:]
		}
	} // for len(format) > 0
	return ret
}

func defaultStyler(key rune, params *TFormatParams) (string, bool) {
	switch key {
	case '~':
		format := params.Format
		replace := ""
		if params.Error != nil {
			replace = "\n    +cause: ~e"
		}
		format = strings.Replace(format, "~e", replace, -1)
		return format, true
	case 'd':
		return params.Time.Format("2006-01-02"), true
	case 't':
		return params.Time.Format("15:04:05"), true //params.Time().Format("2006-01-02 15:04:05"), true
	case 'l':
		return params.LogLevel.String(), true
	case 'm':
		return params.ModuleName, true
	case 'e':
		ret := ""
		if params.Error != nil {
			ret = params.Error.Error()
		}
		return ret, true
	case 'x':
		return params.Text, true
	case 's':
		if params.LogLevel == loglevel.Reset && params.State != 0 {
			return " " + params.State.String() + " ", true
		}
		return " ", true
	} // end of switch
	return "", false
}

func loglevelColor(level loglevel.TLevel) string {
	color := "\x1b[0m"
	switch level {
	case loglevel.Debug:
		color = "\x1b[1;30m" // bright black
	case loglevel.Info:
		color = "\x1b[0m" // reset //white (lightgrey)
	case loglevel.Notice:
		color = "\x1b[1;32m" // bright green
	case loglevel.Reset:
		color = "\x1b[1;36m" // bright cyan
	case loglevel.Warning:
		color = "\x1b[1;33m" // bright yellow
	case loglevel.Error:
		color = "\x1b[1;31m" // bright red
	case loglevel.Panic:
		color = "\x1b[1;31m" // bright red
	}
	return color
}

func ansiStyler(key rune, params *TFormatParams) (string, bool) {
	ret, ok := defaultStyler(key, params)
	if key == '~' {
		ret = loglevelColor(params.LogLevel) + ret + "\x1b[0m"
		ok = true
	}
	return ret, ok
}
<commit_msg>change default format pattern<commit_after>package zlogger

import (
	"io"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/macroblock/zl/core/loglevel"
)

// ~d - date when a message occurred
// ~t - time when a message occurred
// ~l - loglevel
// ~m - module name
// ~s - log state
// ~e - error message
// ~x - text message
// example: "~d ~t (~m) ~l: ~x~e\n"
const defaultFormat = "(~m) ~l~s~x~e\n"

// TStyler -
type TStyler func(key rune, params *TFormatParams) (string, bool)

// Supported stylers
var (
	DefaultStyler TStyler = defaultStyler
	AnsiStyler    TStyler = ansiStyler
)

// TLogger -
type TLogger struct {
	name         string
	writer       io.Writer
	styler       TStyler
	levelFilter  loglevel.TFilter
	moduleFilter []string
	format       string
}

// LevelFilter -
func (o *TLogger) LevelFilter() loglevel.TFilter {
	return o.levelFilter
}

// ModuleFilter -
func (o *TLogger) ModuleFilter() []string {
	return append([]string(nil), o.moduleFilter...)
}

// Format -
func (o *TLogger) Format() string {
	return o.format
}

// Writer -
func (o *TLogger) Writer() io.Writer {
	return o.writer
}

// Styler -
func (o *TLogger) Styler() TStyler {
	return o.styler
}

// CanHandle -
func (o *TLogger) CanHandle(moduleName string) bool {
	if len(o.moduleFilter) == 0 {
		return true
	}
	for _, name := range o.moduleFilter {
		if name == moduleName {
			return true
		}
	}
	return false
}

// CannotHandle -
func (o *TLogger) CannotHandle(moduleName string) bool {
	return !o.CanHandle(moduleName)
}

// TFormatParams -
type TFormatParams struct {
	Format     string
	Time       time.Time
	LogLevel   loglevel.TLevel
	Text       string
	Error      error
	HasError   bool
	State      loglevel.TFilter
	ModuleName string
}

// Formatter -
func (o *TLogger) Formatter(params TFormatParams) string {
	if len(params.Format) == 0 {
		return ""
	}
	ret := ""

	if str, ok := o.styler('~', &params); ok {
		params.Format = str
	}
	format := params.Format
	for len(format) > 0 {
		ch, offs := utf8.DecodeRuneInString(format)
		format = format[offs:]
		//if ch == utf8.RuneError {
		//}
		if ch != '~' {
			ret += string(ch)
			continue
		}
		if len(format) > 0 {
			nextCh, offs := utf8.DecodeRuneInString(format)
			//if ch == utf8.RuneError {
			//}
			if nextCh == '~' {
				ret += "~"
			} else if str, ok := o.styler(nextCh, &params); ok {
				ret += str
			} else {
				ret += string(ch)
				continue
			}
			format = format[offs:]
		}
	} // for len(format) > 0
	return ret
}

func defaultStyler(key rune, params *TFormatParams) (string, bool) {
	switch key {
	case '~':
		format := params.Format
		replace := ""
		if params.Error != nil {
			replace = "\n    +cause: ~e"
		}
		format = strings.Replace(format, "~e", replace, -1)
		return format, true
	case 'd':
		return params.Time.Format("2006-01-02"), true
	case 't':
		return params.Time.Format("15:04:05"), true //params.Time().Format("2006-01-02 15:04:05"), true
	case 'l':
		return params.LogLevel.String(), true
	case 'm':
		return params.ModuleName, true
	case 'e':
		ret := ""
		if params.Error != nil {
			ret = params.Error.Error()
		}
		return ret, true
	case 'x':
		return params.Text, true
	case 's':
		if params.LogLevel == loglevel.Reset && params.State != 0 {
			return " " + params.State.String() + " ", true
		}
		return " ", true
	} // end of switch
	return "", false
}

func loglevelColor(level loglevel.TLevel) string {
	color := "\x1b[0m"
	switch level {
	case loglevel.Debug:
		color = "\x1b[1;30m" // bright black
	case loglevel.Info:
		color = "\x1b[0m" // reset //white (lightgrey)
	case loglevel.Notice:
		color = "\x1b[1;32m" // bright green
	case loglevel.Reset:
		color = "\x1b[1;36m" // bright cyan
	case loglevel.Warning:
		color = "\x1b[1;33m" // bright yellow
	case loglevel.Error:
		color = "\x1b[1;31m" // bright red
	case loglevel.Panic:
		color = "\x1b[1;31m" // bright red
	}
	return color
}

func ansiStyler(key rune, params *TFormatParams) (string, bool) {
	ret, ok := defaultStyler(key, params)
	if key == '~' {
		ret = loglevelColor(params.LogLevel) + ret + "\x1b[0m"
		ok = true
	}
	return ret, ok
}
<|endoftext|>


<commit_before>package water

import (
	"fmt"
	"net/http"
	"runtime"
)

func Recovery() HandlerFunc {
	return func(ctx *Context) {
		defer func() {
			if err := recover(); err != nil {
				content := fmt.Sprintf("HandlerFunc crashed with error: %v", err)
				for skip := 1; ; skip += 1 {
					//当前goroutine的栈上的函数调用信息，主要有当前的PC值和被调用的文件及其行号.
					//skip==0表示当前栈,这里是指Recovery().
					_, file, line, ok := runtime.Caller(skip)
					if !ok {
						break
					} else {
						content += "\n"
					}
					content += fmt.Sprintf("%v %v", file, line)
				}

				if !ctx.written {
					ctx.WriteHeader(http.StatusInternalServerError)
				}

				if Status != Stable {
					ctx.WriteString(content)
				}
			} else {
				if !ctx.written {
					ctx.WriteHeader(http.StatusOK)
				}
			}
		}()

		ctx.Next()
	}
}
<commit_msg>+ log panic info<commit_after>package water

import (
	"fmt"
	"net/http"
	"runtime"
)

func Recovery() HandlerFunc {
	return func(ctx *Context) {
		defer func() {
			if err := recover(); err != nil {
				content := fmt.Sprintf("HandlerFunc crashed with error: %v", err)
				for skip := 1; ; skip += 1 {
					//当前goroutine的栈上的函数调用信息，主要有当前的PC值和被调用的文件及其行号.
					//skip==0表示当前栈,这里是指Recovery().
					_, file, line, ok := runtime.Caller(skip)
					if !ok {
						break
					} else {
						content += "\n"
					}
					content += fmt.Sprintf("%v %v", file, line)
				}

				if !ctx.written {
					ctx.WriteHeader(http.StatusInternalServerError)
				}

				if !LogClose {
					ctx.router.logger.Printf("panic : %v\n", content)
				}

				if Status != Stable {
					ctx.WriteString(content)
				}
			} else {
				if !ctx.written {
					ctx.WriteHeader(http.StatusOK)
				}
			}
		}()

		ctx.Next()
	}
}
<|endoftext|>


<commit_before>package redisserver

import (
	"fmt"
	"github.com/garyburd/redigo/redis"
	"Go-Redis-Admin/src/common/request"
	"Go-Redis-Admin/src/common/response"
)

type redisapi struct {
	input request.Input
	output response.Output
	Redisconn redis.Conn
}

func (re *redisapi) Connect() {
	if re.input.Request.Method!="POST"{
		fmt.Println("请求错误")
	}
	body,_:=re.input.InputBody()
	fmt.Println(body)
	re.input.Request.ParseForm()
	var network string=re.input.Request.FormValue("network")
	var address string=re.input.Request.FormValue("address")
	var password string=re.input.Request.FormValue("password")
	option:=redis.DialOption{}
	if password!=""{
		option=redis.DialPassword(password)
	}
	conn, err := redis.Dial(network, address,option)
	if err != nil {
		fmt.Println("connect error:", err.Error())
	} else {
		fmt.Println("connect redis success")
	}
	re.Redisconn = conn
	re.output.ResponseWriter.WriteHeader(200)
	//c:=NewConnection(conn)
	//conn.Close()
}
func (re *redisapi) Ping() {
	reply,err:=re.Redisconn.Do("PING","test")
	if err!=nil{
		fmt.Println(err.Error())
	}else {
		fmt.Println(redis.String(reply,err))
	}
	result,_:=redis.String(reply,err)
	re.output.WriteString(result)
}
func (re *redisapi)Set(){
	var args []interface{}
	reply,err:=re.Redisconn.Do("SET",args)
	redis.String(reply,err)
	if err!=nil {
		fmt.Println(err.Error())
	}else {
		fmt.Println(reply)
	}
	result,_:=redis.String(reply,err)
	re.output.WriteString(result)
}

func (re *redisapi) Get(){
	var key string
	reply,err:=re.Redisconn.Do("GET",key)
	if err!=nil{
		fmt.Println(err.Error())
	}else {
		fmt.Println(reply)
	}
	result,_:=redis.String(reply,err)
	re.output.WriteString(result)
}

func (re *redisapi) Delete(){
	var args []interface{}
	reply,err:=re.Redisconn.Do("DEL",args)
	if err!=nil{
		fmt.Println(err.Error())
	}else {
		fmt.Println(reply)
	}
	result,_:=redis.String(reply,err)
	re.output.WriteString(result)
}
func (re *redisapi)SetTTL(){
	var args []interface{}
	reply,err:=re.Redisconn.Do("TTL",args)
	if err!=nil{
		fmt.Println(err.Error())
	}else{
		fmt.Println(reply)
	}
	result,_:=redis.String(reply,err)
	re.output.WriteString(result)
}

func(re *redisapi)Pool(){

}<commit_msg>modify redis SetTTl add getttl<commit_after>package redisserver

import (
	"fmt"
	"github.com/garyburd/redigo/redis"
	"Go-Redis-Admin/src/common/request"
	"Go-Redis-Admin/src/common/response"
)

type redisapi struct {
	input request.Input
	output response.Output
	Redisconn redis.Conn
}

func (re *redisapi) Connect() {
	if re.input.Request.Method!="POST"{
		fmt.Println("请求错误")
	}
	body,_:=re.input.InputBody()
	fmt.Println(body)
	re.input.Request.ParseForm()
	var network string=re.input.Request.FormValue("network")
	var address string=re.input.Request.FormValue("address")
	var password string=re.input.Request.FormValue("password")
	option:=redis.DialOption{}
	if password!=""{
		option=redis.DialPassword(password)
	}
	conn, err := redis.Dial(network, address,option)
	if err != nil {
		fmt.Println("connect error:", err.Error())
	} else {
		fmt.Println("connect redis success")
	}
	re.Redisconn = conn
	re.output.ResponseWriter.WriteHeader(200)
	//c:=NewConnection(conn)
	//conn.Close()
}
func (re *redisapi) Ping() {
	reply,err:=re.Redisconn.Do("PING","test")
	if err!=nil{
		fmt.Println(err.Error())
	}else {
		fmt.Println(redis.String(reply,err))
	}
	result,_:=redis.String(reply,err)
	re.output.WriteString(result)
}
func (re *redisapi)Set(){
	var args []interface{}
	reply,err:=re.Redisconn.Do("SET",args)
	redis.String(reply,err)
	if err!=nil {
		fmt.Println(err.Error())
	}else {
		fmt.Println(reply)
	}
	result,_:=redis.String(reply,err)
	re.output.WriteString(result)
}

func (re *redisapi) Get(){
	var key string
	reply,err:=re.Redisconn.Do("GET",key)
	if err!=nil{
		fmt.Println(err.Error())
	}else {
		fmt.Println(reply)
	}
	result,_:=redis.String(reply,err)
	re.output.WriteString(result)
}

func (re *redisapi) Delete(){
	var args []interface{}
	reply,err:=re.Redisconn.Do("DEL",args)
	if err!=nil{
		fmt.Println(err.Error())
	}else {
		fmt.Println(reply)
	}
	result,_:=redis.String(reply,err)
	re.output.WriteString(result)
}
func (re *redisapi)SetTTL(){
	var args []interface{}
	reply,err:=re.Redisconn.Do("EXPIRE",args)
	if err!=nil{
		fmt.Println(err.Error())
	}else{
		fmt.Println(reply)
	}

	result,_:=redis.String(reply,err)
	re.output.WriteString(result)
}

func(re * redisapi)GetTTL(){
	var args []interface{}
	reply,err:=re.Redisconn.Do("TTl",args)
	if err!=nil{
		fmt.Println(err.Error())
	}else{
		fmt.Println(reply)
	}
	result,_:=redis.String(reply,err)
	re.output.WriteString(result)
}

func(re *redisapi)Pool(){

}<|endoftext|>