<commit_before>package pgn

import (
	"fmt"
	"os"
	"strings"
	"testing"
	"text/scanner"

	. "gopkg.in/check.v1"
)

func Test(t *testing.T) { TestingT(t) }

type PGNSuite struct{}

var _ = Suite(&PGNSuite{})

var simple = `[Event "State Ch."]
[Site "New York, USA"]
[Date "1910.??.??"]
[Round "?"]
[White "Capablanca"]
[Black "Jaffe"]
[Result "1-0"]
[ECO "D46"]
[Opening "Queen's Gambit Dec."]
[Annotator "Reinfeld, Fred"]
[WhiteTitle "GM"]
[WhiteCountry "Cuba"]
[BlackCountry "United States"]

1. d4 d5 2. Nf3 Nf6 3. e3 c6 4. c4 e6 5. Nc3 Nbd7 6. Bd3 Bd6
7. O-O O-O 8. e4 dxe4 9. Nxe4 Nxe4 10. Bxe4 Nf6 11. Bc2 h6
12. b3 b6 13. Bb2 Bb7 14. Qd3 g6 15. Rae1 Nh5 16. Bc1 Kg7
17. Rxe6 Nf6 18. Ne5 c5 19. Bxh6+ Kxh6 20. Nxf7+ 1-0
`

func (s *PGNSuite) TestParse(c *C) {
	r := strings.NewReader(simple)
	sc := scanner.Scanner{}
	sc.Init(r)
	game, err := ParseGame(&sc)
	if err != nil {
		c.Fatal(err)
	}
	if game.Tags["Site"] != "New York, USA" {
		c.Fatal("Site tag wrong: ", game.Tags["Site"])
	}
	if len(game.Moves) == 0 || game.Moves[0].From != D2 || game.Moves[0].To != D4 {
		c.Fatal("first move is wrong", game.Moves[0])
	}
	if len(game.Moves) != 39 || game.Moves[38].From != E5 || game.Moves[38].To != F7 {
		c.Fatal("last move is wrong", game.Moves[38])
	}
}

func (s *PGNSuite) TestPGNScanner(c *C) {
	f, err := os.Open("polgar.pgn")
	if err != nil {
		c.Fatal(err)
	}
	ps := NewPGNScanner(f)
	for ps.Next() {
		game, err := ps.Scan()
		if err != nil {
			fmt.Println(game)
			c.Fatal(err)
		}
	}
}

func (s *PGNSuite) TestPGNParseWithCheckmate(c *C) {
	pgnstr := `[Event "Live Chess"]
[Site "Chess.com"]
[Date "2014.10.10"]
[White "MarkoMakaj"]
[Black "AndreyOstrovskiy"]
[Result "1-0"]
[WhiteElo "2196"]
[BlackElo "2226"]
[TimeControl "1|1"]
[Termination "MarkoMakaj won by checkmate"]

1.d4 g6 2.c4 Bg7 3.Nc3 c5 4.Nf3 cxd4 5.Nxd4 Nc6 6.Nc2 Nf6 7.g3 O-O 8.Bg2 b6 9.O-O Bb7 10.b3 Rc8
 11.Bb2 Qc7 12.Qd2 Qb8 13.Ne3 Rfd8 14.Rfd1 e6 15.Rac1 Qa8 16.Nb5 d5 17.cxd5 exd5 18.Bxf6 Bxf6 19.Nxd5 Bg7 20.e4 a6
 21.Nbc3 b5 22.Qf4 Qa7 23.Nf6+ Kh8 24.Ncd5 Nd4 25.Qh4 h6 26.Rxc8 Rxc8 27.e5 Ne6 28.Ng4 Rc2 29.Nde3 Rxa2 30.Nxh6 Bxg2
 31.Kxg2 Bxe5 32.Nxf7+ Kg7 33.Nxe5 Qxe3 34.Qe7+ Kh6 35.Nf7+ Kh5 36.Qh4# 1-0
`
	r := strings.NewReader(pgnstr)
	sc := scanner.Scanner{}
	sc.Init(r)
	game, err := ParseGame(&sc)
	c.Assert(err, IsNil)
	c.Assert(len(game.Moves), Equals, 71)
}

func (s *PGNSuite) TestPGNParseInfiniteLoopF4(c *C) {
	pgnstr := `[Event "BKL-Turnier"]
[Site "Leipzig"]
[Date "1984.??.??"]
[Round "5"]
[White "Polgar, Zsuzsa"]
[Black "Moehring, Guenther"]
[Result "1-0"]
[WhiteElo "2275"]
[BlackElo "2395"]
[ECO "A49"]

1.d4 Nf6 2.Nf3 d6 3.b3 g6 4.Bb2 Bg7 5.g3 c5 6.Bg2 cxd4 7.Nxd4 d5 8.O-O O-O
9.Na3 Re8 10.Nf3 Nc6 11.c4 dxc4 12.Nxc4 Be6 13.Rc1 Rc8 14.Nfe5 Nxe5 15.Bxe5 Bxc4
16.Rxc4 Rxc4 17.bxc4 Qa5 18.Bxf6 Bxf6 19.Bxb7 Rd8 20.Qb3 Rb8 21.e3 h5 22.Rb1 h4
23.Qb5 Qc7 24.a4 hxg3 25.hxg3 Be5 26.Kg2 Bd6 27.a5 Bc5 28.a6 Rd8 29.Qc6 Qxc6+
30.Bxc6 Rd2 31.Kf3 Rc2 32.Rb8+ Kg7 33.Bb5 Kf6 34.Rc8 Bb6 35.Ba4 Ra2 36.Bb5 Rc2
37.Ke4 e6 38.Kd3 Rc1 39.Kd2 Rb1 40.Kc2 Rb4 41.Rb8 Bc5 42.Rc8 Bb6 43.Rc6 Ba5
44.Rd6 g5 45.f4 gxf4 46.gxf4 Kf5 47.Rd7 Bb6 48.Rxf7+ Ke4 49.Rb7 Bc5 50.Kc3 Kxe3
51.Rc7 Bb6 52.Rc6 Ba5 53.Kc2 Kxf4 54.Rxe6 Bd8 55.Kc3 Rb1 56.Kd4 Rd1+ 57.Kc5 Kf5
58.Re8 Bb6+ 59.Kc6 Kf6 60.Kb7 Bg1 61.Ra8 Re1 62.Rf8+ Kg7 63.Rf5 Kg6 64.Rd5 Rc1
65.Ka8 Be3 66.Rd6+ Kf5 67.Rd3 Ke4 68.Rxe3+ Kxe3 69.Kxa7 Kd4 70.Kb6 Rg1 71.a7 Rg8
72.Kb7 Rg7+ 73.Kb6  1-0`

	r := strings.NewReader(pgnstr)
	sc := scanner.Scanner{}
	sc.Init(r)
	game, err := ParseGame(&sc)
	c.Assert(err, IsNil)
	//	fmt.Println(game)
	c.Assert(game.Tags["Site"], Equals, "Leipzig")
	c.Assert(len(game.Moves), Equals, 145)
}

func (s *PGNSuite) TestComments(c *C) {
	pgnstr := `[Event "Ch World (match)"]
[Site "New York (USA)"]
[Date "1886.03.24"]
[EventDate "?"]
[Round "19"]
[Result "0-1"]
[White "Johannes Zukertort"]
[Black "Wilhelm Steinitz"]
[ECO "D53"]
[WhiteElo "?"]
[BlackElo "?"]
[PlyCount "58"]

1. d4 {Notes by Robert James Fischer from a television
interview. } d5 2. c4 e6 3. Nc3 Nf6 4. Bg5 Be7 5. Nf3 O-O
6. c5 {White plays a mistake already; he should just play e3,
naturally.--Fischer} b6 7. b4 bxc5 8. dxc5 a5 9. a3 {Now he
plays this fantastic move; it's the winning move. -- Fischer}
d4 {He can't take with the knight, because of axb4.--Fischer}
10. Bxf6 gxf6 11. Na4 e5 {This kingside weakness is nothing;
the center is easily winning.--Fischer} 12. b5 Be6 13. g3 c6
14. bxc6 Nxc6 15. Bg2 Rb8 {Threatening Bb3.--Fischer} 16. Qc1
d3 17. e3 e4 18. Nd2 f5 19. O-O Re8 {A very modern move; a
quiet positional move. The rook is doing nothing now, but
later...--Fischer} 20. f3 {To break up the center, it's his
only chance.--Fischer} Nd4 21. exd4 Qxd4+ 22. Kh1 e3 23. Nc3
Bf6 24. Ndb1 d2 25. Qc2 Bb3 26. Qxf5 d1=Q 27. Nxd1 Bxd1
28. Nc3 e2 29. Raxd1 Qxc3 0-1`

	r := strings.NewReader(pgnstr)
	sc := scanner.Scanner{}
	sc.Init(r)
	game, err := ParseGame(&sc)
	c.Assert(err, Equals, nil)
	c.Assert(game, NotNil)
	c.Assert(game.Tags["Site"], Equals, "New York (USA)")
	c.Assert(len(game.Moves), Equals, 58)
}
<commit_msg>added new test for ambiguous move based on issue9 feedback<commit_after>package pgn

import (
	"fmt"
	"os"
	"strings"
	"testing"
	"text/scanner"

	. "gopkg.in/check.v1"
)

func Test(t *testing.T) { TestingT(t) }

type PGNSuite struct{}

var _ = Suite(&PGNSuite{})

var simple = `[Event "State Ch."]
[Site "New York, USA"]
[Date "1910.??.??"]
[Round "?"]
[White "Capablanca"]
[Black "Jaffe"]
[Result "1-0"]
[ECO "D46"]
[Opening "Queen's Gambit Dec."]
[Annotator "Reinfeld, Fred"]
[WhiteTitle "GM"]
[WhiteCountry "Cuba"]
[BlackCountry "United States"]

1. d4 d5 2. Nf3 Nf6 3. e3 c6 4. c4 e6 5. Nc3 Nbd7 6. Bd3 Bd6
7. O-O O-O 8. e4 dxe4 9. Nxe4 Nxe4 10. Bxe4 Nf6 11. Bc2 h6
12. b3 b6 13. Bb2 Bb7 14. Qd3 g6 15. Rae1 Nh5 16. Bc1 Kg7
17. Rxe6 Nf6 18. Ne5 c5 19. Bxh6+ Kxh6 20. Nxf7+ 1-0
`

func (s *PGNSuite) TestParse(c *C) {
	r := strings.NewReader(simple)
	sc := scanner.Scanner{}
	sc.Init(r)
	game, err := ParseGame(&sc)
	if err != nil {
		c.Fatal(err)
	}
	if game.Tags["Site"] != "New York, USA" {
		c.Fatal("Site tag wrong: ", game.Tags["Site"])
	}
	if len(game.Moves) == 0 || game.Moves[0].From != D2 || game.Moves[0].To != D4 {
		c.Fatal("first move is wrong", game.Moves[0])
	}
	if len(game.Moves) != 39 || game.Moves[38].From != E5 || game.Moves[38].To != F7 {
		c.Fatal("last move is wrong", game.Moves[38])
	}
}

func (s *PGNSuite) TestPGNScanner(c *C) {
	f, err := os.Open("polgar.pgn")
	if err != nil {
		c.Fatal(err)
	}
	ps := NewPGNScanner(f)
	for ps.Next() {
		game, err := ps.Scan()
		if err != nil {
			fmt.Println(game)
			c.Fatal(err)
		}
	}
}

func (s *PGNSuite) TestPGNParseWithCheckmate(c *C) {
	pgnstr := `[Event "Live Chess"]
[Site "Chess.com"]
[Date "2014.10.10"]
[White "MarkoMakaj"]
[Black "AndreyOstrovskiy"]
[Result "1-0"]
[WhiteElo "2196"]
[BlackElo "2226"]
[TimeControl "1|1"]
[Termination "MarkoMakaj won by checkmate"]

1.d4 g6 2.c4 Bg7 3.Nc3 c5 4.Nf3 cxd4 5.Nxd4 Nc6 6.Nc2 Nf6 7.g3 O-O 8.Bg2 b6 9.O-O Bb7 10.b3 Rc8
 11.Bb2 Qc7 12.Qd2 Qb8 13.Ne3 Rfd8 14.Rfd1 e6 15.Rac1 Qa8 16.Nb5 d5 17.cxd5 exd5 18.Bxf6 Bxf6 19.Nxd5 Bg7 20.e4 a6
 21.Nbc3 b5 22.Qf4 Qa7 23.Nf6+ Kh8 24.Ncd5 Nd4 25.Qh4 h6 26.Rxc8 Rxc8 27.e5 Ne6 28.Ng4 Rc2 29.Nde3 Rxa2 30.Nxh6 Bxg2
 31.Kxg2 Bxe5 32.Nxf7+ Kg7 33.Nxe5 Qxe3 34.Qe7+ Kh6 35.Nf7+ Kh5 36.Qh4# 1-0
`
	r := strings.NewReader(pgnstr)
	sc := scanner.Scanner{}
	sc.Init(r)
	game, err := ParseGame(&sc)
	c.Assert(err, IsNil)
	c.Assert(len(game.Moves), Equals, 71)
}

func (s *PGNSuite) TestPGNParseInfiniteLoopF4(c *C) {
	pgnstr := `[Event "BKL-Turnier"]
[Site "Leipzig"]
[Date "1984.??.??"]
[Round "5"]
[White "Polgar, Zsuzsa"]
[Black "Moehring, Guenther"]
[Result "1-0"]
[WhiteElo "2275"]
[BlackElo "2395"]
[ECO "A49"]

1.d4 Nf6 2.Nf3 d6 3.b3 g6 4.Bb2 Bg7 5.g3 c5 6.Bg2 cxd4 7.Nxd4 d5 8.O-O O-O
9.Na3 Re8 10.Nf3 Nc6 11.c4 dxc4 12.Nxc4 Be6 13.Rc1 Rc8 14.Nfe5 Nxe5 15.Bxe5 Bxc4
16.Rxc4 Rxc4 17.bxc4 Qa5 18.Bxf6 Bxf6 19.Bxb7 Rd8 20.Qb3 Rb8 21.e3 h5 22.Rb1 h4
23.Qb5 Qc7 24.a4 hxg3 25.hxg3 Be5 26.Kg2 Bd6 27.a5 Bc5 28.a6 Rd8 29.Qc6 Qxc6+
30.Bxc6 Rd2 31.Kf3 Rc2 32.Rb8+ Kg7 33.Bb5 Kf6 34.Rc8 Bb6 35.Ba4 Ra2 36.Bb5 Rc2
37.Ke4 e6 38.Kd3 Rc1 39.Kd2 Rb1 40.Kc2 Rb4 41.Rb8 Bc5 42.Rc8 Bb6 43.Rc6 Ba5
44.Rd6 g5 45.f4 gxf4 46.gxf4 Kf5 47.Rd7 Bb6 48.Rxf7+ Ke4 49.Rb7 Bc5 50.Kc3 Kxe3
51.Rc7 Bb6 52.Rc6 Ba5 53.Kc2 Kxf4 54.Rxe6 Bd8 55.Kc3 Rb1 56.Kd4 Rd1+ 57.Kc5 Kf5
58.Re8 Bb6+ 59.Kc6 Kf6 60.Kb7 Bg1 61.Ra8 Re1 62.Rf8+ Kg7 63.Rf5 Kg6 64.Rd5 Rc1
65.Ka8 Be3 66.Rd6+ Kf5 67.Rd3 Ke4 68.Rxe3+ Kxe3 69.Kxa7 Kd4 70.Kb6 Rg1 71.a7 Rg8
72.Kb7 Rg7+ 73.Kb6  1-0`

	r := strings.NewReader(pgnstr)
	sc := scanner.Scanner{}
	sc.Init(r)
	game, err := ParseGame(&sc)
	c.Assert(err, IsNil)
	//	fmt.Println(game)
	c.Assert(game.Tags["Site"], Equals, "Leipzig")
	c.Assert(len(game.Moves), Equals, 145)
}

func (s *PGNSuite) TestComments(c *C) {
	pgnstr := `[Event "Ch World (match)"]
[Site "New York (USA)"]
[Date "1886.03.24"]
[EventDate "?"]
[Round "19"]
[Result "0-1"]
[White "Johannes Zukertort"]
[Black "Wilhelm Steinitz"]
[ECO "D53"]
[WhiteElo "?"]
[BlackElo "?"]
[PlyCount "58"]

1. d4 {Notes by Robert James Fischer from a television
interview. } d5 2. c4 e6 3. Nc3 Nf6 4. Bg5 Be7 5. Nf3 O-O
6. c5 {White plays a mistake already; he should just play e3,
naturally.--Fischer} b6 7. b4 bxc5 8. dxc5 a5 9. a3 {Now he
plays this fantastic move; it's the winning move. -- Fischer}
d4 {He can't take with the knight, because of axb4.--Fischer}
10. Bxf6 gxf6 11. Na4 e5 {This kingside weakness is nothing;
the center is easily winning.--Fischer} 12. b5 Be6 13. g3 c6
14. bxc6 Nxc6 15. Bg2 Rb8 {Threatening Bb3.--Fischer} 16. Qc1
d3 17. e3 e4 18. Nd2 f5 19. O-O Re8 {A very modern move; a
quiet positional move. The rook is doing nothing now, but
later...--Fischer} 20. f3 {To break up the center, it's his
only chance.--Fischer} Nd4 21. exd4 Qxd4+ 22. Kh1 e3 23. Nc3
Bf6 24. Ndb1 d2 25. Qc2 Bb3 26. Qxf5 d1=Q 27. Nxd1 Bxd1
28. Nc3 e2 29. Raxd1 Qxc3 0-1`

	r := strings.NewReader(pgnstr)
	sc := scanner.Scanner{}
	sc.Init(r)
	game, err := ParseGame(&sc)
	c.Assert(err, Equals, nil)
	c.Assert(game, NotNil)
	c.Assert(game.Tags["Site"], Equals, "New York (USA)")
	c.Assert(len(game.Moves), Equals, 58)
}

var issue9 = `[Event "TCh-CAT Gp2 2016"]
[Site "Barcelona ESP"]
[Date "2016.02.06"]
[Round "3.3"]
[White "Montilla Carrillo, Esteban"]
[Black "Garcia Ramos, Daniel"]
[Result "0-1"]
[WhiteElo "2204"]
[BlackElo "2207"]
[ECO "A97"]
[EventDate "2016.01.23"]

1.Nf3 e6 2.c4 f5 3.g3 Nf6 4.Bg2 Be7 5.O-O O-O 6.d4 d6 7.Nc3 Qe8 8.Qd3 Nc6
9.Nb5 Bd8 10.d5 Ne5 11.Qb3 Nxf3+ 12.exf3 e5 13.f4 a6 14.Nc3 exf4 15.Bxf4
Nh5 16.Rfe1 Qf7 17.Be3 f4 18.Bd4 Bf6 19.Qd1 Bg4 20.Qd2 fxg3 21.hxg3 Bxd4
22.Qxd4 Nf6 23.Ne4 Nxe4 24.Rxe4 Bf5 25.Re2 Rfe8 26.Rae1 b6 27.b4 Rxe2 28.
Rxe2 Re8 29.Re3 h6 30.c5 bxc5 31.bxc5 Rxe3 32.Qxe3 Qf6 33.a3 dxc5 34.Qxc5
Qb6 35.Qc3 Kh7 36.Qe5 Qb1+ 37.Kh2 Qc2 38.Qf4 a5 39.g4 Bg6 40.Qd4 Qc1 41.f4
Qxa3 42.f5 Qd6+ 43.Kh1 Bf7 44.Qe4 Kg8 45.Qc4 Kf8 46.Qb5 a4 47.Qb8+ Be8 48.
Qa7 Ke7 49.Qd4 Kf7 50.Qe3 Bd7 51.Qc3 a3 52.Qb3 Kf8 53.Qb8+ Ke7 54.Qb3 Be8
55.Bf3 Qc5 56.Kg2 Bb5 57.d6+ cxd6 58.Qe6+ Kd8 59.Qg8+ Be8 60.Qxg7 a2 61.
Qf6+ Kc7 62.Qb2 Qa5 63.Qb7+ Kd8 64.f6 a1=Q 65.Qe7+ Kc8 66.Qb7+ Kd8 67.Qe7+
Kc8 68.Qxe8+ Qd8 69.Qc6+ Qc7 70.Qe8+ Qd8 71.Bb7+ Kc7 72.Qc6+ Kb8 73.Ba6
Qb2+ 74.Kh3 Qbb6 0-1`

func (s *PGNSuite) TestIssue9(c *C) {
	r := strings.NewReader(issue9)
	sc := scanner.Scanner{}
	sc.Init(r)
	_, err := ParseGame(&sc)
	if err != nil {
		c.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package common

import (
	//"encoding/base64"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"time"
	//"github.com/tdhite/q3-training-journal/journal"
)

type SimpleConsumer struct {
	DataManager string
}

func (sc *SimpleConsumer) saveReservation(msg *Message) {

	fmt.Printf("Sending reservation %s to data-manager at %s\n", msg.ToJson(), sc.DataManager)
	payload := string(msg.Base64[:])
	fmt.Printf("payload: %s\n", payload)

}

func (sc *SimpleConsumer) handleMessages(messages <-chan *Message, topic string) error {
	go func() {
		for msg := range messages {
			fmt.Printf("topic: %s", msg.ToJson())
			fmt.Println()
			if topic == "reservation" {
				sc.saveReservation(msg)
			}

		}
	}()

	return nil
}

func (sc *SimpleConsumer) consume(url string, topic string, listen chan bool) (<-chan *Message, error) {
	messages := make(chan *Message, 1)
	tl := fmt.Sprintf("%s/api/topic/%s", url, topic)

	go func() {
		for <-listen {
			fmt.Printf("Checking queue at %s for topic %s\n", url, topic)

			res, err := http.Get(tl)
			if err != nil {
				log.Print(err)
				continue
			}

			b, err := ioutil.ReadAll(res.Body)
			if err != nil {
				//log.Print(err)
				continue
			}

			//fmt.Printf("raw: %s", b)
			msg := &Message{}

			msg.FromJson(b)
			if msg.Base64 == nil || len(msg.Base64) == 0 {
				//log.Print("no message")
				res.Body.Close()
				continue
			}
			messages <- msg
			res.Body.Close()
			if err != nil {
				log.Fatal(err)
			}

		}
		fmt.Println("Exit consumer")

		close(messages)
	}()

	return messages, nil
}

func (sc *SimpleConsumer) ConsumeMessages(url string, topic string) error {

	listen := make(chan bool, 1)

	messages, _ := sc.consume(url, topic, listen)
	err := sc.handleMessages(messages, topic)
	if err != nil {
		fmt.Errorf("%V", err)
		os.Exit(1)
	}

	for {
		time.Sleep(time.Second * 15)
		listen <- true
	}

	listen <- false
	time.Sleep(time.Second * 3)
	return nil

}
<commit_msg>update for datamanager<commit_after>package common

import (
	//"encoding/base64"
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"time"
	//"github.com/tdhite/q3-training-journal/journal"
)

type SimpleConsumer struct {
	DataManager string
}

func (sc *SimpleConsumer) saveReservation(msg *Message) {
	url := fmt.Sprintf("%s/api/reservations", sc.DataManager)
	fmt.Printf("Sending reservation %s to data-manager at %s\n", msg.ToJson(), url)
	payload := string(msg.Base64[:])
	fmt.Printf("payload: %s\n", payload)
	//var jsonStr = []byte(`{"title":"Buy cheese and bread for breakfast."}`)
	var jsonStr = []byte(payload)

	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr))
	req.Header.Set("X-Custom-Header", "myvalue")
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	fmt.Println("response Status:", resp.Status)
	fmt.Println("response Headers:", resp.Header)
	body, _ := ioutil.ReadAll(resp.Body)
	fmt.Println("response Body:", string(body))

}

func (sc *SimpleConsumer) handleMessages(messages <-chan *Message, topic string) error {
	go func() {
		for msg := range messages {
			fmt.Printf("topic: %s", msg.ToJson())
			fmt.Println()
			if topic == "reservation" {
				sc.saveReservation(msg)
			}

		}
	}()

	return nil
}

func (sc *SimpleConsumer) consume(url string, topic string, listen chan bool) (<-chan *Message, error) {
	messages := make(chan *Message, 1)
	tl := fmt.Sprintf("%s/api/topic/%s", url, topic)

	go func() {
		for <-listen {
			fmt.Printf("Checking queue at %s for topic %s\n", url, topic)

			res, err := http.Get(tl)
			if err != nil {
				log.Print(err)
				continue
			}

			b, err := ioutil.ReadAll(res.Body)
			if err != nil {
				//log.Print(err)
				continue
			}

			//fmt.Printf("raw: %s", b)
			msg := &Message{}

			msg.FromJson(b)
			if msg.Base64 == nil || len(msg.Base64) == 0 {
				//log.Print("no message")
				res.Body.Close()
				continue
			}
			messages <- msg
			res.Body.Close()
			if err != nil {
				log.Fatal(err)
			}

		}
		fmt.Println("Exit consumer")

		close(messages)
	}()

	return messages, nil
}

func (sc *SimpleConsumer) ConsumeMessages(url string, topic string) error {

	listen := make(chan bool, 1)

	messages, _ := sc.consume(url, topic, listen)
	err := sc.handleMessages(messages, topic)
	if err != nil {
		fmt.Errorf("%V", err)
		os.Exit(1)
	}

	for {
		listen <- true
		time.Sleep(time.Second * 15)
	}

	listen <- false
	time.Sleep(time.Second * 3)
	return nil

}
<|endoftext|>


<commit_before>package policies

import (
	"errors"

	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/pagination"
)

// Validation errors returned by create or update operations.
var (
	ErrNoName = errors.New("Policy name cannot by empty.")
	ErrNoArgs = errors.New("Args cannot be nil for schedule policies.")
)

// List returns all scaling policies for a group.
func List(client *gophercloud.ServiceClient, groupID string) pagination.Pager {
	url := listURL(client, groupID)

	createPageFn := func(r pagination.PageResult) pagination.Page {
		return PolicyPage{pagination.SinglePageBase(r)}
	}

	return pagination.NewPager(client, url, createPageFn)
}

// CreateOptsBuilder is the interface responsible for generating the map that
// will be marshalled to JSON for a Create operation.
type CreateOptsBuilder interface {
	ToPolicyCreateMap() ([]map[string]interface{}, error)
}

// Adjustment represents the change in capacity associated with a policy.
type Adjustment struct {
	// The type for this adjustment.
	Type AdjustmentType

	// The value of the adjustment.  For adjustments of type Change or
	// DesiredCapacity, this will be converted to an integer.
	Value float64
}

// AdjustmentType represents the way in which a policy will change a group.
type AdjustmentType string

// Valid types of adjustments for a policy.
const (
	Change          AdjustmentType = "change"
	ChangePercent   AdjustmentType = "changePercent"
	DesiredCapacity AdjustmentType = "desiredCapacity"
)

// CreateOpts is a slice of CreateOpt structs that allow the user to create
// multiple policies in a single operation.
type CreateOpts []CreateOpt

// CreateOpt represents the options to create a policy.
type CreateOpt struct {
	// Name [required] is a name for the policy.
	Name string

	// Type [required] of policy, i.e. either "webhook" or "schedule".
	Type Type

	// Cooldown [required] period in seconds.
	Cooldown int

	// Adjustment [requried] type and value for the policy.
	Adjustment Adjustment

	// Additional configuration options for some types of policy.
	Args map[string]interface{}
}

// ToPolicyCreateMap converts a slice of CreateOpt structs into a map for use
// in the request body of a Create operation.
func (opts CreateOpts) ToPolicyCreateMap() ([]map[string]interface{}, error) {
	var policies []map[string]interface{}

	for _, o := range opts {
		if o.Name == "" {
			return nil, ErrNoName
		}

		if o.Type == Schedule && o.Args == nil {
			return nil, ErrNoArgs
		}

		policy := make(map[string]interface{})

		policy["name"] = o.Name
		policy["type"] = o.Type
		policy["cooldown"] = o.Cooldown

		// TODO: Function to validate and cast key + value?
		policy[string(o.Adjustment.Type)] = o.Adjustment.Value

		if o.Args != nil {
			policy["args"] = o.Args
		}

		policies = append(policies, policy)
	}

	return policies, nil
}

// Create requests a new policy be created and associated with the given group.
func Create(client *gophercloud.ServiceClient, groupID string, opts CreateOptsBuilder) CreateResult {
	var res CreateResult

	reqBody, err := opts.ToPolicyCreateMap()

	if err != nil {
		res.Err = err
		return res
	}

	_, res.Err = client.Post(createURL(client, groupID), reqBody, &res.Body, nil)

	return res
}

// Get requests the details of a single policy with the given ID.
func Get(client *gophercloud.ServiceClient, groupID, policyID string) GetResult {
	var result GetResult

	_, result.Err = client.Get(getURL(client, groupID, policyID), &result.Body, nil)

	return result
}

// UpdateOptsBuilder is the interface responsible for generating the map
// structure for producing JSON for an Update operation.
type UpdateOptsBuilder interface {
	ToPolicyUpdateMap() (map[string]interface{}, error)
}

// UpdateOpts represents the options for updating an existing policy.
//
// Update operations completely replace the configuration being updated. Empty
// values in the update are accepted and overwrite previously specified
// parameters.
type UpdateOpts struct {
	// Name [required] is a name for the policy.
	Name string

	// Type [required] of policy, i.e. either "webhook" or "schedule".
	Type Type

	// Cooldown [required] period in seconds.  If you don't specify a cooldown,
	// it will default to zero, and the policy will be configured as such.
	Cooldown int

	// Adjustment [requried] type and value for the policy.
	Adjustment Adjustment

	// Additional configuration options for some types of policy.
	Args map[string]interface{}
}

// ToPolicyUpdateMap converts an UpdateOpts struct into a map for use as the
// request body in an Update request.
func (opts UpdateOpts) ToPolicyUpdateMap() (map[string]interface{}, error) {
	if opts.Name == "" {
		return nil, ErrNoName
	}

	if opts.Type == Schedule && opts.Args == nil {
		return nil, ErrNoArgs
	}

	policy := make(map[string]interface{})

	policy["name"] = opts.Name
	policy["type"] = opts.Type
	policy["cooldown"] = opts.Cooldown

	// TODO: Function to validate and cast key + value?
	policy[string(opts.Adjustment.Type)] = opts.Adjustment.Value

	if opts.Args != nil {
		policy["args"] = opts.Args
	}

	return policy, nil
}

// Update requests the configuration of the given policy be updated.
func Update(client *gophercloud.ServiceClient, groupID, policyID string, opts UpdateOptsBuilder) UpdateResult {
	var result UpdateResult

	url := updateURL(client, groupID, policyID)
	reqBody, err := opts.ToPolicyUpdateMap()

	if err != nil {
		result.Err = err
		return result
	}

	_, result.Err = client.Put(url, reqBody, nil, &gophercloud.RequestOpts{
		OkCodes: []int{204},
	})

	return result
}

// Delete requests the given policy be permanently deleted.
func Delete(client *gophercloud.ServiceClient, groupID, policyID string) DeleteResult {
	var result DeleteResult

	url := deleteURL(client, groupID, policyID)
	_, result.Err = client.Delete(url, &gophercloud.RequestOpts{
		OkCodes: []int{204},
	})

	return result
}

// Execute requests the given policy be executed immediately.
func Execute(client *gophercloud.ServiceClient, groupID, policyID string) ExecuteResult {
	var result ExecuteResult

	url := executeURL(client, groupID, policyID)
	_, result.Err = client.Post(url, nil, &result.Body, &gophercloud.RequestOpts{
		OkCodes: []int{202},
	})

	return result
}
<commit_msg>Validate Rackspace Auto Scale policy adjustments<commit_after>package policies

import (
	"errors"

	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/pagination"
)

// Validation errors returned by create or update operations.
var (
	ErrNoName            = errors.New("Policy name cannot by empty.")
	ErrNoArgs            = errors.New("Args cannot be nil for schedule policies.")
	ErrInvalidAdjustment = errors.New("Invalid adjustment type.")
)

// List returns all scaling policies for a group.
func List(client *gophercloud.ServiceClient, groupID string) pagination.Pager {
	url := listURL(client, groupID)

	createPageFn := func(r pagination.PageResult) pagination.Page {
		return PolicyPage{pagination.SinglePageBase(r)}
	}

	return pagination.NewPager(client, url, createPageFn)
}

// CreateOptsBuilder is the interface responsible for generating the map that
// will be marshalled to JSON for a Create operation.
type CreateOptsBuilder interface {
	ToPolicyCreateMap() ([]map[string]interface{}, error)
}

// Adjustment represents the change in capacity associated with a policy.
type Adjustment struct {
	// The type for this adjustment.
	Type AdjustmentType

	// The value of the adjustment.  For adjustments of type Change or
	// DesiredCapacity, this will be converted to an integer.
	Value float64
}

// AdjustmentType represents the way in which a policy will change a group.
type AdjustmentType string

// Valid types of adjustments for a policy.
const (
	Change          AdjustmentType = "change"
	ChangePercent   AdjustmentType = "changePercent"
	DesiredCapacity AdjustmentType = "desiredCapacity"
)

// CreateOpts is a slice of CreateOpt structs that allow the user to create
// multiple policies in a single operation.
type CreateOpts []CreateOpt

// CreateOpt represents the options to create a policy.
type CreateOpt struct {
	// Name [required] is a name for the policy.
	Name string

	// Type [required] of policy, i.e. either "webhook" or "schedule".
	Type Type

	// Cooldown [required] period in seconds.
	Cooldown int

	// Adjustment [requried] type and value for the policy.
	Adjustment Adjustment

	// Additional configuration options for some types of policy.
	Args map[string]interface{}
}

// ToPolicyCreateMap converts a slice of CreateOpt structs into a map for use
// in the request body of a Create operation.
func (opts CreateOpts) ToPolicyCreateMap() ([]map[string]interface{}, error) {
	var policies []map[string]interface{}

	for _, o := range opts {
		if o.Name == "" {
			return nil, ErrNoName
		}

		if o.Type == Schedule && o.Args == nil {
			return nil, ErrNoArgs
		}

		policy := make(map[string]interface{})

		policy["name"] = o.Name
		policy["type"] = o.Type
		policy["cooldown"] = o.Cooldown

		if err := setAdjustment(o.Adjustment, policy); err != nil {
			return nil, err
		}

		if o.Args != nil {
			policy["args"] = o.Args
		}

		policies = append(policies, policy)
	}

	return policies, nil
}

// Create requests a new policy be created and associated with the given group.
func Create(client *gophercloud.ServiceClient, groupID string, opts CreateOptsBuilder) CreateResult {
	var res CreateResult

	reqBody, err := opts.ToPolicyCreateMap()

	if err != nil {
		res.Err = err
		return res
	}

	_, res.Err = client.Post(createURL(client, groupID), reqBody, &res.Body, nil)

	return res
}

// Get requests the details of a single policy with the given ID.
func Get(client *gophercloud.ServiceClient, groupID, policyID string) GetResult {
	var result GetResult

	_, result.Err = client.Get(getURL(client, groupID, policyID), &result.Body, nil)

	return result
}

// UpdateOptsBuilder is the interface responsible for generating the map
// structure for producing JSON for an Update operation.
type UpdateOptsBuilder interface {
	ToPolicyUpdateMap() (map[string]interface{}, error)
}

// UpdateOpts represents the options for updating an existing policy.
//
// Update operations completely replace the configuration being updated. Empty
// values in the update are accepted and overwrite previously specified
// parameters.
type UpdateOpts struct {
	// Name [required] is a name for the policy.
	Name string

	// Type [required] of policy, i.e. either "webhook" or "schedule".
	Type Type

	// Cooldown [required] period in seconds.  If you don't specify a cooldown,
	// it will default to zero, and the policy will be configured as such.
	Cooldown int

	// Adjustment [requried] type and value for the policy.
	Adjustment Adjustment

	// Additional configuration options for some types of policy.
	Args map[string]interface{}
}

// ToPolicyUpdateMap converts an UpdateOpts struct into a map for use as the
// request body in an Update request.
func (opts UpdateOpts) ToPolicyUpdateMap() (map[string]interface{}, error) {
	if opts.Name == "" {
		return nil, ErrNoName
	}

	if opts.Type == Schedule && opts.Args == nil {
		return nil, ErrNoArgs
	}

	policy := make(map[string]interface{})

	policy["name"] = opts.Name
	policy["type"] = opts.Type
	policy["cooldown"] = opts.Cooldown

	if err := setAdjustment(opts.Adjustment, policy); err != nil {
		return nil, err
	}

	if opts.Args != nil {
		policy["args"] = opts.Args
	}

	return policy, nil
}

// Update requests the configuration of the given policy be updated.
func Update(client *gophercloud.ServiceClient, groupID, policyID string, opts UpdateOptsBuilder) UpdateResult {
	var result UpdateResult

	url := updateURL(client, groupID, policyID)
	reqBody, err := opts.ToPolicyUpdateMap()

	if err != nil {
		result.Err = err
		return result
	}

	_, result.Err = client.Put(url, reqBody, nil, &gophercloud.RequestOpts{
		OkCodes: []int{204},
	})

	return result
}

// Delete requests the given policy be permanently deleted.
func Delete(client *gophercloud.ServiceClient, groupID, policyID string) DeleteResult {
	var result DeleteResult

	url := deleteURL(client, groupID, policyID)
	_, result.Err = client.Delete(url, &gophercloud.RequestOpts{
		OkCodes: []int{204},
	})

	return result
}

// Execute requests the given policy be executed immediately.
func Execute(client *gophercloud.ServiceClient, groupID, policyID string) ExecuteResult {
	var result ExecuteResult

	url := executeURL(client, groupID, policyID)
	_, result.Err = client.Post(url, nil, &result.Body, &gophercloud.RequestOpts{
		OkCodes: []int{202},
	})

	return result
}

// Validate and set an adjustment on the given request body.
func setAdjustment(adjustment Adjustment, reqBody map[string]interface{}) error {
	key := string(adjustment.Type)

	switch adjustment.Type {
	case ChangePercent:
		reqBody[key] = adjustment.Value

	case Change, DesiredCapacity:
		reqBody[key] = int(adjustment.Value)

	default:
		return ErrInvalidAdjustment
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2013 Kelsey Hightower. All rights reserved.
// Use of this source code is governed by the MIT License that can be found in
// the LICENSE file.

package envconfig

import (
	"os"
	"testing"
)

type Specification struct {
	Debug                        bool
	Port                         int
	Rate                         float32
	User                         string
	MultiWordVar                 string
	MultiWordVarWithAlt          string `envconfig:"MULTI_WORD_VAR_WITH_ALT"`
	MultiWordVarWithLowerCaseAlt string `envconfig:"multi_word_var_with_lower_case_alt"`
	NoPrefixWithAlt              string `envconfig:"SERVICE_HOST"`
	DefaultVar                   string `default:"foobar"`
	RequiredVar                  string `required:"true"`
}

func TestProcess(t *testing.T) {
	var s Specification
	os.Clearenv()
	os.Setenv("ENV_CONFIG_DEBUG", "true")
	os.Setenv("ENV_CONFIG_PORT", "8080")
	os.Setenv("ENV_CONFIG_RATE", "0.5")
	os.Setenv("ENV_CONFIG_USER", "Kelsey")
	os.Setenv("SERVICE_HOST", "127.0.0.1")
	os.Setenv("ENV_CONFIG_REQUIREDVAR", "foo")
	err := Process("env_config", &s)
	if err != nil {
		t.Error(err.Error())
	}
	if s.NoPrefixWithAlt != "127.0.0.1" {
		t.Errorf("expected %v, got %v", "127.0.0.1", s.NoPrefixWithAlt)
	}
	if !s.Debug {
		t.Errorf("expected %v, got %v", true, s.Debug)
	}
	if s.Port != 8080 {
		t.Errorf("expected %d, got %v", 8080, s.Port)
	}
	if s.Rate != 0.5 {
		t.Errorf("expected %f, got %v", 0.5, s.Rate)
	}
	if s.User != "Kelsey" {
		t.Errorf("expected %s, got %s", "Kelsey", s.User)
	}
	if s.RequiredVar != "foo" {
		t.Errorf("expected %s, got %s", "foo", s.RequiredVar)
	}
}

func TestParseErrorBool(t *testing.T) {
	var s Specification
	os.Clearenv()
	os.Setenv("ENV_CONFIG_DEBUG", "string")
	os.Setenv("ENV_CONFIG_REQUIREDVAR", "foo")
	err := Process("env_config", &s)
	v, ok := err.(*ParseError)
	if !ok {
		t.Errorf("expected ParseError, got %v", v)
	}
	if v.FieldName != "Debug" {
		t.Errorf("expected %s, got %v", "Debug", v.FieldName)
	}
	if s.Debug != false {
		t.Errorf("expected %v, got %v", false, s.Debug)
	}
}

func TestParseErrorFloat32(t *testing.T) {
	var s Specification
	os.Clearenv()
	os.Setenv("ENV_CONFIG_RATE", "string")
	os.Setenv("ENV_CONFIG_REQUIREDVAR", "foo")
	err := Process("env_config", &s)
	v, ok := err.(*ParseError)
	if !ok {
		t.Errorf("expected ParseError, got %v", v)
	}
	if v.FieldName != "Rate" {
		t.Errorf("expected %s, got %v", "Rate", v.FieldName)
	}
	if s.Rate != 0 {
		t.Errorf("expected %v, got %v", 0, s.Rate)
	}
}

func TestParseErrorInt(t *testing.T) {
	var s Specification
	os.Clearenv()
	os.Setenv("ENV_CONFIG_PORT", "string")
	os.Setenv("ENV_CONFIG_REQUIREDVAR", "foo")
	err := Process("env_config", &s)
	v, ok := err.(*ParseError)
	if !ok {
		t.Errorf("expected ParseError, got %v", v)
	}
	if v.FieldName != "Port" {
		t.Errorf("expected %s, got %v", "Port", v.FieldName)
	}
	if s.Port != 0 {
		t.Errorf("expected %v, got %v", 0, s.Port)
	}
}

func TestErrInvalidSpecification(t *testing.T) {
	m := make(map[string]string)
	err := Process("env_config", &m)
	if err != ErrInvalidSpecification {
		t.Errorf("expected %v, got %v", ErrInvalidSpecification, err)
	}
}

func TestUnsetVars(t *testing.T) {
	var s Specification
	os.Clearenv()
	os.Setenv("USER", "foo")
	os.Setenv("ENV_CONFIG_REQUIREDVAR", "foo")
	if err := Process("env_config", &s); err != nil {
		t.Error(err.Error())
	}

	// If the var is not defined the non-prefixed version should not be used
	// unless the struct tag says so
	if s.User != "" {
		t.Errorf("expected %q, got %q", "", s.User)
	}
}

func TestAlternateVarNames(t *testing.T) {
	var s Specification
	os.Clearenv()
	os.Setenv("ENV_CONFIG_MULTI_WORD_VAR", "foo")
	os.Setenv("ENV_CONFIG_MULTI_WORD_VAR_WITH_ALT", "bar")
	os.Setenv("ENV_CONFIG_MULTI_WORD_VAR_WITH_LOWER_CASE_ALT", "baz")
	os.Setenv("ENV_CONFIG_REQUIREDVAR", "foo")
	if err := Process("env_config", &s); err != nil {
		t.Error(err.Error())
	}

	// Setting the alt version of the var in the environment has no effect if
	// the struct tag is not supplied
	if s.MultiWordVar != "" {
		t.Errorf("expected %q, got %q", "", s.MultiWordVar)
	}

	// Setting the alt version of the var in the environment correctly sets
	// the value if the struct tag IS supplied
	if s.MultiWordVarWithAlt != "bar" {
		t.Errorf("expected %q, got %q", "bar", s.MultiWordVarWithAlt)
	}

	// Alt value is not case sensitive and is treated as all uppercase
	if s.MultiWordVarWithLowerCaseAlt != "baz" {
		t.Errorf("expected %q, got %q", "baz", s.MultiWordVarWithLowerCaseAlt)
	}
}

func TestRequiredVar(t *testing.T) {
	var s Specification
	os.Clearenv()
	os.Setenv("ENV_CONFIG_REQUIREDVAR", "foobar")
	if err := Process("env_config", &s); err != nil {
		t.Error(err.Error())
	}

	if s.RequiredVar != "foobar" {
		t.Errorf("expected %s, got %s", "foobar", s.RequiredVar)
	}
}

func TestBlankDefaultVar(t *testing.T) {
	var s Specification
	os.Clearenv()
	os.Setenv("ENV_CONFIG_REQUIREDVAR", "requiredvalue")
	if err := Process("env_config", &s); err != nil {
		t.Error(err.Error())
	}

	if s.DefaultVar != "foobar" {
		t.Errorf("expected %s, got %s", "foobar", s.DefaultVar)
	}
}

func TestNonBlankDefaultVar(t *testing.T) {
	var s Specification
	os.Clearenv()
	os.Setenv("ENV_CONFIG_DEFAULTVAR", "nondefaultval")
	os.Setenv("ENV_CONFIG_REQUIREDVAR", "requiredvalue")
	if err := Process("env_config", &s); err != nil {
		t.Error(err.Error())
	}

	if s.DefaultVar != "nondefaultval" {
		t.Errorf("expected %s, got %s", "nondefaultval", s.DefaultVar)
	}
}
<commit_msg>adding multiple tags tests<commit_after>// Copyright (c) 2013 Kelsey Hightower. All rights reserved.
// Use of this source code is governed by the MIT License that can be found in
// the LICENSE file.

package envconfig

import (
	"os"
	"testing"
)

type Specification struct {
	Debug                        bool
	Port                         int
	Rate                         float32
	User                         string
	MultiWordVar                 string
	MultiWordVarWithAlt          string `envconfig:"MULTI_WORD_VAR_WITH_ALT"`
	MultiWordVarWithLowerCaseAlt string `envconfig:"multi_word_var_with_lower_case_alt"`
	NoPrefixWithAlt              string `envconfig:"SERVICE_HOST"`
	DefaultVar                   string `default:"foobar"`
	RequiredVar                  string `required:"true"`
	NoPrefixDefault              string `envconfig:"BROKER" default:"127.0.0.1"`
	RequiredDefault              string `required:"true" default:"foo2bar"`
}

func TestProcess(t *testing.T) {
	var s Specification
	os.Clearenv()
	os.Setenv("ENV_CONFIG_DEBUG", "true")
	os.Setenv("ENV_CONFIG_PORT", "8080")
	os.Setenv("ENV_CONFIG_RATE", "0.5")
	os.Setenv("ENV_CONFIG_USER", "Kelsey")
	os.Setenv("SERVICE_HOST", "127.0.0.1")
	os.Setenv("ENV_CONFIG_REQUIREDVAR", "foo")
	err := Process("env_config", &s)
	if err != nil {
		t.Error(err.Error())
	}
	if s.NoPrefixWithAlt != "127.0.0.1" {
		t.Errorf("expected %v, got %v", "127.0.0.1", s.NoPrefixWithAlt)
	}
	if !s.Debug {
		t.Errorf("expected %v, got %v", true, s.Debug)
	}
	if s.Port != 8080 {
		t.Errorf("expected %d, got %v", 8080, s.Port)
	}
	if s.Rate != 0.5 {
		t.Errorf("expected %f, got %v", 0.5, s.Rate)
	}
	if s.User != "Kelsey" {
		t.Errorf("expected %s, got %s", "Kelsey", s.User)
	}
	if s.RequiredVar != "foo" {
		t.Errorf("expected %s, got %s", "foo", s.RequiredVar)
	}
}

func TestParseErrorBool(t *testing.T) {
	var s Specification
	os.Clearenv()
	os.Setenv("ENV_CONFIG_DEBUG", "string")
	os.Setenv("ENV_CONFIG_REQUIREDVAR", "foo")
	err := Process("env_config", &s)
	v, ok := err.(*ParseError)
	if !ok {
		t.Errorf("expected ParseError, got %v", v)
	}
	if v.FieldName != "Debug" {
		t.Errorf("expected %s, got %v", "Debug", v.FieldName)
	}
	if s.Debug != false {
		t.Errorf("expected %v, got %v", false, s.Debug)
	}
}

func TestParseErrorFloat32(t *testing.T) {
	var s Specification
	os.Clearenv()
	os.Setenv("ENV_CONFIG_RATE", "string")
	os.Setenv("ENV_CONFIG_REQUIREDVAR", "foo")
	err := Process("env_config", &s)
	v, ok := err.(*ParseError)
	if !ok {
		t.Errorf("expected ParseError, got %v", v)
	}
	if v.FieldName != "Rate" {
		t.Errorf("expected %s, got %v", "Rate", v.FieldName)
	}
	if s.Rate != 0 {
		t.Errorf("expected %v, got %v", 0, s.Rate)
	}
}

func TestParseErrorInt(t *testing.T) {
	var s Specification
	os.Clearenv()
	os.Setenv("ENV_CONFIG_PORT", "string")
	os.Setenv("ENV_CONFIG_REQUIREDVAR", "foo")
	err := Process("env_config", &s)
	v, ok := err.(*ParseError)
	if !ok {
		t.Errorf("expected ParseError, got %v", v)
	}
	if v.FieldName != "Port" {
		t.Errorf("expected %s, got %v", "Port", v.FieldName)
	}
	if s.Port != 0 {
		t.Errorf("expected %v, got %v", 0, s.Port)
	}
}

func TestErrInvalidSpecification(t *testing.T) {
	m := make(map[string]string)
	err := Process("env_config", &m)
	if err != ErrInvalidSpecification {
		t.Errorf("expected %v, got %v", ErrInvalidSpecification, err)
	}
}

func TestUnsetVars(t *testing.T) {
	var s Specification
	os.Clearenv()
	os.Setenv("USER", "foo")
	os.Setenv("ENV_CONFIG_REQUIREDVAR", "foo")
	if err := Process("env_config", &s); err != nil {
		t.Error(err.Error())
	}

	// If the var is not defined the non-prefixed version should not be used
	// unless the struct tag says so
	if s.User != "" {
		t.Errorf("expected %q, got %q", "", s.User)
	}
}

func TestAlternateVarNames(t *testing.T) {
	var s Specification
	os.Clearenv()
	os.Setenv("ENV_CONFIG_MULTI_WORD_VAR", "foo")
	os.Setenv("ENV_CONFIG_MULTI_WORD_VAR_WITH_ALT", "bar")
	os.Setenv("ENV_CONFIG_MULTI_WORD_VAR_WITH_LOWER_CASE_ALT", "baz")
	os.Setenv("ENV_CONFIG_REQUIREDVAR", "foo")
	if err := Process("env_config", &s); err != nil {
		t.Error(err.Error())
	}

	// Setting the alt version of the var in the environment has no effect if
	// the struct tag is not supplied
	if s.MultiWordVar != "" {
		t.Errorf("expected %q, got %q", "", s.MultiWordVar)
	}

	// Setting the alt version of the var in the environment correctly sets
	// the value if the struct tag IS supplied
	if s.MultiWordVarWithAlt != "bar" {
		t.Errorf("expected %q, got %q", "bar", s.MultiWordVarWithAlt)
	}

	// Alt value is not case sensitive and is treated as all uppercase
	if s.MultiWordVarWithLowerCaseAlt != "baz" {
		t.Errorf("expected %q, got %q", "baz", s.MultiWordVarWithLowerCaseAlt)
	}
}

func TestRequiredVar(t *testing.T) {
	var s Specification
	os.Clearenv()
	os.Setenv("ENV_CONFIG_REQUIREDVAR", "foobar")
	if err := Process("env_config", &s); err != nil {
		t.Error(err.Error())
	}

	if s.RequiredVar != "foobar" {
		t.Errorf("expected %s, got %s", "foobar", s.RequiredVar)
	}
}

func TestBlankDefaultVar(t *testing.T) {
	var s Specification
	os.Clearenv()
	os.Setenv("ENV_CONFIG_REQUIREDVAR", "requiredvalue")
	if err := Process("env_config", &s); err != nil {
		t.Error(err.Error())
	}

	if s.DefaultVar != "foobar" {
		t.Errorf("expected %s, got %s", "foobar", s.DefaultVar)
	}
}

func TestNonBlankDefaultVar(t *testing.T) {
	var s Specification
	os.Clearenv()
	os.Setenv("ENV_CONFIG_DEFAULTVAR", "nondefaultval")
	os.Setenv("ENV_CONFIG_REQUIREDVAR", "requiredvalue")
	if err := Process("env_config", &s); err != nil {
		t.Error(err.Error())
	}

	if s.DefaultVar != "nondefaultval" {
		t.Errorf("expected %s, got %s", "nondefaultval", s.DefaultVar)
	}
}

func TestAlternateNameDefaultVar(t *testing.T) {
	var s Specification
	os.Clearenv()
	os.Setenv("BROKER", "betterbroker")
	os.Setenv("ENV_CONFIG_REQUIREDVAR", "foo")
	if err := Process("env_config", &s); err != nil {
		t.Error(err.Error())
	}

	if s.NoPrefixDefault != "betterbroker" {
		t.Errorf("expected %q, got %q", "betterbroker", s.NoPrefixDefault)
	}

	os.Clearenv()
	os.Setenv("ENV_CONFIG_REQUIREDVAR", "foo")
	if err := Process("env_config", &s); err != nil {
		t.Error(err.Error())
	}

	if s.NoPrefixDefault != "127.0.0.1" {
		t.Errorf("expected %q, got %q", "127.0.0.1", s.NoPrefixDefault)
	}
}

func TestRequiredDefault(t *testing.T) {
	var s Specification
	os.Clearenv()
	os.Setenv("ENV_CONFIG_REQUIREDVAR", "foo")
	if err := Process("env_config", &s); err != nil {
		t.Error(err.Error())
	}

	if s.RequiredDefault != "foo2bar" {
		t.Errorf("expected %q, got %q", "foo2bar", s.RequiredDefault)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"strings"
)

type Options struct {
	cmd       string
	args      []string
	zkHosts   string
	xmlFile   string
	znodePath string
	depth     int
	force     bool
}

func parseCmdLine() (*Options, error) {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, `Usage:

  %s [options] <command> [args]

Command:

  import    Imports the zookeeper tree from XML file.
            Must be specified with -zookeeper AND -xmlfile options.
            Optionally takes -path for importing subtree

  export    Exports the zookeeper tree to XML file.
            Must be specified with -zookeeper option.
            Optionally takes -path for exporting subtree

  update    Updates zookeeper tree with changes from XML file.
            Update operation is interactive unless specified with -force option.
            Must be specified with -zookeeper AND -xmlfile options.
            Optionally takes -path for updating subtree.

  diff      Creates a list of diff actions on ZK tree based on XML data.
            Must be specified with -zookeeper OR -xmlfile options.
            Optionally takes -path for subtree diff

  dump      Dumps the entire ZK (sub)tree to standard output.
            Must be specified with --zookeeper OR --xmlfile options.
            Optionally takes --path and --depth for dumping subtree.

Options:

`, os.Args[0])

		flag.PrintDefaults()
	}

	var opts Options

	flag.StringVar(&opts.zkHosts, "zookeeper", "localhost:2181", "specifies information to connect to zookeeper.")
	flag.StringVar(&opts.xmlFile, "xmlfile", "", "Zookeeper tree-data XML file.")
	flag.StringVar(&opts.znodePath, "path", "/", "Path to the zookeeper subtree rootnode.")
	flag.IntVar(&opts.depth, "depth", -1, "Depth of the ZK tree to be dumped (ignored for XML dump).")
	flag.BoolVar(&opts.force, "force", false, "Forces cleanup before import; also used for forceful update.")

	flag.Parse()

	if flag.NArg() == 0 {
		return nil, errors.New("missing command")
	}

	cmd := flag.Arg(0)

	switch cmd {
	case "import", "update", "diff":
		if len(opts.zkHosts) == 0 || len(opts.xmlFile) == 0 {
			return nil, errors.New("missing params")
		}

	case "export", "dump":
		if len(opts.zkHosts) == 0 {
			return nil, errors.New("missing params")
		}

	default:
		return nil, fmt.Errorf("unknown command: %s", cmd)
	}

	opts.cmd = cmd
	opts.args = flag.Args()[1:]

	return &opts, nil
}

func main() {
	if opts, err := parseCmdLine(); err != nil {
		fmt.Fprintf(os.Stderr, "error: %s\n\n", err.Error())

		flag.Usage()

		os.Exit(-1)
	} else {
		switch opts.cmd {
		case "import":
			if liveTree, err := NewZkTree(strings.Split(opts.zkHosts, ";"), opts.znodePath); err != nil {
				log.Fatalf("fail to connect %s, %s", opts.zkHosts, err)
			} else if loadedTree, err := LoadZkTree(opts.xmlFile); err != nil {
				log.Fatalf("fail to load from %s, %s", opts.xmlFile, err)
			} else if err := liveTree.Write(loadedTree, opts.force); err != nil {
				log.Fatalf("fail to write to %s, %s", opts.znodePath, err)
			} else {
				log.Println("import successful!")
			}

		case "export":
			if liveTree, err := NewZkTree(strings.Split(opts.zkHosts, ";"), opts.znodePath); err != nil {
				log.Fatalf("fail to connect %s, %s", opts.zkHosts, err)
			} else if xml, err := liveTree.Xml(); err != nil {
				log.Fatalf("fail to dump XML from %s, %s", opts.znodePath, err)
			} else {
				os.Stdout.Write(xml)
			}

		case "update":
			if liveTree, err := NewZkTree(strings.Split(opts.zkHosts, ";"), opts.znodePath); err != nil {
				log.Fatalf("fail to connect %s, %s", opts.zkHosts, err)
			} else if loadedTree, err := LoadZkTree(opts.xmlFile); err != nil {
				log.Fatalf("fail to load from %s, %s", opts.xmlFile, err)
			} else if actions, err := liveTree.Diff(loadedTree); err != nil {
				log.Fatalf("fail to diff tree at %s, %s", opts.znodePath, err)
			} else {
				var handler ZkActionHandler

				if opts.force {
					handler = &ZkActionExecutor{}
				} else {
					handler = &ZkActionInteractiveExecutor{}
				}

				if err := liveTree.Execute(actions, handler); err != nil {
					log.Fatalf("fail to execute actions, %s", err)
				} else {
					log.Println("update successful!")
				}
			}

		case "diff":
			if liveTree, err := NewZkTree(strings.Split(opts.zkHosts, ";"), opts.znodePath); err != nil {
				log.Fatalf("fail to connect %s, %s", opts.zkHosts, err)
			} else if loadedTree, err := LoadZkTree(opts.xmlFile); err != nil {
				log.Fatalf("fail to load from %s, %s", opts.xmlFile, err)
			} else if actions, err := liveTree.Diff(loadedTree); err != nil {
				log.Fatalf("fail to diff tree at %s, %s", opts.znodePath, err)
			} else if err := liveTree.Execute(actions, &ZkActionPrinter{os.Stdout}); err != nil {
				log.Fatalf("fail to execute actions, %s", err)
			}

		case "dump":
			var tree ZkTree

			if len(opts.zkHosts) > 0 {
				if liveTree, err := NewZkTree(strings.Split(opts.zkHosts, ";"), opts.znodePath); err != nil {
					log.Fatalf("fail to connect %s, %s", opts.zkHosts, err)
				} else {
					tree = liveTree
				}
			} else if len(opts.xmlFile) > 0 {
				if loadedTree, err := LoadZkTree(opts.xmlFile); err != nil {
					log.Fatalf("fail to load from %s, %s", opts.xmlFile, err)
				} else {
					tree = loadedTree
				}
			}

			if out, err := tree.Dump(opts.depth); err != nil {
				log.Fatalf("fail to dump tree, %s", err)
			} else {
				os.Stdout.WriteString(out)
			}
		}
	}
}
<commit_msg>support export to XML file<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"
)

type Options struct {
	cmd       string
	args      []string
	zkHosts   string
	xmlFile   string
	znodePath string
	depth     int
	force     bool
}

func parseCmdLine() (*Options, error) {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, `Usage:

  %s [options] <command> [args]

Command:

  import    Imports the zookeeper tree from XML file.
            Must be specified with -zookeeper AND -xmlfile options.
            Optionally takes -path for importing subtree

  export    Exports the zookeeper tree to XML file.
            Must be specified with -zookeeper option.
            Optionally takes -path for exporting subtree

  update    Updates zookeeper tree with changes from XML file.
            Update operation is interactive unless specified with -force option.
            Must be specified with -zookeeper AND -xmlfile options.
            Optionally takes -path for updating subtree.

  diff      Creates a list of diff actions on ZK tree based on XML data.
            Must be specified with -zookeeper OR -xmlfile options.
            Optionally takes -path for subtree diff

  dump      Dumps the entire ZK (sub)tree to standard output.
            Must be specified with --zookeeper OR --xmlfile options.
            Optionally takes --path and --depth for dumping subtree.

Options:

`, os.Args[0])

		flag.PrintDefaults()
	}

	var opts Options

	flag.StringVar(&opts.zkHosts, "zookeeper", "localhost:2181", "specifies information to connect to zookeeper.")
	flag.StringVar(&opts.xmlFile, "xmlfile", "", "Zookeeper tree-data XML file.")
	flag.StringVar(&opts.znodePath, "path", "/", "Path to the zookeeper subtree rootnode.")
	flag.IntVar(&opts.depth, "depth", -1, "Depth of the ZK tree to be dumped (ignored for XML dump).")
	flag.BoolVar(&opts.force, "force", false, "Forces cleanup before import; also used for forceful update.")

	flag.Parse()

	if flag.NArg() == 0 {
		return nil, errors.New("missing command")
	}

	cmd := flag.Arg(0)

	switch cmd {
	case "import", "update", "diff":
		if len(opts.zkHosts) == 0 || len(opts.xmlFile) == 0 {
			return nil, errors.New("missing params")
		}

	case "export", "dump":
		if len(opts.zkHosts) == 0 {
			return nil, errors.New("missing params")
		}

	default:
		return nil, fmt.Errorf("unknown command: %s", cmd)
	}

	opts.cmd = cmd
	opts.args = flag.Args()[1:]

	return &opts, nil
}

func main() {
	if opts, err := parseCmdLine(); err != nil {
		fmt.Fprintf(os.Stderr, "error: %s\n\n", err.Error())

		flag.Usage()

		os.Exit(-1)
	} else {
		switch opts.cmd {
		case "import":
			if liveTree, err := NewZkTree(strings.Split(opts.zkHosts, ";"), opts.znodePath); err != nil {
				log.Fatalf("fail to connect %s, %s", opts.zkHosts, err)
			} else if loadedTree, err := LoadZkTree(opts.xmlFile); err != nil {
				log.Fatalf("fail to load from %s, %s", opts.xmlFile, err)
			} else if err := liveTree.Write(loadedTree, opts.force); err != nil {
				log.Fatalf("fail to write to %s, %s", opts.znodePath, err)
			} else {
				log.Println("import successful!")
			}

		case "export":
			if liveTree, err := NewZkTree(strings.Split(opts.zkHosts, ";"), opts.znodePath); err != nil {
				log.Fatalf("fail to connect %s, %s", opts.zkHosts, err)
			} else if xml, err := liveTree.Xml(); err != nil {
				log.Fatalf("fail to dump XML from %s, %s", opts.znodePath, err)
			} else if len(opts.xmlFile) == 0 {
				os.Stdout.Write(xml)
			} else if err := ioutil.WriteFile(opts.xmlFile, xml, 0644); err != nil {
				log.Fatalf("fail to write XML file `%s`, %s", opts.xmlFile, err)
			}

		case "update":
			if liveTree, err := NewZkTree(strings.Split(opts.zkHosts, ";"), opts.znodePath); err != nil {
				log.Fatalf("fail to connect %s, %s", opts.zkHosts, err)
			} else if loadedTree, err := LoadZkTree(opts.xmlFile); err != nil {
				log.Fatalf("fail to load from %s, %s", opts.xmlFile, err)
			} else if actions, err := liveTree.Diff(loadedTree); err != nil {
				log.Fatalf("fail to diff tree at %s, %s", opts.znodePath, err)
			} else {
				var handler ZkActionHandler

				if opts.force {
					handler = &ZkActionExecutor{}
				} else {
					handler = &ZkActionInteractiveExecutor{}
				}

				if err := liveTree.Execute(actions, handler); err != nil {
					log.Fatalf("fail to execute actions, %s", err)
				} else {
					log.Println("update successful!")
				}
			}

		case "diff":
			if liveTree, err := NewZkTree(strings.Split(opts.zkHosts, ";"), opts.znodePath); err != nil {
				log.Fatalf("fail to connect %s, %s", opts.zkHosts, err)
			} else if loadedTree, err := LoadZkTree(opts.xmlFile); err != nil {
				log.Fatalf("fail to load from %s, %s", opts.xmlFile, err)
			} else if actions, err := liveTree.Diff(loadedTree); err != nil {
				log.Fatalf("fail to diff tree at %s, %s", opts.znodePath, err)
			} else if err := liveTree.Execute(actions, &ZkActionPrinter{os.Stdout}); err != nil {
				log.Fatalf("fail to execute actions, %s", err)
			}

		case "dump":
			var tree ZkTree

			if len(opts.zkHosts) > 0 {
				if liveTree, err := NewZkTree(strings.Split(opts.zkHosts, ";"), opts.znodePath); err != nil {
					log.Fatalf("fail to connect %s, %s", opts.zkHosts, err)
				} else {
					tree = liveTree
				}
			} else if len(opts.xmlFile) > 0 {
				if loadedTree, err := LoadZkTree(opts.xmlFile); err != nil {
					log.Fatalf("fail to load from %s, %s", opts.xmlFile, err)
				} else {
					tree = loadedTree
				}
			}

			if out, err := tree.Dump(opts.depth); err != nil {
				log.Fatalf("fail to dump tree, %s", err)
			} else {
				os.Stdout.WriteString(out)
			}
		}
	}
}
<|endoftext|>


<commit_before>// package bitswap implements the IPFS Exchange interface with the BitSwap
// bilateral exchange protocol.
package bitswap

import (
	"math"
	"sync"
	"time"

	process "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess"
	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
	blocks "github.com/jbenet/go-ipfs/blocks"
	blockstore "github.com/jbenet/go-ipfs/blocks/blockstore"
	exchange "github.com/jbenet/go-ipfs/exchange"
	decision "github.com/jbenet/go-ipfs/exchange/bitswap/decision"
	bsmsg "github.com/jbenet/go-ipfs/exchange/bitswap/message"
	bsnet "github.com/jbenet/go-ipfs/exchange/bitswap/network"
	notifications "github.com/jbenet/go-ipfs/exchange/bitswap/notifications"
	wantlist "github.com/jbenet/go-ipfs/exchange/bitswap/wantlist"
	peer "github.com/jbenet/go-ipfs/p2p/peer"
	"github.com/jbenet/go-ipfs/thirdparty/delay"
	eventlog "github.com/jbenet/go-ipfs/thirdparty/eventlog"
	u "github.com/jbenet/go-ipfs/util"
	errors "github.com/jbenet/go-ipfs/util/debugerror"
	pset "github.com/jbenet/go-ipfs/util/peerset" // TODO move this to peerstore
)

var log = eventlog.Logger("bitswap")

const (
	// maxProvidersPerRequest specifies the maximum number of providers desired
	// from the network. This value is specified because the network streams
	// results.
	// TODO: if a 'non-nice' strategy is implemented, consider increasing this value
	maxProvidersPerRequest = 3
	providerRequestTimeout = time.Second * 10
	hasBlockTimeout        = time.Second * 15
	provideTimeout         = time.Second * 15
	sizeBatchRequestChan   = 32
	// kMaxPriority is the max priority as defined by the bitswap protocol
	kMaxPriority = math.MaxInt32

	HasBlockBufferSize = 256
	provideWorkers     = 4
)

var (
	rebroadcastDelay = delay.Fixed(time.Second * 10)
)

// New initializes a BitSwap instance that communicates over the provided
// BitSwapNetwork. This function registers the returned instance as the network
// delegate.
// Runs until context is cancelled.
func New(parent context.Context, p peer.ID, network bsnet.BitSwapNetwork,
	bstore blockstore.Blockstore, nice bool) exchange.Interface {

	// important to use provided parent context (since it may include important
	// loggable data). It's probably not a good idea to allow bitswap to be
	// coupled to the concerns of the IPFS daemon in this way.
	//
	// FIXME(btc) Now that bitswap manages itself using a process, it probably
	// shouldn't accept a context anymore. Clients should probably use Close()
	// exclusively. We should probably find another way to share logging data
	ctx, cancelFunc := context.WithCancel(parent)

	notif := notifications.New()
	px := process.WithTeardown(func() error {
		notif.Shutdown()
		return nil
	})

	go func() {
		<-px.Closing() // process closes first
		cancelFunc()
	}()
	go func() {
		<-ctx.Done() // parent cancelled first
		px.Close()
	}()

	bs := &Bitswap{
		self:          p,
		blockstore:    bstore,
		notifications: notif,
		engine:        decision.NewEngine(ctx, bstore), // TODO close the engine with Close() method
		network:       network,
		wantlist:      wantlist.NewThreadSafe(),
		batchRequests: make(chan *blockRequest, sizeBatchRequestChan),
		process:       px,
		newBlocks:     make(chan *blocks.Block, HasBlockBufferSize),
		provideKeys:   make(chan u.Key),
	}
	network.SetDelegate(bs)

	// Start up bitswaps async worker routines
	bs.startWorkers(px, ctx)
	return bs
}

// Bitswap instances implement the bitswap protocol.
type Bitswap struct {

	// the ID of the peer to act on behalf of
	self peer.ID

	// network delivers messages on behalf of the session
	network bsnet.BitSwapNetwork

	// blockstore is the local database
	// NB: ensure threadsafety
	blockstore blockstore.Blockstore

	notifications notifications.PubSub

	// Requests for a set of related blocks
	// the assumption is made that the same peer is likely to
	// have more than a single block in the set
	batchRequests chan *blockRequest

	engine *decision.Engine

	wantlist *wantlist.ThreadSafe

	process process.Process

	newBlocks chan *blocks.Block

	provideKeys chan u.Key
}

type blockRequest struct {
	keys []u.Key
	ctx  context.Context
}

// GetBlock attempts to retrieve a particular block from peers within the
// deadline enforced by the context.
func (bs *Bitswap) GetBlock(parent context.Context, k u.Key) (*blocks.Block, error) {

	// Any async work initiated by this function must end when this function
	// returns. To ensure this, derive a new context. Note that it is okay to
	// listen on parent in this scope, but NOT okay to pass |parent| to
	// functions called by this one. Otherwise those functions won't return
	// when this context's cancel func is executed. This is difficult to
	// enforce. May this comment keep you safe.

	ctx, cancelFunc := context.WithCancel(parent)

	ctx = eventlog.ContextWithLoggable(ctx, eventlog.Uuid("GetBlockRequest"))
	defer log.EventBegin(ctx, "GetBlockRequest", &k).Done()

	defer func() {
		cancelFunc()
	}()

	promise, err := bs.GetBlocks(ctx, []u.Key{k})
	if err != nil {
		return nil, err
	}

	select {
	case block, ok := <-promise:
		if !ok {
			select {
			case <-ctx.Done():
				return nil, ctx.Err()
			default:
				return nil, errors.New("promise channel was closed")
			}
		}
		return block, nil
	case <-parent.Done():
		return nil, parent.Err()
	}
}

// GetBlocks returns a channel where the caller may receive blocks that
// correspond to the provided |keys|. Returns an error if BitSwap is unable to
// begin this request within the deadline enforced by the context.
//
// NB: Your request remains open until the context expires. To conserve
// resources, provide a context with a reasonably short deadline (ie. not one
// that lasts throughout the lifetime of the server)
func (bs *Bitswap) GetBlocks(ctx context.Context, keys []u.Key) (<-chan *blocks.Block, error) {
	select {
	case <-bs.process.Closing():
		return nil, errors.New("bitswap is closed")
	default:
	}
	promise := bs.notifications.Subscribe(ctx, keys...)

	req := &blockRequest{
		keys: keys,
		ctx:  ctx,
	}
	select {
	case bs.batchRequests <- req:
		return promise, nil
	case <-ctx.Done():
		return nil, ctx.Err()
	}
}

// HasBlock announces the existance of a block to this bitswap service. The
// service will potentially notify its peers.
func (bs *Bitswap) HasBlock(ctx context.Context, blk *blocks.Block) error {
	log.Event(ctx, "hasBlock", blk)
	select {
	case <-bs.process.Closing():
		return errors.New("bitswap is closed")
	default:
	}
	if err := bs.blockstore.Put(blk); err != nil {
		return err
	}
	bs.wantlist.Remove(blk.Key())
	bs.notifications.Publish(blk)
	select {
	case bs.newBlocks <- blk:
	case <-ctx.Done():
		return ctx.Err()
	}
	return nil
}

func (bs *Bitswap) sendWantlistMsgToPeers(ctx context.Context, m bsmsg.BitSwapMessage, peers <-chan peer.ID) error {
	set := pset.New()
	wg := sync.WaitGroup{}

loop:
	for {
		select {
		case peerToQuery, ok := <-peers:
			if !ok {
				break loop
			}

			if !set.TryAdd(peerToQuery) { //Do once per peer
				continue
			}

			wg.Add(1)
			go func(p peer.ID) {
				defer wg.Done()
				if err := bs.send(ctx, p, m); err != nil {
					log.Debug(err) // TODO remove if too verbose
				}
			}(peerToQuery)
		case <-ctx.Done():
			return nil
		}
	}
	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	select {
	case <-done:
	case <-ctx.Done():
	}
	return nil
}

func (bs *Bitswap) sendWantlistToPeers(ctx context.Context, peers <-chan peer.ID) error {
	message := bsmsg.New()
	message.SetFull(true)
	for _, wanted := range bs.wantlist.Entries() {
		message.AddEntry(wanted.Key, wanted.Priority)
	}
	return bs.sendWantlistMsgToPeers(ctx, message, peers)
}

func (bs *Bitswap) sendWantlistToProviders(ctx context.Context, entries []wantlist.Entry) {

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	// prepare a channel to hand off to sendWantlistToPeers
	sendToPeers := make(chan peer.ID)

	// Get providers for all entries in wantlist (could take a while)
	wg := sync.WaitGroup{}
	for _, e := range entries {
		wg.Add(1)
		go func(k u.Key) {
			defer wg.Done()

			child, cancel := context.WithTimeout(ctx, providerRequestTimeout)
			defer cancel()
			providers := bs.network.FindProvidersAsync(child, k, maxProvidersPerRequest)
			for prov := range providers {
				sendToPeers <- prov
			}
		}(e.Key)
	}

	go func() {
		wg.Wait() // make sure all our children do finish.
		close(sendToPeers)
	}()

	err := bs.sendWantlistToPeers(ctx, sendToPeers)
	if err != nil {
		log.Debugf("sendWantlistToPeers error: %s", err)
	}
}

// TODO(brian): handle errors
func (bs *Bitswap) ReceiveMessage(ctx context.Context, p peer.ID, incoming bsmsg.BitSwapMessage) (
	peer.ID, bsmsg.BitSwapMessage) {
	defer log.EventBegin(ctx, "receiveMessage", p, incoming).Done()

	if p == "" {
		log.Debug("Received message from nil peer!")
		// TODO propagate the error upward
		return "", nil
	}
	if incoming == nil {
		log.Debug("Got nil bitswap message!")
		// TODO propagate the error upward
		return "", nil
	}

	// This call records changes to wantlists, blocks received,
	// and number of bytes transfered.
	bs.engine.MessageReceived(p, incoming)
	// TODO: this is bad, and could be easily abused.
	// Should only track *useful* messages in ledger

	for _, block := range incoming.Blocks() {
		hasBlockCtx, cancel := context.WithTimeout(ctx, hasBlockTimeout)
		if err := bs.HasBlock(hasBlockCtx, block); err != nil {
			log.Debug(err)
		}
		cancel()
	}

	var keys []u.Key
	for _, block := range incoming.Blocks() {
		keys = append(keys, block.Key())
	}
	bs.cancelBlocks(ctx, keys)

	// TODO: consider changing this function to not return anything
	return "", nil
}

// Connected/Disconnected warns bitswap about peer connections
func (bs *Bitswap) PeerConnected(p peer.ID) {
	// TODO: add to clientWorker??
	peers := make(chan peer.ID, 1)
	peers <- p
	close(peers)
	err := bs.sendWantlistToPeers(context.TODO(), peers)
	if err != nil {
		log.Debugf("error sending wantlist: %s", err)
	}
}

// Connected/Disconnected warns bitswap about peer connections
func (bs *Bitswap) PeerDisconnected(p peer.ID) {
	bs.engine.PeerDisconnected(p)
}

func (bs *Bitswap) cancelBlocks(ctx context.Context, bkeys []u.Key) {
	if len(bkeys) < 1 {
		return
	}
	message := bsmsg.New()
	message.SetFull(false)
	for _, k := range bkeys {
		message.Cancel(k)
	}
	for _, p := range bs.engine.Peers() {
		err := bs.send(ctx, p, message)
		if err != nil {
			log.Debugf("Error sending message: %s", err)
		}
	}
}

func (bs *Bitswap) wantNewBlocks(ctx context.Context, bkeys []u.Key) {
	if len(bkeys) < 1 {
		return
	}

	message := bsmsg.New()
	message.SetFull(false)
	for i, k := range bkeys {
		message.AddEntry(k, kMaxPriority-i)
	}

	wg := sync.WaitGroup{}
	for _, p := range bs.engine.Peers() {
		wg.Add(1)
		go func(p peer.ID) {
			defer wg.Done()
			err := bs.send(ctx, p, message)
			if err != nil {
				log.Debugf("Error sending message: %s", err)
			}
		}(p)
	}
	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()
	select {
	case <-done:
	case <-ctx.Done():
	}
}

func (bs *Bitswap) ReceiveError(err error) {
	log.Debugf("Bitswap ReceiveError: %s", err)
	// TODO log the network error
	// TODO bubble the network error up to the parent context/error logger
}

// send strives to ensure that accounting is always performed when a message is
// sent
func (bs *Bitswap) send(ctx context.Context, p peer.ID, m bsmsg.BitSwapMessage) error {
	defer log.EventBegin(ctx, "sendMessage", p, m).Done()
	if err := bs.network.SendMessage(ctx, p, m); err != nil {
		return errors.Wrap(err)
	}
	return bs.engine.MessageSent(p, m)
}

func (bs *Bitswap) Close() error {
	return bs.process.Close()
}

func (bs *Bitswap) GetWantlist() []u.Key {
	var out []u.Key
	for _, e := range bs.wantlist.Entries() {
		out = append(out, e.Key)
	}
	return out
}
<commit_msg>add warning comment about possibly leaked goroutines<commit_after>// package bitswap implements the IPFS Exchange interface with the BitSwap
// bilateral exchange protocol.
package bitswap

import (
	"math"
	"sync"
	"time"

	process "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess"
	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
	blocks "github.com/jbenet/go-ipfs/blocks"
	blockstore "github.com/jbenet/go-ipfs/blocks/blockstore"
	exchange "github.com/jbenet/go-ipfs/exchange"
	decision "github.com/jbenet/go-ipfs/exchange/bitswap/decision"
	bsmsg "github.com/jbenet/go-ipfs/exchange/bitswap/message"
	bsnet "github.com/jbenet/go-ipfs/exchange/bitswap/network"
	notifications "github.com/jbenet/go-ipfs/exchange/bitswap/notifications"
	wantlist "github.com/jbenet/go-ipfs/exchange/bitswap/wantlist"
	peer "github.com/jbenet/go-ipfs/p2p/peer"
	"github.com/jbenet/go-ipfs/thirdparty/delay"
	eventlog "github.com/jbenet/go-ipfs/thirdparty/eventlog"
	u "github.com/jbenet/go-ipfs/util"
	errors "github.com/jbenet/go-ipfs/util/debugerror"
	pset "github.com/jbenet/go-ipfs/util/peerset" // TODO move this to peerstore
)

var log = eventlog.Logger("bitswap")

const (
	// maxProvidersPerRequest specifies the maximum number of providers desired
	// from the network. This value is specified because the network streams
	// results.
	// TODO: if a 'non-nice' strategy is implemented, consider increasing this value
	maxProvidersPerRequest = 3
	providerRequestTimeout = time.Second * 10
	hasBlockTimeout        = time.Second * 15
	provideTimeout         = time.Second * 15
	sizeBatchRequestChan   = 32
	// kMaxPriority is the max priority as defined by the bitswap protocol
	kMaxPriority = math.MaxInt32

	HasBlockBufferSize = 256
	provideWorkers     = 4
)

var (
	rebroadcastDelay = delay.Fixed(time.Second * 10)
)

// New initializes a BitSwap instance that communicates over the provided
// BitSwapNetwork. This function registers the returned instance as the network
// delegate.
// Runs until context is cancelled.
func New(parent context.Context, p peer.ID, network bsnet.BitSwapNetwork,
	bstore blockstore.Blockstore, nice bool) exchange.Interface {

	// important to use provided parent context (since it may include important
	// loggable data). It's probably not a good idea to allow bitswap to be
	// coupled to the concerns of the IPFS daemon in this way.
	//
	// FIXME(btc) Now that bitswap manages itself using a process, it probably
	// shouldn't accept a context anymore. Clients should probably use Close()
	// exclusively. We should probably find another way to share logging data
	ctx, cancelFunc := context.WithCancel(parent)

	notif := notifications.New()
	px := process.WithTeardown(func() error {
		notif.Shutdown()
		return nil
	})

	go func() {
		<-px.Closing() // process closes first
		cancelFunc()
	}()
	go func() {
		<-ctx.Done() // parent cancelled first
		px.Close()
	}()

	bs := &Bitswap{
		self:          p,
		blockstore:    bstore,
		notifications: notif,
		engine:        decision.NewEngine(ctx, bstore), // TODO close the engine with Close() method
		network:       network,
		wantlist:      wantlist.NewThreadSafe(),
		batchRequests: make(chan *blockRequest, sizeBatchRequestChan),
		process:       px,
		newBlocks:     make(chan *blocks.Block, HasBlockBufferSize),
		provideKeys:   make(chan u.Key),
	}
	network.SetDelegate(bs)

	// Start up bitswaps async worker routines
	bs.startWorkers(px, ctx)
	return bs
}

// Bitswap instances implement the bitswap protocol.
type Bitswap struct {

	// the ID of the peer to act on behalf of
	self peer.ID

	// network delivers messages on behalf of the session
	network bsnet.BitSwapNetwork

	// blockstore is the local database
	// NB: ensure threadsafety
	blockstore blockstore.Blockstore

	notifications notifications.PubSub

	// Requests for a set of related blocks
	// the assumption is made that the same peer is likely to
	// have more than a single block in the set
	batchRequests chan *blockRequest

	engine *decision.Engine

	wantlist *wantlist.ThreadSafe

	process process.Process

	newBlocks chan *blocks.Block

	provideKeys chan u.Key
}

type blockRequest struct {
	keys []u.Key
	ctx  context.Context
}

// GetBlock attempts to retrieve a particular block from peers within the
// deadline enforced by the context.
func (bs *Bitswap) GetBlock(parent context.Context, k u.Key) (*blocks.Block, error) {

	// Any async work initiated by this function must end when this function
	// returns. To ensure this, derive a new context. Note that it is okay to
	// listen on parent in this scope, but NOT okay to pass |parent| to
	// functions called by this one. Otherwise those functions won't return
	// when this context's cancel func is executed. This is difficult to
	// enforce. May this comment keep you safe.

	ctx, cancelFunc := context.WithCancel(parent)

	ctx = eventlog.ContextWithLoggable(ctx, eventlog.Uuid("GetBlockRequest"))
	defer log.EventBegin(ctx, "GetBlockRequest", &k).Done()

	defer func() {
		cancelFunc()
	}()

	promise, err := bs.GetBlocks(ctx, []u.Key{k})
	if err != nil {
		return nil, err
	}

	select {
	case block, ok := <-promise:
		if !ok {
			select {
			case <-ctx.Done():
				return nil, ctx.Err()
			default:
				return nil, errors.New("promise channel was closed")
			}
		}
		return block, nil
	case <-parent.Done():
		return nil, parent.Err()
	}
}

// GetBlocks returns a channel where the caller may receive blocks that
// correspond to the provided |keys|. Returns an error if BitSwap is unable to
// begin this request within the deadline enforced by the context.
//
// NB: Your request remains open until the context expires. To conserve
// resources, provide a context with a reasonably short deadline (ie. not one
// that lasts throughout the lifetime of the server)
func (bs *Bitswap) GetBlocks(ctx context.Context, keys []u.Key) (<-chan *blocks.Block, error) {
	select {
	case <-bs.process.Closing():
		return nil, errors.New("bitswap is closed")
	default:
	}
	promise := bs.notifications.Subscribe(ctx, keys...)

	req := &blockRequest{
		keys: keys,
		ctx:  ctx,
	}
	select {
	case bs.batchRequests <- req:
		return promise, nil
	case <-ctx.Done():
		return nil, ctx.Err()
	}
}

// HasBlock announces the existance of a block to this bitswap service. The
// service will potentially notify its peers.
func (bs *Bitswap) HasBlock(ctx context.Context, blk *blocks.Block) error {
	log.Event(ctx, "hasBlock", blk)
	select {
	case <-bs.process.Closing():
		return errors.New("bitswap is closed")
	default:
	}
	if err := bs.blockstore.Put(blk); err != nil {
		return err
	}
	bs.wantlist.Remove(blk.Key())
	bs.notifications.Publish(blk)
	select {
	case bs.newBlocks <- blk:
	case <-ctx.Done():
		return ctx.Err()
	}
	return nil
}

func (bs *Bitswap) sendWantlistMsgToPeers(ctx context.Context, m bsmsg.BitSwapMessage, peers <-chan peer.ID) error {
	set := pset.New()
	wg := sync.WaitGroup{}

loop:
	for {
		select {
		case peerToQuery, ok := <-peers:
			if !ok {
				break loop
			}

			if !set.TryAdd(peerToQuery) { //Do once per peer
				continue
			}

			wg.Add(1)
			go func(p peer.ID) {
				defer wg.Done()
				if err := bs.send(ctx, p, m); err != nil {
					log.Debug(err) // TODO remove if too verbose
				}
			}(peerToQuery)
		case <-ctx.Done():
			return nil
		}
	}
	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	select {
	case <-done:
	case <-ctx.Done():
		// NB: we may be abandoning goroutines here before they complete
		// this shouldnt be an issue because they will complete soon anyways
		// we just don't want their being slow to impact bitswap transfer speeds
	}
	return nil
}

func (bs *Bitswap) sendWantlistToPeers(ctx context.Context, peers <-chan peer.ID) error {
	message := bsmsg.New()
	message.SetFull(true)
	for _, wanted := range bs.wantlist.Entries() {
		message.AddEntry(wanted.Key, wanted.Priority)
	}
	return bs.sendWantlistMsgToPeers(ctx, message, peers)
}

func (bs *Bitswap) sendWantlistToProviders(ctx context.Context, entries []wantlist.Entry) {

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	// prepare a channel to hand off to sendWantlistToPeers
	sendToPeers := make(chan peer.ID)

	// Get providers for all entries in wantlist (could take a while)
	wg := sync.WaitGroup{}
	for _, e := range entries {
		wg.Add(1)
		go func(k u.Key) {
			defer wg.Done()

			child, cancel := context.WithTimeout(ctx, providerRequestTimeout)
			defer cancel()
			providers := bs.network.FindProvidersAsync(child, k, maxProvidersPerRequest)
			for prov := range providers {
				sendToPeers <- prov
			}
		}(e.Key)
	}

	go func() {
		wg.Wait() // make sure all our children do finish.
		close(sendToPeers)
	}()

	err := bs.sendWantlistToPeers(ctx, sendToPeers)
	if err != nil {
		log.Debugf("sendWantlistToPeers error: %s", err)
	}
}

// TODO(brian): handle errors
func (bs *Bitswap) ReceiveMessage(ctx context.Context, p peer.ID, incoming bsmsg.BitSwapMessage) (
	peer.ID, bsmsg.BitSwapMessage) {
	defer log.EventBegin(ctx, "receiveMessage", p, incoming).Done()

	if p == "" {
		log.Debug("Received message from nil peer!")
		// TODO propagate the error upward
		return "", nil
	}
	if incoming == nil {
		log.Debug("Got nil bitswap message!")
		// TODO propagate the error upward
		return "", nil
	}

	// This call records changes to wantlists, blocks received,
	// and number of bytes transfered.
	bs.engine.MessageReceived(p, incoming)
	// TODO: this is bad, and could be easily abused.
	// Should only track *useful* messages in ledger

	for _, block := range incoming.Blocks() {
		hasBlockCtx, cancel := context.WithTimeout(ctx, hasBlockTimeout)
		if err := bs.HasBlock(hasBlockCtx, block); err != nil {
			log.Debug(err)
		}
		cancel()
	}

	var keys []u.Key
	for _, block := range incoming.Blocks() {
		keys = append(keys, block.Key())
	}
	bs.cancelBlocks(ctx, keys)

	// TODO: consider changing this function to not return anything
	return "", nil
}

// Connected/Disconnected warns bitswap about peer connections
func (bs *Bitswap) PeerConnected(p peer.ID) {
	// TODO: add to clientWorker??
	peers := make(chan peer.ID, 1)
	peers <- p
	close(peers)
	err := bs.sendWantlistToPeers(context.TODO(), peers)
	if err != nil {
		log.Debugf("error sending wantlist: %s", err)
	}
}

// Connected/Disconnected warns bitswap about peer connections
func (bs *Bitswap) PeerDisconnected(p peer.ID) {
	bs.engine.PeerDisconnected(p)
}

func (bs *Bitswap) cancelBlocks(ctx context.Context, bkeys []u.Key) {
	if len(bkeys) < 1 {
		return
	}
	message := bsmsg.New()
	message.SetFull(false)
	for _, k := range bkeys {
		message.Cancel(k)
	}
	for _, p := range bs.engine.Peers() {
		err := bs.send(ctx, p, message)
		if err != nil {
			log.Debugf("Error sending message: %s", err)
		}
	}
}

func (bs *Bitswap) wantNewBlocks(ctx context.Context, bkeys []u.Key) {
	if len(bkeys) < 1 {
		return
	}

	message := bsmsg.New()
	message.SetFull(false)
	for i, k := range bkeys {
		message.AddEntry(k, kMaxPriority-i)
	}

	wg := sync.WaitGroup{}
	for _, p := range bs.engine.Peers() {
		wg.Add(1)
		go func(p peer.ID) {
			defer wg.Done()
			err := bs.send(ctx, p, message)
			if err != nil {
				log.Debugf("Error sending message: %s", err)
			}
		}(p)
	}
	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()
	select {
	case <-done:
	case <-ctx.Done():
		// NB: we may be abandoning goroutines here before they complete
		// this shouldnt be an issue because they will complete soon anyways
		// we just don't want their being slow to impact bitswap transfer speeds
	}
}

func (bs *Bitswap) ReceiveError(err error) {
	log.Debugf("Bitswap ReceiveError: %s", err)
	// TODO log the network error
	// TODO bubble the network error up to the parent context/error logger
}

// send strives to ensure that accounting is always performed when a message is
// sent
func (bs *Bitswap) send(ctx context.Context, p peer.ID, m bsmsg.BitSwapMessage) error {
	defer log.EventBegin(ctx, "sendMessage", p, m).Done()
	if err := bs.network.SendMessage(ctx, p, m); err != nil {
		return errors.Wrap(err)
	}
	return bs.engine.MessageSent(p, m)
}

func (bs *Bitswap) Close() error {
	return bs.process.Close()
}

func (bs *Bitswap) GetWantlist() []u.Key {
	var out []u.Key
	for _, e := range bs.wantlist.Entries() {
		out = append(out, e.Key)
	}
	return out
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/grafov/m3u8"
	"gopkg.in/redis.v1"
	"log"
	"net/http"
	"os/exec"
	"time"
)

var broadcastCursor = make(chan int)
var currentPlaylist string
var client *redis.Client

func init() {
	client = redis.NewTCPClient(&redis.Options{
		Addr: "localhost:6379",
	})

	pong, err := client.Ping().Result()
	log.Println(pong, err)
}

type PlaylistGenerator struct {
	cursor chan int
}

func (pl PlaylistGenerator) VideoFileForSequence(seq int) string {
	generated := fmt.Sprintf("http://www.smick.tv/media/truedetectives2e1movie%05d.ts", seq)
	return generated
}

func (pl *PlaylistGenerator) KeepPlaylistUpdated() {
	p, e := m3u8.NewMediaPlaylist(1000, 1000)
	if e != nil {
		log.Println("Error creating media playlist:", e)
		return
	}
	currentPlaylist = p.Encode().String()

	for seqnum := 1; seqnum < 1854; seqnum = <-pl.cursor {
		videoFile := pl.VideoFileForSequence(seqnum)
		if err := p.Append(videoFile, 5.0, ""); err != nil {
			log.Println("Error appending item to playlist:", err, fmt.Sprintf("movie2m%5d.ts", seqnum))
		}
		currentPlaylist = p.Encode().String()
	}
}

func (pl *PlaylistGenerator) Start() {
	pl.cursor = make(chan int, 1000)

	go pl.KeepPlaylistUpdated()
	for i := 1; i < 1854; i++ {
		log.Println(i)
		pl.cursor <- i
		time.Sleep(5 * time.Second)
	}
}

func (pl PlaylistGenerator) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, currentPlaylist)
}
<commit_msg>removed unnessecary import<commit_after>package main

import (
	"fmt"
	"github.com/grafov/m3u8"
	"gopkg.in/redis.v1"
	"log"
	"net/http"
	"time"
)

var broadcastCursor = make(chan int)
var currentPlaylist string
var client *redis.Client

func init() {
	client = redis.NewTCPClient(&redis.Options{
		Addr: "localhost:6379",
	})

	pong, err := client.Ping().Result()
	log.Println(pong, err)
}

type PlaylistGenerator struct {
	cursor chan int
}

func (pl PlaylistGenerator) VideoFileForSequence(seq int) string {
	generated := fmt.Sprintf("http://www.smick.tv/media/truedetectives2e1movie%05d.ts", seq)
	return generated
}

func (pl *PlaylistGenerator) KeepPlaylistUpdated() {
	p, e := m3u8.NewMediaPlaylist(1000, 1000)
	if e != nil {
		log.Println("Error creating media playlist:", e)
		return
	}
	currentPlaylist = p.Encode().String()

	for seqnum := 1; seqnum < 1854; seqnum = <-pl.cursor {
		videoFile := pl.VideoFileForSequence(seqnum)
		if err := p.Append(videoFile, 5.0, ""); err != nil {
			log.Println("Error appending item to playlist:", err, fmt.Sprintf("movie2m%5d.ts", seqnum))
		}
		currentPlaylist = p.Encode().String()
	}
}

func (pl *PlaylistGenerator) Start() {
	pl.cursor = make(chan int, 1000)

	go pl.KeepPlaylistUpdated()
	for i := 1; i < 1854; i++ {
		log.Println(i)
		pl.cursor <- i
		time.Sleep(5 * time.Second)
	}
}

func (pl PlaylistGenerator) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, currentPlaylist)
}
<|endoftext|>