<commit_before>package goat

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"log"
	"net"
	"net/url"
	"strconv"
	"strings"
	"sync/atomic"
)

// Handshake for UDP tracker protocol
const udpInitID = 4497486125440

// Handle incoming UDP connections and return response
func handleUDP(l *net.UDPConn, sendChan chan bool, recvChan chan bool) {
	// Create shutdown function
	go func(l *net.UDPConn, sendChan chan bool, recvChan chan bool) {
		// Wait for done signal
		<-sendChan

		// Close listener
		if err := l.Close(); err != nil {
			log.Println(err.Error())
		}

		log.Println("UDP listener stopped")
		recvChan <- true
	}(l, sendChan, recvChan)

	// Count incoming connections
	atomic.AddInt64(&static.UDP.Current, 1)
	atomic.AddInt64(&static.UDP.Total, 1)

	first := true
	for {
		buf := make([]byte, 2048)
		rlen, addr, err := l.ReadFromUDP(buf)

		// Triggered on graceful shutdown
		if err != nil {
			return
		}

		// Verify length is at least 16 bytes
		if rlen < 16 {
			log.Println("Invalid length")
			continue
		}

		// Current connection ID (initially handshake, then generated by tracker)
		connID := binary.BigEndian.Uint64(buf[0:8])
		// Action integer (connect: 0, announce: 1)
		action := binary.BigEndian.Uint32(buf[8:12])
		// Transaction ID, to match between requests
		transID := buf[12:16]

		// On first run, verify valid connection ID
		if first {
			if connID != udpInitID {
				log.Println("Invalid connection handshake")
				_, err = l.WriteToUDP(udpTrackerError("Invalid connection handshake", transID), addr)
				if err != nil {
					log.Println(err.Error())
					return
				}
				continue
			}
			first = false
		}

		// Action switch
		switch action {
		// Connect
		case 0:
			res := bytes.NewBuffer(make([]byte, 0))

			// Action
			err = binary.Write(res, binary.BigEndian, uint32(0))
			if err != nil {
				log.Println(err.Error())
				return
			}

			// Transaction ID
			err = binary.Write(res, binary.BigEndian, transID)
			if err != nil {
				log.Println(err.Error())
				return
			}

			// Connection ID, generated for this session
			err = binary.Write(res, binary.BigEndian, uint64(randRange(0, 1000000000)))
			if err != nil {
				log.Println(err.Error())
				return
			}

			_, err := l.WriteToUDP(res.Bytes(), addr)
			if err != nil {
				log.Println(err.Error())
				return
			}

			continue
		// Announce
		case 1:
			query := url.Values{}

			// Ignoring these for now, because clients function sanely without them
			// Connection ID: buf[0:8]
			// Action: buf[8:12]

			// Mark client as UDP
			query.Set("udp", "1")

			// Transaction ID
			transID := buf[12:16]

			// Info hash
			query.Set("info_hash", string(buf[16:36]))

			// Skipped: peer_id: buf[36:56]

			// Downloaded
			t, err := strconv.ParseInt(hex.EncodeToString(buf[56:64]), 16, 64)
			if err != nil {
				log.Println(err.Error())
				return
			}
			query.Set("downloaded", strconv.FormatInt(t, 10))

			// Left
			t, err = strconv.ParseInt(hex.EncodeToString(buf[64:72]), 16, 64)
			if err != nil {
				log.Println(err.Error())
				return
			}
			query.Set("left", strconv.FormatInt(t, 10))

			// Uploaded
			t, err = strconv.ParseInt(hex.EncodeToString(buf[72:80]), 16, 64)
			if err != nil {
				log.Println(err.Error())
				return
			}
			query.Set("uploaded", strconv.FormatInt(t, 10))

			// Event
			t, err = strconv.ParseInt(hex.EncodeToString(buf[80:84]), 16, 32)
			if err != nil {
				log.Println(err.Error())
				return
			}
			event := strconv.FormatInt(t, 10)

			// Convert event to actual string
			switch event {
			case "0":
				query.Set("event", "")
			case "1":
				query.Set("event", "completed")
			case "2":
				query.Set("event", "started")
			case "3":
				query.Set("event", "stopped")
			}

			// IP address
			t, err = strconv.ParseInt(hex.EncodeToString(buf[84:88]), 16, 32)
			if err != nil {
				log.Println(err.Error())
				return
			}
			query.Set("ip", strconv.FormatInt(t, 10))

			// If no IP address set, use the UDP source
			if query.Get("ip") == "0" {
				query.Set("ip", strings.Split(addr.String(), ":")[0])
			}

			// Key
			query.Set("key", hex.EncodeToString(buf[88:92]))

			// Numwant
			query.Set("numwant", hex.EncodeToString(buf[92:96]))

			// If numwant is hex max value, default to 50
			if query.Get("numwant") == "ffffffff" {
				query.Set("numwant", "50")
			}

			// Port
			t, err = strconv.ParseInt(hex.EncodeToString(buf[96:98]), 16, 32)
			if err != nil {
				log.Println(err.Error())
				return
			}
			query.Set("port", strconv.FormatInt(t, 10))

			// Trigger an anonymous announce
			_, err = l.WriteToUDP(trackerAnnounce(userRecord{}, query, transID), addr)
			if err != nil {
				log.Println(err.Error())
				return
			}
		default:
			log.Println("Invalid action")
			continue
		}
	}
}
<commit_msg>Fix UDP router, properly handle UDP tracker connections between sessions<commit_after>package goat

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"errors"
	"log"
	"net"
	"net/url"
	"strconv"
	"strings"
	"sync/atomic"
)

// Handshake for UDP tracker protocol
const udpInitID = 4497486125440

// UDP errors
var (
	// udpActionError is returned when a client requests an invalid tracker action
	udpActionError = errors.New("udp: client did not send a valid UDP tracker action")
	// udpHandshakeError is returned when a client does not send the proper handshake ID
	udpHandshakeError = errors.New("udp: client did not send proper UDP tracker handshake")
	// udpIntegerError is returned when a client sends an invalid integer parameter
	udpIntegerError = errors.New("udp: client sent an invalid integer parameter")
	// udpWriteError is returned when the tracker cannot generate a proper response
	udpWriteError = errors.New("udp: tracker cannot generate UDP tracker response")
)

// UDP address to connection ID map
var udpAddrToID map[string]uint64 = map[string]uint64{}

// Handle incoming UDP connections and return response
func handleUDP(l *net.UDPConn, sendChan chan bool, recvChan chan bool) {
	// Create shutdown function
	go func(l *net.UDPConn, sendChan chan bool, recvChan chan bool) {
		// Wait for done signal
		<-sendChan

		// Close listener
		if err := l.Close(); err != nil {
			log.Println(err.Error())
		}

		log.Println("UDP listener stopped")
		recvChan <- true
	}(l, sendChan, recvChan)

	// Loop and read connections
	for {
		buf := make([]byte, 2048)
		rlen, addr, err := l.ReadFromUDP(buf)

		// Count incoming connections
		atomic.AddInt64(&static.UDP.Current, 1)
		atomic.AddInt64(&static.UDP.Total, 1)

		// Triggered on graceful shutdown
		if err != nil {
			log.Println(err.Error())
			return
		}

		// Verify length is at least 16 bytes
		if rlen < 16 {
			log.Println("Invalid length")
			continue
		}

		// Spawn a goroutine to handle the connection and send back the response
		go func(l *net.UDPConn, buf []byte, addr *net.UDPAddr) {
			// Capture initial response from buffer
			res, err := parseUDP(buf, addr)
			if err != nil {
				// Client sent a malformed UDP handshake
				log.Println(err.Error())

				// If error, client did not handshake correctly, so boot them with error message
				_, err2 := l.WriteToUDP(res, addr)
				if err2 != nil {
					log.Println(err2.Error())
				}

				return
			}

			// Write response
			_, err = l.WriteToUDP(res, addr)
			if err != nil {
				log.Println(err.Error())
			}

			return
		}(l, buf, addr)
	}
}

// Parse a UDP byte buffer, return response from tracker
func parseUDP(buf []byte, addr *net.UDPAddr) ([]byte, error) {
	// Current connection ID (initially handshake, then generated by tracker)
	connID := binary.BigEndian.Uint64(buf[0:8])
	// Action integer (connect: 0, announce: 1)
	action := binary.BigEndian.Uint32(buf[8:12])
	// Transaction ID, to match between requests
	transID := buf[12:16]

	// Action switch
	switch action {
	// Connect
	case 0:
		// Validate UDP tracker handshake
		if connID != udpInitID {
			return udpTrackerError("Invalid UDP tracker handshake", transID), udpHandshakeError
		}

		res := bytes.NewBuffer(make([]byte, 0))

		// Action
		err := binary.Write(res, binary.BigEndian, uint32(0))
		if err != nil {
			log.Println(err.Error())
			return udpTrackerError("Could not generate UDP tracker response", transID), udpWriteError
		}

		// Transaction ID
		err = binary.Write(res, binary.BigEndian, transID)
		if err != nil {
			log.Println(err.Error())
			return udpTrackerError("Could not generate UDP tracker response", transID), udpWriteError
		}

		// Generate a connection ID, which will be expected for this client next call
		expID := uint64(randRange(1, 1000000000))

		// Store this client's address and ID in map
		udpAddrToID[addr.String()] = expID

		// Connection ID, generated for this session
		err = binary.Write(res, binary.BigEndian, expID)
		if err != nil {
			log.Println(err.Error())
			return udpTrackerError("Could not generate UDP tracker response", transID), udpWriteError
		}

		return res.Bytes(), nil
	// Announce
	case 1:
		// Validate expected connection ID using map
		if connID != udpAddrToID[addr.String()] {
			return udpTrackerError("Invalid UDP connection ID", transID), udpHandshakeError
		}

		// Clear this IP from the connection map
		delete(udpAddrToID, addr.String())

		// Generate connection query
		query := url.Values{}

		// Mark client as UDP
		query.Set("udp", "1")

		// Transaction ID
		transID := buf[12:16]

		// Info hash
		query.Set("info_hash", string(buf[16:36]))

		// Skipped: peer_id: buf[36:56]

		// Downloaded
		t, err := strconv.ParseInt(hex.EncodeToString(buf[56:64]), 16, 64)
		if err != nil {
			log.Println(err.Error())
			return udpTrackerError("Invalid integer parameter: downloaded", transID), udpIntegerError
		}
		query.Set("downloaded", strconv.FormatInt(t, 10))

		// Left
		t, err = strconv.ParseInt(hex.EncodeToString(buf[64:72]), 16, 64)
		if err != nil {
			log.Println(err.Error())
			return udpTrackerError("Invalid integer parameter: left", transID), udpIntegerError
		}
		query.Set("left", strconv.FormatInt(t, 10))

		// Uploaded
		t, err = strconv.ParseInt(hex.EncodeToString(buf[72:80]), 16, 64)
		if err != nil {
			log.Println(err.Error())
			return udpTrackerError("Invalid integer parameter: uploaded", transID), udpIntegerError
		}
		query.Set("uploaded", strconv.FormatInt(t, 10))

		// Event
		t, err = strconv.ParseInt(hex.EncodeToString(buf[80:84]), 16, 32)
		if err != nil {
			log.Println(err.Error())
			return udpTrackerError("Invalid integer parameter: event", transID), udpIntegerError
		}
		event := strconv.FormatInt(t, 10)

		// Convert event to actual string
		switch event {
		case "0":
			query.Set("event", "")
		case "1":
			query.Set("event", "completed")
		case "2":
			query.Set("event", "started")
		case "3":
			query.Set("event", "stopped")
		}

		// IP address
		t, err = strconv.ParseInt(hex.EncodeToString(buf[84:88]), 16, 32)
		if err != nil {
			log.Println(err.Error())
			return udpTrackerError("Invalid integer parameter: ip", transID), udpIntegerError
		}
		query.Set("ip", strconv.FormatInt(t, 10))

		// If no IP address set, use the UDP source
		if query.Get("ip") == "0" {
			query.Set("ip", strings.Split(addr.String(), ":")[0])
		}

		// Key
		query.Set("key", hex.EncodeToString(buf[88:92]))

		// Numwant
		query.Set("numwant", hex.EncodeToString(buf[92:96]))

		// If numwant is hex max value, default to 50
		if query.Get("numwant") == "ffffffff" {
			query.Set("numwant", "50")
		}

		// Port
		t, err = strconv.ParseInt(hex.EncodeToString(buf[96:98]), 16, 32)
		if err != nil {
			log.Println(err.Error())
			return udpTrackerError("Invalid integer parameter: port", transID), udpIntegerError
		}
		query.Set("port", strconv.FormatInt(t, 10))

		// Trigger an anonymous announce
		return trackerAnnounce(userRecord{}, query, transID), nil
	default:
		return udpTrackerError("Invalid action", transID), udpActionError
	}
}
<|endoftext|>


<commit_before>package pomodoro

import (
	"fmt"
	"time"
)

//Pomodoro is the struct used to call methods on the timer
type Pomodoro struct{}

//StatusChan will allow the caller to check the current time of the Pomodoro and check if it's finished
type StatusChan struct {
	doneChan      chan bool
	currentSecond chan time.Duration
}

const (
	pomodoroTime   time.Duration = 1 * time.Minute
	shortBreakTime time.Duration = 1 * time.Minute
	longBreakTime  time.Duration = 1 * time.Minute
)

var pomodoroCount int

//SetTimer will start the pomodoro timer
func (p *Pomodoro) SetTimer() {
	fmt.Println("Pomodoro Started")

	printStatus()

	pomodoroCount++

	fmt.Println()
	fmt.Println("Pomodoro Finished")
}

func runLoop() StatusChan {
	statusChan := StatusChan{}

	tickerChannel := time.NewTicker(time.Second).C
	doneChan := make(chan bool)

	statusChan.doneChan = doneChan
	//statusChan.currentSecond <- time.Minute * 25

	go func() {
		time.Sleep(time.Second * 5)
		doneChan <- true
	}()

	go func() {
		timeCount := time.Minute * 25

		for {
			select {
			case <-tickerChannel:
				//fmt.Printf("\r%s", timeCount)
				timeCount = timeCount - time.Second
				statusChan.currentSecond <- timeCount
			case <-doneChan:
				return
			}
		}
	}()

	return statusChan
}

func printStatus() {
	statusChan := runLoop()

	for {
		select {
		case <-statusChan.doneChan:
			return
		case currentSecond := <-statusChan.currentSecond:
			fmt.Printf("\r%s", currentSecond)
		}
	}
}

//SetBreak will start the break timer
func (p *Pomodoro) SetBreak() {
	fmt.Println("Break Started")

	var breakTime time.Duration

	if pomodoroCount < 4 {
		breakTime = shortBreakTime
	} else {
		breakTime = longBreakTime
		pomodoroCount = 0
	}

	timer := time.NewTimer(breakTime)

	<-timer.C

	fmt.Println("Break Ended")
}
<commit_msg>setup return channel<commit_after>package pomodoro

import (
	"fmt"
	"time"
)

//Pomodoro is the struct used to call methods on the timer
type Pomodoro struct{}

const (
	pomodoroTime   time.Duration = 1 * time.Minute
	shortBreakTime time.Duration = 1 * time.Minute
	longBreakTime  time.Duration = 1 * time.Minute
)

var pomodoroCount int

//SetTimer will start the pomodoro timer
func (p *Pomodoro) SetTimer() {
	fmt.Println("Pomodoro Started")

	printStatus()

	pomodoroCount++

	fmt.Println()
	fmt.Println("Pomodoro Finished")
}

func runLoop() chan time.Duration {
	statusChan := make(chan time.Duration)

	tickerChannel := time.NewTicker(time.Second).C
	doneChan := make(chan bool)

	go func() {
		time.Sleep(time.Second * 5)
		doneChan <- true
	}()

	go func() {
		timeCount := time.Minute * 25
		statusChan <- timeCount

		for {
			select {
			case <-tickerChannel:
				//fmt.Printf("\r%s", timeCount)
				timeCount = timeCount - time.Second
				statusChan <- timeCount
			case <-doneChan:
				return
			}
		}
	}()

	return statusChan
}

func printStatus() {
	statusChan := runLoop()

	for {
		select {
		case currentSecond := <-statusChan:
			fmt.Printf("\r%s", currentSecond)
		default:
			return
		}
	}
}

//SetBreak will start the break timer
func (p *Pomodoro) SetBreak() {
	fmt.Println("Break Started")

	var breakTime time.Duration

	if pomodoroCount < 4 {
		breakTime = shortBreakTime
	} else {
		breakTime = longBreakTime
		pomodoroCount = 0
	}

	timer := time.NewTimer(breakTime)

	<-timer.C

	fmt.Println("Break Ended")
}
<|endoftext|>


<commit_before>// Package postmark provides a convenient wrapper for the Postmark API
package postmark

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"net/mail"
	"net/url"
	"strings"
)

const (
	DefaultHost = "api.postmarkapp.com"
)

type Message struct {
	From        *mail.Address
	To          []*mail.Address
	Cc          []*mail.Address
	Bcc         []*mail.Address
	Subject     string
	Tag         string
	HtmlBody    io.Reader
	TextBody    io.Reader
	ReplyTo     *mail.Address
	Headers     mail.Header
	Attachments []Attachment
}

func (m *Message) MarshalJSON() ([]byte, error) {
	doc := &struct {
		From        string
		To          string
		Cc          string
		Bcc         string
		Subject     string
		Tag         string
		HtmlBody    string
		TextBody    string
		ReplyTo     string
		Headers     []map[string]string
		Attachments []Attachment `json:"omitempty"`
	}{}

	doc.From = m.From.String()
	to := []string{}
	for _, addr := range m.To {
		to = append(to, addr.String())
	}
	doc.To = strings.Join(to, ", ")
	cc := []string{}
	for _, addr := range m.Cc {
		cc = append(cc, addr.String())
	}
	doc.Cc = strings.Join(cc, ", ")
	bcc := []string{}
	for _, addr := range m.Bcc {
		bcc = append(bcc, addr.String())
	}
	doc.Bcc = strings.Join(bcc, ", ")
	doc.Subject = m.Subject
	doc.Tag = m.Tag
	if m.HtmlBody != nil {
		htmlBody, err := ioutil.ReadAll(m.HtmlBody)
		if err != nil {
			return nil, err
		}
		doc.HtmlBody = string(htmlBody)
	}
	if m.TextBody != nil {
		textBody, err := ioutil.ReadAll(m.TextBody)
		if err != nil {
			return nil, err
		}
		doc.TextBody = string(textBody)
	}
	if m.ReplyTo != nil {
		doc.ReplyTo = m.ReplyTo.String()
	}
	headers := []map[string]string{}
	for k, vs := range m.Headers {
		for _, v := range vs {
			headers = append(headers, map[string]string{
				"Name":  k,
				"Value": v,
			})
		}
	}
	doc.Headers = headers
	doc.Attachments = m.Attachments

	return json.Marshal(doc)
}

type Attachment struct {
	Name        string
	Content     io.Reader
	ContentType string
}

func (a *Attachment) MarshalJSON() ([]byte, error) {
	doc := &struct {
		Name        string
		Content     string
		ContentType string
	}{}

	doc.Name = a.Name
	content, err := ioutil.ReadAll(a.Content)
	if err != nil {
		return nil, err
	}
	doc.Content = base64.StdEncoding.EncodeToString(content)
	doc.ContentType = a.ContentType

	return json.Marshal(doc)
}

type Client struct {
	ApiKey string
	Secure bool

	Host string // Host for the API endpoints, DefaultHost if ""
}

func (c *Client) endpoint(path string) *url.URL {
	url := &url.URL{}
	if c.Secure {
		url.Scheme = "https"
	} else {
		url.Scheme = "http"
	}

	if c.Host == "" {
		url.Host = DefaultHost
	} else {
		url.Host = c.Host
	}

	url.Path = path

	return url
}

// Send sends a single message
func (c *Client) Send(msg *Message) (*Result, error) {
	buf := bytes.Buffer{}
	err := json.NewEncoder(&buf).Encode(msg)
	if err != nil {
		return nil, err
	}

	req, err := http.NewRequest("POST", c.endpoint("email").String(), &buf)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Accept", "application/json")
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("X-Postmark-Server-Token", c.ApiKey)

	resp, err := (&http.Client{}).Do(req)
	if err != nil {
		return nil, err
	}

	res := &Result{}
	json.NewDecoder(resp.Body).Decode(res)
	return res, nil
}

// SendBatch sends multiple messages using the batch API
func (c *Client) SendBatch(msg []*Message) ([]*Result, error) {
	buf := bytes.Buffer{}
	err := json.NewEncoder(&buf).Encode(msg)
	if err != nil {
		return nil, err
	}

	req, err := http.NewRequest("POST", c.endpoint("email/batch").String(), &buf)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Accept", "application/json")
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("X-Postmark-Server-Token", c.ApiKey)

	resp, err := (&http.Client{}).Do(req)
	if err != nil {
		return nil, err
	}

	res := []*Result{}
	json.NewDecoder(resp.Body).Decode(res)
	return res, nil
}

type Result struct {
	ErrorCode   int
	Message     string
	MessageID   string
	SubmittedAt string
	To          string
}
<commit_msg>Implement support for templates<commit_after>// Package postmark provides a convenient wrapper for the Postmark API
package postmark

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"net/mail"
	"net/url"
	"strings"
)

const (
	DefaultHost = "api.postmarkapp.com"
)

type Message struct {
	From          *mail.Address
	To            []*mail.Address
	Cc            []*mail.Address
	Bcc           []*mail.Address
	Subject       string
	Tag           string
	HtmlBody      io.Reader
	TextBody      io.Reader
	TemplateId    int
	TemplateModel map[string]interface{}
	ReplyTo       *mail.Address
	Headers       mail.Header
	Attachments   []Attachment
}

func (m *Message) MarshalJSON() ([]byte, error) {
	doc := &struct {
		From          string
		To            string
		Cc            string
		Bcc           string
		Subject       string `json:",omitempty"`
		Tag           string
		HtmlBody      string `json:",omitempty"`
		TextBody      string `json:",omitempty"`
		TemplateId    int
		TemplateModel map[string]interface{}
		ReplyTo       string
		Headers       []map[string]string
		Attachments   []Attachment `json:"omitempty"`
	}{}

	doc.From = m.From.String()
	to := []string{}
	for _, addr := range m.To {
		to = append(to, addr.String())
	}
	doc.To = strings.Join(to, ", ")
	cc := []string{}
	for _, addr := range m.Cc {
		cc = append(cc, addr.String())
	}
	doc.Cc = strings.Join(cc, ", ")
	bcc := []string{}
	for _, addr := range m.Bcc {
		bcc = append(bcc, addr.String())
	}
	doc.Bcc = strings.Join(bcc, ", ")
	doc.Subject = m.Subject
	doc.Tag = m.Tag
	if m.HtmlBody != nil {
		htmlBody, err := ioutil.ReadAll(m.HtmlBody)
		if err != nil {
			return nil, err
		}
		doc.HtmlBody = string(htmlBody)
	}
	if m.TextBody != nil {
		textBody, err := ioutil.ReadAll(m.TextBody)
		if err != nil {
			return nil, err
		}
		doc.TextBody = string(textBody)
	}
	doc.TemplateId = m.TemplateId
	doc.TemplateModel = m.TemplateModel
	if m.ReplyTo != nil {
		doc.ReplyTo = m.ReplyTo.String()
	}
	headers := []map[string]string{}
	for k, vs := range m.Headers {
		for _, v := range vs {
			headers = append(headers, map[string]string{
				"Name":  k,
				"Value": v,
			})
		}
	}
	doc.Headers = headers
	doc.Attachments = m.Attachments

	return json.Marshal(doc)
}

type Attachment struct {
	Name        string
	Content     io.Reader
	ContentType string
}

func (a *Attachment) MarshalJSON() ([]byte, error) {
	doc := &struct {
		Name        string
		Content     string
		ContentType string
	}{}

	doc.Name = a.Name
	content, err := ioutil.ReadAll(a.Content)
	if err != nil {
		return nil, err
	}
	doc.Content = base64.StdEncoding.EncodeToString(content)
	doc.ContentType = a.ContentType

	return json.Marshal(doc)
}

type Client struct {
	ApiKey string
	Secure bool

	Host string // Host for the API endpoints, DefaultHost if ""
}

func (c *Client) endpoint(path string) *url.URL {
	url := &url.URL{}
	if c.Secure {
		url.Scheme = "https"
	} else {
		url.Scheme = "http"
	}

	if c.Host == "" {
		url.Host = DefaultHost
	} else {
		url.Host = c.Host
	}

	url.Path = path

	return url
}

// Send sends a single message
func (c *Client) Send(msg *Message) (*Result, error) {
	buf := bytes.Buffer{}
	err := json.NewEncoder(&buf).Encode(msg)
	if err != nil {
		return nil, err
	}

	url := c.endpoint("email")
	if msg.TemplateId != 0 {
		url = c.endpoint("email/withTemplate")
	}

	req, err := http.NewRequest("POST", url.String(), &buf)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Accept", "application/json")
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("X-Postmark-Server-Token", c.ApiKey)

	resp, err := (&http.Client{}).Do(req)
	if err != nil {
		return nil, err
	}

	res := &Result{}
	json.NewDecoder(resp.Body).Decode(res)
	return res, nil
}

// SendBatch sends multiple messages using the batch API
func (c *Client) SendBatch(msg []*Message) ([]*Result, error) {
	buf := bytes.Buffer{}
	err := json.NewEncoder(&buf).Encode(msg)
	if err != nil {
		return nil, err
	}

	req, err := http.NewRequest("POST", c.endpoint("email/batch").String(), &buf)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Accept", "application/json")
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("X-Postmark-Server-Token", c.ApiKey)

	resp, err := (&http.Client{}).Do(req)
	if err != nil {
		return nil, err
	}

	res := []*Result{}
	json.NewDecoder(resp.Body).Decode(res)
	return res, nil
}

type Result struct {
	ErrorCode   int
	Message     string
	MessageID   string
	SubmittedAt string
	To          string
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"errors"
	"flag"
	"fmt"
	ggio "github.com/gogo/protobuf/io"
	mux "github.com/gorilla/mux"
	p2p_peer "github.com/ipfs/go-libp2p-peer"
	p2p_pstore "github.com/ipfs/go-libp2p-peerstore"
	multiaddr "github.com/jbenet/go-multiaddr"
	p2p_host "github.com/libp2p/go-libp2p/p2p/host"
	p2p_net "github.com/libp2p/go-libp2p/p2p/net"
	mc "github.com/mediachain/concat/mc"
	pb "github.com/mediachain/concat/proto"
	"log"
	"net/http"
	"os"
	"time"
)

type Node struct {
	mc.Identity
	host p2p_host.Host
	dir  p2p_pstore.PeerInfo
}

func (node *Node) pingHandler(s p2p_net.Stream) {
	defer s.Close()

	pid := s.Conn().RemotePeer()
	log.Printf("node/ping: new stream from %s", pid.Pretty())

	var ping pb.Ping
	var pong pb.Pong
	r := ggio.NewDelimitedReader(s, mc.MaxMessageSize)
	w := ggio.NewDelimitedWriter(s)

	for {
		err := r.ReadMsg(&ping)
		if err != nil {
			return
		}

		log.Printf("node/ping: ping from %s; ponging", pid.Pretty())

		err = w.WriteMsg(&pong)
		if err != nil {
			return
		}
	}
}

func (node *Node) registerPeer(addrs ...multiaddr.Multiaddr) {
	// directory failure is a fatality for now
	ctx := context.Background()

	err := node.host.Connect(ctx, node.dir)
	if err != nil {
		log.Printf("Failed to connect to directory")
		log.Fatal(err)
	}

	s, err := node.host.NewStream(ctx, node.dir.ID, "/mediachain/dir/register")
	if err != nil {
		log.Printf("Failed to open directory stream")
		log.Fatal(err)
	}
	defer s.Close()

	pinfo := p2p_pstore.PeerInfo{node.ID, addrs}
	var pbpi pb.PeerInfo
	mc.PBFromPeerInfo(&pbpi, pinfo)
	msg := pb.RegisterPeer{&pbpi}

	w := ggio.NewDelimitedWriter(s)
	for {
		log.Printf("Registering with directory")
		err = w.WriteMsg(&msg)
		if err != nil {
			log.Printf("Failed to register with directory")
			log.Fatal(err)
		}

		time.Sleep(5 * time.Minute)
	}
}

func (node *Node) httpId(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, node.Identity.Pretty())
}

func (node *Node) httpPing(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	peerId := vars["peerId"]
	pid, err := p2p_peer.IDB58Decode(peerId)
	if err != nil {
		fmt.Fprintf(w, "Error: Bad id: %s\n", err.Error())
		return
	}

	err = node.doPing(r.Context(), pid)
	if err != nil {
		fmt.Fprintf(w, "Error: %s\n", err.Error())
		return
	}

	fmt.Fprintf(w, "OK\n")
}

func (node *Node) doPing(ctx context.Context, pid p2p_peer.ID) error {
	pinfo, err := node.doLookup(ctx, pid)
	if err != nil {
		return err
	}

	err = node.host.Connect(ctx, pinfo)
	if err != nil {
		return err
	}

	s, err := node.host.NewStream(ctx, pinfo.ID, "/mediachain/node/ping")
	if err != nil {
		return err
	}
	defer s.Close()

	var ping pb.Ping
	w := ggio.NewDelimitedWriter(s)
	err = w.WriteMsg(&ping)
	if err != nil {
		return err
	}

	var pong pb.Pong
	r := ggio.NewDelimitedReader(s, mc.MaxMessageSize)
	err = r.ReadMsg(&pong)

	return err
}

var UnknownPeer = errors.New("Unknown peer")

func (node *Node) doLookup(ctx context.Context, pid p2p_peer.ID) (empty p2p_pstore.PeerInfo, err error) {
	s, err := node.host.NewStream(ctx, node.dir.ID, "/mediachain/dir/lookup")
	if err != nil {
		return empty, err
	}
	defer s.Close()

	req := pb.LookupPeerRequest{string(pid)}
	w := ggio.NewDelimitedWriter(s)
	err = w.WriteMsg(&req)
	if err != nil {
		return empty, err
	}

	var resp pb.LookupPeerResponse
	r := ggio.NewDelimitedReader(s, mc.MaxMessageSize)
	err = r.ReadMsg(&resp)
	if err != nil {
		return empty, err
	}

	if resp.Peer == nil {
		return empty, UnknownPeer
	}

	pinfo, err := mc.PBToPeerInfo(resp.Peer)
	if err != nil {
		return empty, err
	}

	return pinfo, nil
}

func main() {
	pport := flag.Int("l", 9001, "Peer listen port")
	cport := flag.Int("c", 9002, "Peer control interface port [http]")
	home := flag.String("d", "/tmp/mcnode", "Node home")
	flag.Parse()

	if len(flag.Args()) != 1 {
		fmt.Fprintf(os.Stderr, "Usage: %s [-l port] [-c port] directory\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(1)
	}

	addr, err := mc.ParseAddress(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", *pport))
	if err != nil {
		log.Fatal(err)
	}

	dir, err := mc.ParseHandle(flag.Arg(0))
	if err != nil {
		log.Fatal(err)
	}

	os.MkdirAll(*home, 0755)
	id, err := mc.NodeIdentity(*home)
	if err != nil {
		log.Fatal(err)
	}

	host, err := mc.NewHost(id, addr)
	if err != nil {
		log.Fatal(err)
	}

	node := &Node{Identity: id, host: host, dir: dir}
	host.SetStreamHandler("/mediachain/node/ping", node.pingHandler)
	go node.registerPeer(addr)

	log.Printf("I am %s/%s", addr, id.Pretty())

	haddr := fmt.Sprintf("127.0.0.1:%d", *cport)
	router := mux.NewRouter().StrictSlash(true)
	router.HandleFunc("/id", node.httpId)
	router.HandleFunc("/ping/{peerId}", node.httpPing)

	log.Printf("Serving client interface at %s", haddr)
	err = http.ListenAndServe(haddr, router)
	if err != nil {
		log.Fatal(err)
	}

	select {}
}
<commit_msg>mcnode: a little vanity for usage info<commit_after>package main

import (
	"context"
	"errors"
	"flag"
	"fmt"
	ggio "github.com/gogo/protobuf/io"
	mux "github.com/gorilla/mux"
	p2p_peer "github.com/ipfs/go-libp2p-peer"
	p2p_pstore "github.com/ipfs/go-libp2p-peerstore"
	multiaddr "github.com/jbenet/go-multiaddr"
	p2p_host "github.com/libp2p/go-libp2p/p2p/host"
	p2p_net "github.com/libp2p/go-libp2p/p2p/net"
	mc "github.com/mediachain/concat/mc"
	pb "github.com/mediachain/concat/proto"
	"log"
	"net/http"
	"os"
	"time"
)

type Node struct {
	mc.Identity
	host p2p_host.Host
	dir  p2p_pstore.PeerInfo
}

func (node *Node) pingHandler(s p2p_net.Stream) {
	defer s.Close()

	pid := s.Conn().RemotePeer()
	log.Printf("node/ping: new stream from %s", pid.Pretty())

	var ping pb.Ping
	var pong pb.Pong
	r := ggio.NewDelimitedReader(s, mc.MaxMessageSize)
	w := ggio.NewDelimitedWriter(s)

	for {
		err := r.ReadMsg(&ping)
		if err != nil {
			return
		}

		log.Printf("node/ping: ping from %s; ponging", pid.Pretty())

		err = w.WriteMsg(&pong)
		if err != nil {
			return
		}
	}
}

func (node *Node) registerPeer(addrs ...multiaddr.Multiaddr) {
	// directory failure is a fatality for now
	ctx := context.Background()

	err := node.host.Connect(ctx, node.dir)
	if err != nil {
		log.Printf("Failed to connect to directory")
		log.Fatal(err)
	}

	s, err := node.host.NewStream(ctx, node.dir.ID, "/mediachain/dir/register")
	if err != nil {
		log.Printf("Failed to open directory stream")
		log.Fatal(err)
	}
	defer s.Close()

	pinfo := p2p_pstore.PeerInfo{node.ID, addrs}
	var pbpi pb.PeerInfo
	mc.PBFromPeerInfo(&pbpi, pinfo)
	msg := pb.RegisterPeer{&pbpi}

	w := ggio.NewDelimitedWriter(s)
	for {
		log.Printf("Registering with directory")
		err = w.WriteMsg(&msg)
		if err != nil {
			log.Printf("Failed to register with directory")
			log.Fatal(err)
		}

		time.Sleep(5 * time.Minute)
	}
}

func (node *Node) httpId(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, node.Identity.Pretty())
}

func (node *Node) httpPing(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	peerId := vars["peerId"]
	pid, err := p2p_peer.IDB58Decode(peerId)
	if err != nil {
		fmt.Fprintf(w, "Error: Bad id: %s\n", err.Error())
		return
	}

	err = node.doPing(r.Context(), pid)
	if err != nil {
		fmt.Fprintf(w, "Error: %s\n", err.Error())
		return
	}

	fmt.Fprintf(w, "OK\n")
}

func (node *Node) doPing(ctx context.Context, pid p2p_peer.ID) error {
	pinfo, err := node.doLookup(ctx, pid)
	if err != nil {
		return err
	}

	err = node.host.Connect(ctx, pinfo)
	if err != nil {
		return err
	}

	s, err := node.host.NewStream(ctx, pinfo.ID, "/mediachain/node/ping")
	if err != nil {
		return err
	}
	defer s.Close()

	var ping pb.Ping
	w := ggio.NewDelimitedWriter(s)
	err = w.WriteMsg(&ping)
	if err != nil {
		return err
	}

	var pong pb.Pong
	r := ggio.NewDelimitedReader(s, mc.MaxMessageSize)
	err = r.ReadMsg(&pong)

	return err
}

var UnknownPeer = errors.New("Unknown peer")

func (node *Node) doLookup(ctx context.Context, pid p2p_peer.ID) (empty p2p_pstore.PeerInfo, err error) {
	s, err := node.host.NewStream(ctx, node.dir.ID, "/mediachain/dir/lookup")
	if err != nil {
		return empty, err
	}
	defer s.Close()

	req := pb.LookupPeerRequest{string(pid)}
	w := ggio.NewDelimitedWriter(s)
	err = w.WriteMsg(&req)
	if err != nil {
		return empty, err
	}

	var resp pb.LookupPeerResponse
	r := ggio.NewDelimitedReader(s, mc.MaxMessageSize)
	err = r.ReadMsg(&resp)
	if err != nil {
		return empty, err
	}

	if resp.Peer == nil {
		return empty, UnknownPeer
	}

	pinfo, err := mc.PBToPeerInfo(resp.Peer)
	if err != nil {
		return empty, err
	}

	return pinfo, nil
}

func main() {
	pport := flag.Int("l", 9001, "Peer listen port")
	cport := flag.Int("c", 9002, "Peer control interface port [http]")
	home := flag.String("d", "/tmp/mcnode", "Node home")
	flag.Parse()

	if len(flag.Args()) != 1 {
		fmt.Fprintf(os.Stderr, "Usage: %s [options ...] directory\nOptions:\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(1)
	}

	addr, err := mc.ParseAddress(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", *pport))
	if err != nil {
		log.Fatal(err)
	}

	dir, err := mc.ParseHandle(flag.Arg(0))
	if err != nil {
		log.Fatal(err)
	}

	os.MkdirAll(*home, 0755)
	id, err := mc.NodeIdentity(*home)
	if err != nil {
		log.Fatal(err)
	}

	host, err := mc.NewHost(id, addr)
	if err != nil {
		log.Fatal(err)
	}

	node := &Node{Identity: id, host: host, dir: dir}
	host.SetStreamHandler("/mediachain/node/ping", node.pingHandler)
	go node.registerPeer(addr)

	log.Printf("I am %s/%s", addr, id.Pretty())

	haddr := fmt.Sprintf("127.0.0.1:%d", *cport)
	router := mux.NewRouter().StrictSlash(true)
	router.HandleFunc("/id", node.httpId)
	router.HandleFunc("/ping/{peerId}", node.httpPing)

	log.Printf("Serving client interface at %s", haddr)
	err = http.ListenAndServe(haddr, router)
	if err != nil {
		log.Fatal(err)
	}

	select {}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"github.com/codegangsta/cli"
	"io"
	"io/ioutil"
	"log"
	"os"
	"regexp"
	"strconv"
	"strings"
	"text/template"
	"unicode"
)

const (
	TK_TYPE = iota
	TK_NAME
	TK_PAYLOAD
	TK_COLON
	TK_STRING
	TK_NUMBER
	TK_EOF
	TK_DESC
)

const (
	MAX_PROTO_NUM = 1000 // agent能处理的最大协议号
)

var (
	keywords = map[string]int{
		"packet_type": TK_TYPE,
		"name":        TK_NAME,
		"payload":     TK_PAYLOAD,
		"desc":        TK_DESC,
	}
)

var (
	TOKEN_EOF   = &token{typ: TK_EOF}
	TOKEN_COLON = &token{typ: TK_COLON}
)

type api_expr struct {
	PacketType int
	Name       string
	Payload    string
	Desc       string
}

type token struct {
	typ     int
	literal string
	number  int
}

func syntax_error(p *Parser) {
	log.Fatal("syntax error @line:", p.lexer.lineno)
}

type Lexer struct {
	reader *bytes.Buffer
	lineno int
}

func (lex *Lexer) init(r io.Reader) {
	bts, err := ioutil.ReadAll(r)
	if err != nil {
		log.Println(err)
	}

	// 清除注释
	re := regexp.MustCompile("(?m:^#(.*)$)")
	bts = re.ReplaceAllLiteral(bts, nil)
	lex.reader = bytes.NewBuffer(bts)
	lex.lineno = 1
}

func (lex *Lexer) read_desc() string {
	var runes []rune
	for {
		r, _, err := lex.reader.ReadRune()
		if err == io.EOF {
			break
		} else if r == '\r' {
			break
		} else if r == '\n' {
			lex.lineno++
			break
		} else {
			runes = append(runes, r)
		}
	}

	return string(runes)
}

func (lex *Lexer) eof() bool {
	for {
		r, _, err := lex.reader.ReadRune()
		if err == io.EOF {
			return true
		} else if unicode.IsSpace(r) {
			if r == '\n' {
				lex.lineno++
			}
			continue
		} else {
			lex.reader.UnreadRune()
			return false
		}
	}
}

func (lex *Lexer) next() (t *token) {
	defer func() {
		//	log.Println(t)
	}()
	var r rune
	var err error
	for {
		r, _, err = lex.reader.ReadRune()
		if err == io.EOF {
			return TOKEN_EOF
		} else if unicode.IsSpace(r) {
			if r == '\n' {
				lex.lineno++
			}
			continue
		}
		break
	}

	var runes []rune
	if unicode.IsLetter(r) {
		for {
			runes = append(runes, r)
			r, _, err = lex.reader.ReadRune()
			if err == io.EOF {
				break
			} else if unicode.IsLetter(r) || unicode.IsNumber(r) || r == '_' {
				continue
			} else {
				lex.reader.UnreadRune()
				break
			}
		}
		t := &token{}
		if tkid, ok := keywords[string(runes)]; ok {
			t.typ = tkid
		} else {
			t.typ = TK_STRING
			t.literal = string(runes)
		}
		return t
	} else if unicode.IsNumber(r) {
		for {
			runes = append(runes, r)
			r, _, err = lex.reader.ReadRune()
			if err == io.EOF {
				break
			} else if unicode.IsNumber(r) {
				continue
			} else {
				lex.reader.UnreadRune()
				break
			}
		}
		t := &token{}
		t.typ = TK_NUMBER
		n, _ := strconv.Atoi(string(runes))
		t.number = n
		return t
	} else if r == ':' {
		return TOKEN_COLON
	} else {
		log.Fatal("lex error @line:", lex.lineno)
	}
	return nil
}

//////////////////////////////////////////////////////////////
type Parser struct {
	exprs []api_expr
	lexer *Lexer
}

func (p *Parser) init(lex *Lexer) {
	p.lexer = lex
}

func (p *Parser) match(typ int) *token {
	t := p.lexer.next()
	if t.typ != typ {
		syntax_error(p)
	}
	return t
}

func (p *Parser) expr() bool {
	if p.lexer.eof() {
		return false
	}
	api := api_expr{}

	p.match(TK_TYPE)
	p.match(TK_COLON)
	t := p.match(TK_NUMBER)
	api.PacketType = t.number

	p.match(TK_NAME)
	p.match(TK_COLON)
	t = p.match(TK_STRING)
	api.Name = t.literal

	p.match(TK_PAYLOAD)
	p.match(TK_COLON)
	t = p.match(TK_STRING)
	api.Payload = t.literal

	p.match(TK_DESC)
	p.match(TK_COLON)
	api.Desc = p.lexer.read_desc()

	p.exprs = append(p.exprs, api)
	return true
}

func main() {
	app := cli.NewApp()
	app.Name = "API Protocol Generator"
	app.Usage = "See go run api.go -h"
	app.Authors = []cli.Author{{Name: "xtaci"}, {Name: "ycs"}}
	app.Version = "1.0"
	app.Flags = []cli.Flag{
		cli.StringFlag{Name: "file,f", Value: "./api.txt", Usage: "input api.txt file"},
		cli.IntFlag{Name: "min_proto,min", Value: 0, Usage: "minimum proto number"},
		cli.IntFlag{Name: "max_proto,max", Value: 1000, Usage: "maximum proto number"},
		cli.StringFlag{Name: "template,t", Value: "./templates/game/api.tmpl", Usage: "template file"},
	}
	app.Action = func(c *cli.Context) {
		// parse
		file, err := os.Open(c.String("file"))
		if err != nil {
			log.Fatal(err)
		}
		lexer := Lexer{}
		lexer.init(file)
		p := Parser{}
		p.init(&lexer)
		for p.expr() {
		}

		// exclude protos outside of [min_proto, max_proto]
		var exprs []api_expr
		for k := range p.exprs {
			if p.exprs[k].PacketType >= c.Int("min_proto") && p.exprs[k].PacketType <= c.Int("max_proto") {
				exprs = append(exprs, p.exprs[k])
			}
		}

		// use template to generate final output
		funcMap := template.FuncMap{
			"isReq": func(api api_expr) bool {
				if strings.HasSuffix(api.Name, "_req") {
					return true
				}
				return false
			},
		}
		tmpl, err := template.New("api.tmpl").Funcs(funcMap).ParseFiles(c.String("template"))
		if err != nil {
			log.Fatal(err)
		}
		err = tmpl.Execute(os.Stdout, exprs)
		if err != nil {
			log.Fatal(err)
		}
	}
	app.Run(os.Args)
}
<commit_msg>update<commit_after>package main

import (
	"bytes"
	"github.com/codegangsta/cli"
	"io"
	"io/ioutil"
	"log"
	"os"
	"regexp"
	"strconv"
	"strings"
	"text/template"
	"unicode"
)

const (
	TK_TYPE = iota
	TK_NAME
	TK_PAYLOAD
	TK_COLON
	TK_STRING
	TK_NUMBER
	TK_EOF
	TK_DESC
)

var (
	keywords = map[string]int{
		"packet_type": TK_TYPE,
		"name":        TK_NAME,
		"payload":     TK_PAYLOAD,
		"desc":        TK_DESC,
	}
)

var (
	TOKEN_EOF   = &token{typ: TK_EOF}
	TOKEN_COLON = &token{typ: TK_COLON}
)

type api_expr struct {
	PacketType int
	Name       string
	Payload    string
	Desc       string
}

type token struct {
	typ     int
	literal string
	number  int
}

func syntax_error(p *Parser) {
	log.Fatal("syntax error @line:", p.lexer.lineno)
}

type Lexer struct {
	reader *bytes.Buffer
	lineno int
}

func (lex *Lexer) init(r io.Reader) {
	bts, err := ioutil.ReadAll(r)
	if err != nil {
		log.Println(err)
	}

	// 清除注释
	re := regexp.MustCompile("(?m:^#(.*)$)")
	bts = re.ReplaceAllLiteral(bts, nil)
	lex.reader = bytes.NewBuffer(bts)
	lex.lineno = 1
}

func (lex *Lexer) read_desc() string {
	var runes []rune
	for {
		r, _, err := lex.reader.ReadRune()
		if err == io.EOF {
			break
		} else if r == '\r' {
			break
		} else if r == '\n' {
			lex.lineno++
			break
		} else {
			runes = append(runes, r)
		}
	}

	return string(runes)
}

func (lex *Lexer) eof() bool {
	for {
		r, _, err := lex.reader.ReadRune()
		if err == io.EOF {
			return true
		} else if unicode.IsSpace(r) {
			if r == '\n' {
				lex.lineno++
			}
			continue
		} else {
			lex.reader.UnreadRune()
			return false
		}
	}
}

func (lex *Lexer) next() (t *token) {
	defer func() {
		//	log.Println(t)
	}()
	var r rune
	var err error
	for {
		r, _, err = lex.reader.ReadRune()
		if err == io.EOF {
			return TOKEN_EOF
		} else if unicode.IsSpace(r) {
			if r == '\n' {
				lex.lineno++
			}
			continue
		}
		break
	}

	var runes []rune
	if unicode.IsLetter(r) {
		for {
			runes = append(runes, r)
			r, _, err = lex.reader.ReadRune()
			if err == io.EOF {
				break
			} else if unicode.IsLetter(r) || unicode.IsNumber(r) || r == '_' {
				continue
			} else {
				lex.reader.UnreadRune()
				break
			}
		}
		t := &token{}
		if tkid, ok := keywords[string(runes)]; ok {
			t.typ = tkid
		} else {
			t.typ = TK_STRING
			t.literal = string(runes)
		}
		return t
	} else if unicode.IsNumber(r) {
		for {
			runes = append(runes, r)
			r, _, err = lex.reader.ReadRune()
			if err == io.EOF {
				break
			} else if unicode.IsNumber(r) {
				continue
			} else {
				lex.reader.UnreadRune()
				break
			}
		}
		t := &token{}
		t.typ = TK_NUMBER
		n, _ := strconv.Atoi(string(runes))
		t.number = n
		return t
	} else if r == ':' {
		return TOKEN_COLON
	} else {
		log.Fatal("lex error @line:", lex.lineno)
	}
	return nil
}

//////////////////////////////////////////////////////////////
type Parser struct {
	exprs []api_expr
	lexer *Lexer
}

func (p *Parser) init(lex *Lexer) {
	p.lexer = lex
}

func (p *Parser) match(typ int) *token {
	t := p.lexer.next()
	if t.typ != typ {
		syntax_error(p)
	}
	return t
}

func (p *Parser) expr() bool {
	if p.lexer.eof() {
		return false
	}
	api := api_expr{}

	p.match(TK_TYPE)
	p.match(TK_COLON)
	t := p.match(TK_NUMBER)
	api.PacketType = t.number

	p.match(TK_NAME)
	p.match(TK_COLON)
	t = p.match(TK_STRING)
	api.Name = t.literal

	p.match(TK_PAYLOAD)
	p.match(TK_COLON)
	t = p.match(TK_STRING)
	api.Payload = t.literal

	p.match(TK_DESC)
	p.match(TK_COLON)
	api.Desc = p.lexer.read_desc()

	p.exprs = append(p.exprs, api)
	return true
}

func main() {
	app := cli.NewApp()
	app.Name = "API Protocol Generator"
	app.Usage = "See go run api.go -h"
	app.Authors = []cli.Author{{Name: "xtaci"}, {Name: "ycs"}}
	app.Version = "1.0"
	app.Flags = []cli.Flag{
		cli.StringFlag{Name: "file,f", Value: "./api.txt", Usage: "input api.txt file"},
		cli.IntFlag{Name: "min_proto,min", Value: 0, Usage: "minimum proto number"},
		cli.IntFlag{Name: "max_proto,max", Value: 1000, Usage: "maximum proto number"},
		cli.StringFlag{Name: "template,t", Value: "./templates/game/api.tmpl", Usage: "template file"},
	}
	app.Action = func(c *cli.Context) {
		// parse
		file, err := os.Open(c.String("file"))
		if err != nil {
			log.Fatal(err)
		}
		lexer := Lexer{}
		lexer.init(file)
		p := Parser{}
		p.init(&lexer)
		for p.expr() {
		}

		// exclude protos outside of [min_proto, max_proto]
		var exprs []api_expr
		for k := range p.exprs {
			if p.exprs[k].PacketType >= c.Int("min_proto") && p.exprs[k].PacketType <= c.Int("max_proto") {
				exprs = append(exprs, p.exprs[k])
			}
		}

		// use template to generate final output
		funcMap := template.FuncMap{
			"isReq": func(api api_expr) bool {
				if strings.HasSuffix(api.Name, "_req") {
					return true
				}
				return false
			},
		}
		tmpl, err := template.New("api.tmpl").Funcs(funcMap).ParseFiles(c.String("template"))
		if err != nil {
			log.Fatal(err)
		}
		err = tmpl.Execute(os.Stdout, exprs)
		if err != nil {
			log.Fatal(err)
		}
	}
	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package main

import (
	"container/heap"
	"fmt"
	"github.com/jessevdk/go-flags"
	"github.com/tj/go-dropbox"
	"golang.org/x/text/unicode/norm"
	"math"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"
)

/* TODO

- Performance improvements:
	- Test if buffered channels improve performance in the parallel local file processing
	- Profile to find other bottlenecks?
	- Could printing progress for each local file result slow things down?
- Clean up output formatting
- Ignore more file names in skipLocalFile - see https://www.dropbox.com/help/syncing-uploads/files-not-syncing
- Do a real retry + backoff for Dropbox API errors (do we have access to the Retry-After header?)
*/

// File stores the result of either Dropbox API or local file listing
type File struct {
	Path        string
	ContentHash string
}

// FileError records a local file that could not be read due to an error
type FileError struct {
	Path  string
	Error error
}

// FileHeap is a list of Files sorted by path
type FileHeap []*File

func (h FileHeap) Len() int           { return len(h) }
func (h FileHeap) Less(i, j int) bool { return h[i].Path < h[j].Path }
func (h FileHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }

// Push a File onto the heap
func (h *FileHeap) Push(x interface{}) {
	// Push and Pop use pointer receivers because they modify the slice's length,
	// not just its contents.
	*h = append(*h, x.(*File))
}

// Pop a File off the heap
func (h *FileHeap) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[0 : n-1]
	return x
}

// PopOrNil pops a File off the heap or returns nil if there's nothing left
func (h *FileHeap) PopOrNil() *File {
	if h.Len() > 0 {
		return heap.Pop(h).(*File)
	}
	return nil
}

// ManifestComparison records the relative paths that differ between remote and
// local versions of a directory
type ManifestComparison struct {
	OnlyRemote      []string
	OnlyLocal       []string
	ContentMismatch []string
	Errored         []*FileError
	Matches         int
	Misses          int
}

type progressType int

const (
	remoteProgress progressType = iota
	localProgress
	errorProgress
)

type scanProgressUpdate struct {
	Type  progressType
	Count int
}

func main() {
	token := os.Getenv("DROPBOX_ACCESS_TOKEN")
	if token == "" {
		fmt.Fprintln(os.Stderr, "Missing Dropbox OAuth token! Please set the DROPBOX_ACCESS_TOKEN environment variable.")
		os.Exit(1)
	}

	var opts struct {
		Verbose          bool   `short:"v" long:"verbose" description:"Show verbose debug information"`
		RemoteRoot       string `short:"r" long:"remote" description:"Directory in Dropbox to verify" default:"/"`
		LocalRoot        string `short:"l" long:"local" description:"Local directory to compare to Dropbox contents" default:"."`
		CheckContentHash bool   `long:"check" description:"Check content hash of local files"`
		WorkerCount      int    `short:"w" long:"workers" description:"Number of worker threads to use (defaults to 8)" default:"8"`
	}

	_, err := flags.Parse(&opts)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}

	// Dropbox API uses empty string for root, but for figuring out relative
	// paths of the returned entries it's easier to use "/". Conversion is
	// handled before the API call.
	if opts.RemoteRoot == "" {
		opts.RemoteRoot = "/"
	}
	if opts.RemoteRoot[0] != '/' {
		opts.RemoteRoot = "/" + opts.RemoteRoot
	}

	localRoot, _ := filepath.Abs(opts.LocalRoot)

	dbxClient := dropbox.New(dropbox.NewConfig(token))

	fmt.Printf("Comparing Dropbox directory \"%v\" to local directory \"%v\"\n", opts.RemoteRoot, localRoot)
	if opts.CheckContentHash {
		fmt.Println("Checking content hashes.")
	}
	fmt.Println("")

	progressChan := make(chan *scanProgressUpdate)
	var wg sync.WaitGroup
	wg.Add(2)

	var dropboxManifest *FileHeap
	var dropboxErr error
	go func() {
		dropboxManifest, dropboxErr = getDropboxManifest(progressChan, dbxClient, opts.RemoteRoot)
		wg.Done()
	}()

	var localManifest *FileHeap
	var errored []*FileError
	var localErr error
	go func() {
		localManifest, errored, localErr = getLocalManifest(progressChan, localRoot, opts.CheckContentHash, opts.WorkerCount)
		wg.Done()
	}()

	go func() {
		remoteCount := 0
		localCount := 0
		errorCount := 0
		for update := range progressChan {
			switch update.Type {
			case remoteProgress:
				remoteCount = update.Count
			case localProgress:
				localCount = update.Count
			case errorProgress:
				errorCount = update.Count
			}

			fmt.Fprintf(os.Stderr, "Scanning: %d (remote) %d (local) %d (errored)\r", remoteCount, localCount, errorCount)
		}
		fmt.Fprintf(os.Stderr, "\n")
	}()

	// wait until remote and local scans are complete, then close progress reporting channel
	wg.Wait()
	close(progressChan)
	fmt.Printf("\nGenerated manifests for %d remote files, %d local files, with %d local errors\n\n", dropboxManifest.Len(), localManifest.Len(), len(errored))

	// check for fatal errors
	if dropboxErr != nil {
		panic(dropboxErr)
	}
	if localErr != nil {
		panic(localErr)
	}

	manifestComparison := compareManifests(dropboxManifest, localManifest, errored)

	fmt.Println("")

	printFileList(manifestComparison.OnlyRemote, "Files only in remote")
	printFileList(manifestComparison.OnlyLocal, "Files only in local")
	printFileList(manifestComparison.ContentMismatch, "Files whose contents don't match")

	fmt.Printf("Errored: %d\n\n", len(manifestComparison.Errored))
	if len(manifestComparison.Errored) > 0 {
		for _, rec := range manifestComparison.Errored {
			fmt.Printf("%s: %s\n", rec.Path, rec.Error)
		}
		if len(manifestComparison.Errored) > 0 {
			fmt.Print("\n\n")
		}
	}

	total := manifestComparison.Matches + manifestComparison.Misses
	fmt.Println("SUMMARY:")
	fmt.Printf("Files matched: %d/%d\n", manifestComparison.Matches, total)
	fmt.Printf("Files not matched: %d/%d\n", manifestComparison.Misses, total)
}

func getDropboxManifest(progressChan chan<- *scanProgressUpdate, dbxClient *dropbox.Client, rootPath string) (manifest *FileHeap, err error) {
	manifest = &FileHeap{}
	heap.Init(manifest)
	cursor := ""
	keepGoing := true

	for keepGoing {
		var resp *dropbox.ListFolderOutput
		if cursor != "" {
			arg := &dropbox.ListFolderContinueInput{Cursor: cursor}
			resp, err = dbxClient.Files.ListFolderContinue(arg)
		} else {
			apiPath := rootPath
			if apiPath == "/" {
				apiPath = ""
			}
			arg := &dropbox.ListFolderInput{
				Path:             apiPath,
				Recursive:        true,
				IncludeMediaInfo: false,
				IncludeDeleted:   false,
			}
			resp, err = dbxClient.Files.ListFolder(arg)
		}
		if err != nil {
			if strings.HasPrefix(err.Error(), "too_many_requests") {
				fmt.Fprint(os.Stderr, "Dropbox returned too many requests error, sleeping 60 seconds...\n")
				fmt.Fprintf(os.Stderr, "Error: %v\n", err)
				fmt.Fprintf(os.Stderr, "Response: %v\n", resp)
				time.Sleep(60 * time.Second)
				continue
			}
			return
		}
		for _, entry := range resp.Entries {
			if entry.Tag == "file" {

				var relPath string
				relPath, err = normalizePath(rootPath, entry.PathLower)
				if err != nil {
					return
				}
				heap.Push(manifest, &File{
					Path:        relPath,
					ContentHash: entry.ContentHash,
				})
			}
		}

		cursor = resp.Cursor
		keepGoing = resp.HasMore

		progressChan <- &scanProgressUpdate{Type: remoteProgress, Count: manifest.Len()}
	}

	return
}

func getLocalManifest(progressChan chan<- *scanProgressUpdate, localRoot string, contentHash bool, workerCount int) (manifest *FileHeap, errored []*FileError, err error) {
	localRootLowercase := strings.ToLower(localRoot)
	manifest = &FileHeap{}
	heap.Init(manifest)
	if workerCount <= 0 {
		workerCount = int(math.Max(1, float64(runtime.NumCPU())))
	}
	processChan := make(chan string)
	resultChan := make(chan *File)
	errorChan := make(chan *FileError)
	var wg sync.WaitGroup

	for i := 0; i < workerCount; i++ {
		// spin up workers
		wg.Add(1)
		go handleLocalFile(localRootLowercase, contentHash, processChan, resultChan, errorChan, &wg)
	}

	// walk in separate goroutine so that sends to errorChan don't block
	go func() {
		filepath.Walk(localRoot, func(entryPath string, info os.FileInfo, err error) error {
			if err != nil {
				errorChan <- &FileError{Path: entryPath, Error: err}
				return nil
			}

			if info.Mode().IsDir() && skipLocalDir(entryPath) {
				return filepath.SkipDir
			}

			if info.Mode().IsRegular() && !skipLocalFile(entryPath) {
				processChan <- entryPath
			}

			return nil
		})

		close(processChan)
	}()

	// Once processing goroutines are done, close result and error channels to indicate no more results streaming in
	go func() {
		wg.Wait()
		close(resultChan)
		close(errorChan)
	}()

	for {
		select {
		case result, ok := <-resultChan:
			if ok {
				heap.Push(manifest, result)
				progressChan <- &scanProgressUpdate{Type: localProgress, Count: manifest.Len()}
			} else {
				resultChan = nil
			}

		case e, ok := <-errorChan:
			if ok {
				errored = append(errored, e)
				progressChan <- &scanProgressUpdate{Type: errorProgress, Count: len(errored)}
			} else {
				errorChan = nil
			}
		}

		if resultChan == nil && errorChan == nil {
			break
		}
	}

	return
}

// fill in args etc
func handleLocalFile(localRootLowercase string, contentHash bool, processChan <-chan string, resultChan chan<- *File, errorChan chan<- *FileError, wg *sync.WaitGroup) {
	for entryPath := range processChan {

		relPath, err := normalizePath(localRootLowercase, strings.ToLower(entryPath))
		if err != nil {
			errorChan <- &FileError{Path: entryPath, Error: err}
			continue
		}

		hash := ""
		if contentHash {
			hash, err = dropbox.FileContentHash(entryPath)
			if err != nil {
				errorChan <- &FileError{Path: relPath, Error: err}
				continue
			}
		}

		resultChan <- &File{
			Path:        relPath,
			ContentHash: hash,
		}
	}
	wg.Done()
}

func normalizePath(root string, entryPath string) (string, error) {
	relPath, err := filepath.Rel(root, entryPath)
	if err != nil {
		return "", err
	}
	if relPath[0:3] == "../" {
		// try lowercase root instead
		relPath, err = filepath.Rel(strings.ToLower(root), entryPath)
		if err != nil {
			return "", err
		}
	}

	// Normalize Unicode combining characters
	relPath = norm.NFC.String(relPath)
	return relPath, nil
}

func skipLocalFile(path string) bool {
	if filepath.Base(path) == ".DS_Store" {
		return true
	}
	return false
}

func skipLocalDir(path string) bool {
	if filepath.Base(path) == "@eaDir" {
		return true
	}
	return false
}

func compareManifests(remoteManifest, localManifest *FileHeap, errored []*FileError) *ManifestComparison {
	// 1. Pop a path off both remote and local manifests.
	// 2. While remote & local are both not nil:
	//    Compare remote & local:
	//    a. If local is nil or local > remote, this file is only in remote. Record and pop remote again.
	//    b. If remote is nil or local < remote, this file is only in local. Record and pop local again.
	//    c. If local == remote, check for content mismatch. Record if necessary and pop both again.
	comparison := &ManifestComparison{Errored: errored}
	local := localManifest.PopOrNil()
	remote := remoteManifest.PopOrNil()
	for local != nil || remote != nil {
		if local == nil {
			comparison.OnlyRemote = append(comparison.OnlyRemote, remote.Path)
			comparison.Misses++
			remote = remoteManifest.PopOrNil()
		} else if remote == nil {
			comparison.OnlyLocal = append(comparison.OnlyLocal, local.Path)
			comparison.Misses++
			local = localManifest.PopOrNil()
		} else if local.Path > remote.Path {
			comparison.OnlyRemote = append(comparison.OnlyRemote, remote.Path)
			comparison.Misses++
			remote = remoteManifest.PopOrNil()
		} else if local.Path < remote.Path {
			comparison.OnlyLocal = append(comparison.OnlyLocal, local.Path)
			comparison.Misses++
			local = localManifest.PopOrNil()
		} else {
			// this must mean that remote.Path == local.Path
			if compareFileContents(remote, local) {
				comparison.Matches++
			} else {
				comparison.ContentMismatch = append(comparison.ContentMismatch, local.Path)
				comparison.Misses++
			}
			local = localManifest.PopOrNil()
			remote = remoteManifest.PopOrNil()
		}
	}
	return comparison
}

func compareFileContents(remote, local *File) bool {
	if remote.ContentHash == "" || local.ContentHash == "" {
		// Missing content hash for one of the files, possibly intentionally,
		// so can't compare. Assume that presence of both is enough to
		// validate.
		return true
	}
	return remote.ContentHash == local.ContentHash
}

func printFileList(files []string, description string) {
	fmt.Printf("%s: %d\n\n", description, len(files))
	for _, path := range files {
		fmt.Println(path)
	}
	if len(files) > 0 {
		fmt.Print("\n\n")
	}
}
<commit_msg>Todo notes<commit_after>package main

import (
	"container/heap"
	"fmt"
	"github.com/jessevdk/go-flags"
	"github.com/tj/go-dropbox"
	"golang.org/x/text/unicode/norm"
	"math"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"
)

/*
TODO
- Performance improvements:
	- Test if buffered channels improve performance in the parallel local file processing
	- Profile to find other bottlenecks?
	- Could printing progress for each local file result slow things down? (When processing lots of small files)
- Print I/O usage? i.e. how many MB/s are we processing
- Clean up output formatting
- Ignore more file names in skipLocalFile - see https://www.dropbox.com/help/syncing-uploads/files-not-syncing
- Do a real retry + backoff for Dropbox API errors (do we have access to the Retry-After header?)
*/

// File stores the result of either Dropbox API or local file listing
type File struct {
	Path        string
	ContentHash string
}

// FileError records a local file that could not be read due to an error
type FileError struct {
	Path  string
	Error error
}

// FileHeap is a list of Files sorted by path
type FileHeap []*File

func (h FileHeap) Len() int           { return len(h) }
func (h FileHeap) Less(i, j int) bool { return h[i].Path < h[j].Path }
func (h FileHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }

// Push a File onto the heap
func (h *FileHeap) Push(x interface{}) {
	// Push and Pop use pointer receivers because they modify the slice's length,
	// not just its contents.
	*h = append(*h, x.(*File))
}

// Pop a File off the heap
func (h *FileHeap) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[0 : n-1]
	return x
}

// PopOrNil pops a File off the heap or returns nil if there's nothing left
func (h *FileHeap) PopOrNil() *File {
	if h.Len() > 0 {
		return heap.Pop(h).(*File)
	}
	return nil
}

// ManifestComparison records the relative paths that differ between remote and
// local versions of a directory
type ManifestComparison struct {
	OnlyRemote      []string
	OnlyLocal       []string
	ContentMismatch []string
	Errored         []*FileError
	Matches         int
	Misses          int
}

type progressType int

const (
	remoteProgress progressType = iota
	localProgress
	errorProgress
)

type scanProgressUpdate struct {
	Type  progressType
	Count int
}

func main() {
	token := os.Getenv("DROPBOX_ACCESS_TOKEN")
	if token == "" {
		fmt.Fprintln(os.Stderr, "Missing Dropbox OAuth token! Please set the DROPBOX_ACCESS_TOKEN environment variable.")
		os.Exit(1)
	}

	var opts struct {
		Verbose          bool   `short:"v" long:"verbose" description:"Show verbose debug information"`
		RemoteRoot       string `short:"r" long:"remote" description:"Directory in Dropbox to verify" default:"/"`
		LocalRoot        string `short:"l" long:"local" description:"Local directory to compare to Dropbox contents" default:"."`
		CheckContentHash bool   `long:"check" description:"Check content hash of local files"`
		WorkerCount      int    `short:"w" long:"workers" description:"Number of worker threads to use (defaults to 8)" default:"8"`
	}

	_, err := flags.Parse(&opts)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}

	// Dropbox API uses empty string for root, but for figuring out relative
	// paths of the returned entries it's easier to use "/". Conversion is
	// handled before the API call.
	if opts.RemoteRoot == "" {
		opts.RemoteRoot = "/"
	}
	if opts.RemoteRoot[0] != '/' {
		opts.RemoteRoot = "/" + opts.RemoteRoot
	}

	localRoot, _ := filepath.Abs(opts.LocalRoot)

	dbxClient := dropbox.New(dropbox.NewConfig(token))

	fmt.Printf("Comparing Dropbox directory \"%v\" to local directory \"%v\"\n", opts.RemoteRoot, localRoot)
	if opts.CheckContentHash {
		fmt.Println("Checking content hashes.")
	}
	fmt.Println("")

	progressChan := make(chan *scanProgressUpdate)
	var wg sync.WaitGroup
	wg.Add(2)

	var dropboxManifest *FileHeap
	var dropboxErr error
	go func() {
		dropboxManifest, dropboxErr = getDropboxManifest(progressChan, dbxClient, opts.RemoteRoot)
		wg.Done()
	}()

	var localManifest *FileHeap
	var errored []*FileError
	var localErr error
	go func() {
		localManifest, errored, localErr = getLocalManifest(progressChan, localRoot, opts.CheckContentHash, opts.WorkerCount)
		wg.Done()
	}()

	go func() {
		remoteCount := 0
		localCount := 0
		errorCount := 0
		for update := range progressChan {
			switch update.Type {
			case remoteProgress:
				remoteCount = update.Count
			case localProgress:
				localCount = update.Count
			case errorProgress:
				errorCount = update.Count
			}

			fmt.Fprintf(os.Stderr, "Scanning: %d (remote) %d (local) %d (errored)\r", remoteCount, localCount, errorCount)
		}
		fmt.Fprintf(os.Stderr, "\n")
	}()

	// wait until remote and local scans are complete, then close progress reporting channel
	wg.Wait()
	close(progressChan)
	fmt.Printf("\nGenerated manifests for %d remote files, %d local files, with %d local errors\n\n", dropboxManifest.Len(), localManifest.Len(), len(errored))

	// check for fatal errors
	if dropboxErr != nil {
		panic(dropboxErr)
	}
	if localErr != nil {
		panic(localErr)
	}

	manifestComparison := compareManifests(dropboxManifest, localManifest, errored)

	fmt.Println("")

	printFileList(manifestComparison.OnlyRemote, "Files only in remote")
	printFileList(manifestComparison.OnlyLocal, "Files only in local")
	printFileList(manifestComparison.ContentMismatch, "Files whose contents don't match")

	fmt.Printf("Errored: %d\n\n", len(manifestComparison.Errored))
	if len(manifestComparison.Errored) > 0 {
		for _, rec := range manifestComparison.Errored {
			fmt.Printf("%s: %s\n", rec.Path, rec.Error)
		}
		if len(manifestComparison.Errored) > 0 {
			fmt.Print("\n\n")
		}
	}

	total := manifestComparison.Matches + manifestComparison.Misses
	fmt.Println("SUMMARY:")
	fmt.Printf("Files matched: %d/%d\n", manifestComparison.Matches, total)
	fmt.Printf("Files not matched: %d/%d\n", manifestComparison.Misses, total)
}

func getDropboxManifest(progressChan chan<- *scanProgressUpdate, dbxClient *dropbox.Client, rootPath string) (manifest *FileHeap, err error) {
	manifest = &FileHeap{}
	heap.Init(manifest)
	cursor := ""
	keepGoing := true

	for keepGoing {
		var resp *dropbox.ListFolderOutput
		if cursor != "" {
			arg := &dropbox.ListFolderContinueInput{Cursor: cursor}
			resp, err = dbxClient.Files.ListFolderContinue(arg)
		} else {
			apiPath := rootPath
			if apiPath == "/" {
				apiPath = ""
			}
			arg := &dropbox.ListFolderInput{
				Path:             apiPath,
				Recursive:        true,
				IncludeMediaInfo: false,
				IncludeDeleted:   false,
			}
			resp, err = dbxClient.Files.ListFolder(arg)
		}
		if err != nil {
			if strings.HasPrefix(err.Error(), "too_many_requests") {
				fmt.Fprint(os.Stderr, "Dropbox returned too many requests error, sleeping 60 seconds...\n")
				fmt.Fprintf(os.Stderr, "Error: %v\n", err)
				fmt.Fprintf(os.Stderr, "Response: %v\n", resp)
				time.Sleep(60 * time.Second)
				continue
			}
			return
		}
		for _, entry := range resp.Entries {
			if entry.Tag == "file" {

				var relPath string
				relPath, err = normalizePath(rootPath, entry.PathLower)
				if err != nil {
					return
				}
				heap.Push(manifest, &File{
					Path:        relPath,
					ContentHash: entry.ContentHash,
				})
			}
		}

		cursor = resp.Cursor
		keepGoing = resp.HasMore

		progressChan <- &scanProgressUpdate{Type: remoteProgress, Count: manifest.Len()}
	}

	return
}

func getLocalManifest(progressChan chan<- *scanProgressUpdate, localRoot string, contentHash bool, workerCount int) (manifest *FileHeap, errored []*FileError, err error) {
	localRootLowercase := strings.ToLower(localRoot)
	manifest = &FileHeap{}
	heap.Init(manifest)
	if workerCount <= 0 {
		workerCount = int(math.Max(1, float64(runtime.NumCPU())))
	}
	processChan := make(chan string)
	resultChan := make(chan *File)
	errorChan := make(chan *FileError)
	var wg sync.WaitGroup

	for i := 0; i < workerCount; i++ {
		// spin up workers
		wg.Add(1)
		go handleLocalFile(localRootLowercase, contentHash, processChan, resultChan, errorChan, &wg)
	}

	// walk in separate goroutine so that sends to errorChan don't block
	go func() {
		filepath.Walk(localRoot, func(entryPath string, info os.FileInfo, err error) error {
			if err != nil {
				errorChan <- &FileError{Path: entryPath, Error: err}
				return nil
			}

			if info.Mode().IsDir() && skipLocalDir(entryPath) {
				return filepath.SkipDir
			}

			if info.Mode().IsRegular() && !skipLocalFile(entryPath) {
				processChan <- entryPath
			}

			return nil
		})

		close(processChan)
	}()

	// Once processing goroutines are done, close result and error channels to indicate no more results streaming in
	go func() {
		wg.Wait()
		close(resultChan)
		close(errorChan)
	}()

	for {
		select {
		case result, ok := <-resultChan:
			if ok {
				heap.Push(manifest, result)
				progressChan <- &scanProgressUpdate{Type: localProgress, Count: manifest.Len()}
			} else {
				resultChan = nil
			}

		case e, ok := <-errorChan:
			if ok {
				errored = append(errored, e)
				progressChan <- &scanProgressUpdate{Type: errorProgress, Count: len(errored)}
			} else {
				errorChan = nil
			}
		}

		if resultChan == nil && errorChan == nil {
			break
		}
	}

	return
}

// fill in args etc
func handleLocalFile(localRootLowercase string, contentHash bool, processChan <-chan string, resultChan chan<- *File, errorChan chan<- *FileError, wg *sync.WaitGroup) {
	for entryPath := range processChan {

		relPath, err := normalizePath(localRootLowercase, strings.ToLower(entryPath))
		if err != nil {
			errorChan <- &FileError{Path: entryPath, Error: err}
			continue
		}

		hash := ""
		if contentHash {
			hash, err = dropbox.FileContentHash(entryPath)
			if err != nil {
				errorChan <- &FileError{Path: relPath, Error: err}
				continue
			}
		}

		resultChan <- &File{
			Path:        relPath,
			ContentHash: hash,
		}
	}
	wg.Done()
}

func normalizePath(root string, entryPath string) (string, error) {
	relPath, err := filepath.Rel(root, entryPath)
	if err != nil {
		return "", err
	}
	if relPath[0:3] == "../" {
		// try lowercase root instead
		relPath, err = filepath.Rel(strings.ToLower(root), entryPath)
		if err != nil {
			return "", err
		}
	}

	// Normalize Unicode combining characters
	relPath = norm.NFC.String(relPath)
	return relPath, nil
}

func skipLocalFile(path string) bool {
	if filepath.Base(path) == ".DS_Store" {
		return true
	}
	return false
}

func skipLocalDir(path string) bool {
	if filepath.Base(path) == "@eaDir" {
		return true
	}
	return false
}

func compareManifests(remoteManifest, localManifest *FileHeap, errored []*FileError) *ManifestComparison {
	// 1. Pop a path off both remote and local manifests.
	// 2. While remote & local are both not nil:
	//    Compare remote & local:
	//    a. If local is nil or local > remote, this file is only in remote. Record and pop remote again.
	//    b. If remote is nil or local < remote, this file is only in local. Record and pop local again.
	//    c. If local == remote, check for content mismatch. Record if necessary and pop both again.
	comparison := &ManifestComparison{Errored: errored}
	local := localManifest.PopOrNil()
	remote := remoteManifest.PopOrNil()
	for local != nil || remote != nil {
		if local == nil {
			comparison.OnlyRemote = append(comparison.OnlyRemote, remote.Path)
			comparison.Misses++
			remote = remoteManifest.PopOrNil()
		} else if remote == nil {
			comparison.OnlyLocal = append(comparison.OnlyLocal, local.Path)
			comparison.Misses++
			local = localManifest.PopOrNil()
		} else if local.Path > remote.Path {
			comparison.OnlyRemote = append(comparison.OnlyRemote, remote.Path)
			comparison.Misses++
			remote = remoteManifest.PopOrNil()
		} else if local.Path < remote.Path {
			comparison.OnlyLocal = append(comparison.OnlyLocal, local.Path)
			comparison.Misses++
			local = localManifest.PopOrNil()
		} else {
			// this must mean that remote.Path == local.Path
			if compareFileContents(remote, local) {
				comparison.Matches++
			} else {
				comparison.ContentMismatch = append(comparison.ContentMismatch, local.Path)
				comparison.Misses++
			}
			local = localManifest.PopOrNil()
			remote = remoteManifest.PopOrNil()
		}
	}
	return comparison
}

func compareFileContents(remote, local *File) bool {
	if remote.ContentHash == "" || local.ContentHash == "" {
		// Missing content hash for one of the files, possibly intentionally,
		// so can't compare. Assume that presence of both is enough to
		// validate.
		return true
	}
	return remote.ContentHash == local.ContentHash
}

func printFileList(files []string, description string) {
	fmt.Printf("%s: %d\n\n", description, len(files))
	for _, path := range files {
		fmt.Println(path)
	}
	if len(files) > 0 {
		fmt.Print("\n\n")
	}
}
<|endoftext|>


<commit_before>package main
<commit_msg>impl test/pwm_test<commit_after>// PWM test program
package main

import (
	"fmt"

	. "github.com/conclave/pcduino/core"
)

func init() {
	Init()
	setup()
}

func main() {
	for {
		loop()
	}
}

var pwm_id byte = 5
var freq uint = 781
var value int = MAX_PWM_LEVEL / 2

func setup() {
	step := PWMFreqSet(pwm_id, freq)
	fmt.Printf("PWM%d set freq %d and valid duty cycle range [0, %d]\n", pwm_id, freq, step)
	if step > 0 {
		fmt.Printf("PWM%d test with duty cycle %d\n", pwm_id, value)
		AnalogWrite(pwm_id, value)
	}
}

func loop() {
	DelayMicrosends(200000)
}
<|endoftext|>


<commit_before>package oauth

import (
	"net/http"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"
	"golang.org/x/net/http2"

	"k8s.io/client-go/rest"

	exutil "github.com/openshift/origin/test/extended/util"
)

var _ = g.Describe("[sig-auth][Feature:OAuthServer] OAuth server", func() {
	defer g.GinkgoRecover()

	oc := exutil.NewCLI("oauth")

	g.It("should use http1.1 only to prevent http2 connection reuse", func() {
		metadata := getOAuthWellKnownData(oc)

		tlsClientConfig, err := rest.TLSConfigFor(oc.AdminConfig())
		o.Expect(err).NotTo(o.HaveOccurred())

		rt := http2.Transport{
			TLSClientConfig: tlsClientConfig,
		}

		req, err := http.NewRequest(http.MethodHead, metadata.Issuer, nil)
		o.Expect(err).NotTo(o.HaveOccurred())

		_, err = rt.RoundTrip(req)
		o.Expect(err).NotTo(o.BeNil(), "http2 only request to OAuth server should fail")
		o.Expect(err.Error()).To(o.Equal(`http2: unexpected ALPN protocol ""; want "h2"`))
	})
})
<commit_msg>don't test oauth-server using http1.1 in proxied environment<commit_after>package oauth

import (
	"net/http"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"
	"golang.org/x/net/http2"

	"k8s.io/client-go/rest"

	exutil "github.com/openshift/origin/test/extended/util"
)

var _ = g.Describe("[sig-auth][Feature:OAuthServer] OAuth server", func() {
	defer g.GinkgoRecover()

	oc := exutil.NewCLI("oauth")

	g.It("should use http1.1 only to prevent http2 connection reuse", func() {
		metadata := getOAuthWellKnownData(oc)

		tlsClientConfig, err := rest.TLSConfigFor(oc.AdminConfig())
		o.Expect(err).NotTo(o.HaveOccurred())

		rt := http2.Transport{
			TLSClientConfig: tlsClientConfig,
		}

		req, err := http.NewRequest(http.MethodHead, metadata.Issuer, nil)
		o.Expect(err).NotTo(o.HaveOccurred())

		// there is no HTTP2 proxying implemented in golang, skip
		if url, _ := http.ProxyFromEnvironment(req); url != nil {
			g.Skip("this test does not run in proxied environment")
		}

		_, err = rt.RoundTrip(req)
		o.Expect(err).NotTo(o.BeNil(), "http2 only request to OAuth server should fail")
		o.Expect(err.Error()).To(o.Equal(`http2: unexpected ALPN protocol ""; want "h2"`))
	})
})
<|endoftext|>


<commit_before>package router

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"time"

	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/util/wait"
	e2e "k8s.io/kubernetes/test/e2e/framework"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"

	configv1 "github.com/openshift/api/config/v1"
	unidlingapi "github.com/openshift/api/unidling/v1alpha1"
	exutil "github.com/openshift/origin/test/extended/util"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

var _ = g.Describe("[sig-network-edge][Conformance][Area:Networking][Feature:Router]", func() {
	defer g.GinkgoRecover()

	var (
		configPath = exutil.FixturePath("testdata", "router", "router-idle.yaml")
		oc         = exutil.NewCLI("router-idling")
	)

	// this hook must be registered before the framework namespace teardown
	// hook
	g.AfterEach(func() {
		if g.CurrentGinkgoTestDescription().Failed {
			exutil.DumpPodLogsStartingWithInNamespace("router", "openshift-ingress", oc.AsAdmin())
		}
	})

	g.Describe("The HAProxy router", func() {
		g.It("should be able to connect to a service that is idled because a GET on the route will unidle it", func() {
			infra, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get(context.Background(), "cluster", metav1.GetOptions{})
			o.Expect(err).NotTo(o.HaveOccurred(), "failed to get cluster-wide infrastructure")
			switch infra.Status.PlatformStatus.Type {
			case configv1.OvirtPlatformType, configv1.KubevirtPlatformType, configv1.LibvirtPlatformType, configv1.VSpherePlatformType:
				// Skip on platforms where the default
				// router is not exposed by a load
				// balancer service.
				g.Skip("https://bugzilla.redhat.com/show_bug.cgi?id=1933114")
			}

			timeout := 15 * time.Minute

			g.By(fmt.Sprintf("creating test fixture from a config file %q", configPath))
			err = oc.Run("new-app").Args("-f", configPath).Execute()
			o.Expect(err).NotTo(o.HaveOccurred(), "failed to create test fixture")

			g.By("Waiting for pods to be running")
			err = waitForRunningPods(oc, 1, exutil.ParseLabelsOrDie("app=idle-test"), timeout)
			o.Expect(err).NotTo(o.HaveOccurred(), "pods not running")

			g.By("Getting a 200 status code when accessing the route")
			hostname, err := getHostnameForRoute(oc, "idle-test")
			o.Expect(err).NotTo(o.HaveOccurred())
			err = waitHTTPGetStatus(hostname, http.StatusOK, timeout)
			o.Expect(err).NotTo(o.HaveOccurred(), "expected status 200 from the GET request")

			g.By("Idling the service")
			_, err = oc.Run("idle").Args("idle-test").Output()
			o.Expect(err).NotTo(o.HaveOccurred(), "failed to idle the service")

			var annotations map[string]string

			g.By("Fetching the endpoints and checking the idle annotations are present")
			err = wait.PollImmediate(time.Second, timeout, func() (bool, error) {
				endpoints, err := oc.KubeClient().CoreV1().Endpoints(oc.Namespace()).Get(context.Background(), "idle-test", metav1.GetOptions{})
				if err != nil {
					e2e.Logf("Error getting endpoints: %v", err)
					return false, nil
				}
				annotations = endpoints.Annotations
				_, idledAt := annotations[unidlingapi.IdledAtAnnotation]
				_, unidleTarget := annotations[unidlingapi.UnidleTargetAnnotation]
				return idledAt && unidleTarget, nil
			})
			o.Expect(err).NotTo(o.HaveOccurred(), "failed to fetch the endpoints")
			mustVerifyIdleAnnotationValues(annotations)

			g.By("Fetching the service and checking the idle annotations are present")
			err = wait.PollImmediate(time.Second, timeout, func() (bool, error) {
				service, err := oc.KubeClient().CoreV1().Services(oc.Namespace()).Get(context.Background(), "idle-test", metav1.GetOptions{})
				if err != nil {
					e2e.Logf("Error getting service: %v", err)
					return false, nil
				}
				annotations = service.Annotations
				_, idledAt := annotations[unidlingapi.IdledAtAnnotation]
				_, unidleTarget := annotations[unidlingapi.UnidleTargetAnnotation]
				return idledAt && unidleTarget, nil
			})
			o.Expect(err).NotTo(o.HaveOccurred(), "failed to fetch the service")
			mustVerifyIdleAnnotationValues(annotations)

			g.By("Unidling the service by making a GET request on the route")
			err = waitHTTPGetStatus(hostname, http.StatusOK, timeout)
			o.Expect(err).NotTo(o.HaveOccurred(), "expected status 200 from the GET request")

			g.By("Validating that the idle annotations have been removed from the endpoints")
			err = wait.PollImmediate(time.Second, timeout, func() (bool, error) {
				endpoints, err := oc.KubeClient().CoreV1().Endpoints(oc.Namespace()).Get(context.Background(), "idle-test", metav1.GetOptions{})
				if err != nil {
					e2e.Logf("Error getting endpoints: %v", err)
					return false, nil
				}
				_, idledAt := endpoints.Annotations[unidlingapi.IdledAtAnnotation]
				_, unidleTarget := endpoints.Annotations[unidlingapi.UnidleTargetAnnotation]
				return !idledAt && !unidleTarget, nil
			})
			o.Expect(err).NotTo(o.HaveOccurred(), "idle annotations not removed from endpoints")

			g.By("Validating that the idle annotations have been removed from the service")
			err = wait.PollImmediate(time.Second, timeout, func() (bool, error) {
				service, err := oc.KubeClient().CoreV1().Services(oc.Namespace()).Get(context.Background(), "idle-test", metav1.GetOptions{})
				if err != nil {
					e2e.Logf("Error getting service: %v", err)
					return false, nil
				}
				_, idledAt := service.Annotations[unidlingapi.IdledAtAnnotation]
				_, unidleTarget := service.Annotations[unidlingapi.UnidleTargetAnnotation]
				return !idledAt && !unidleTarget, nil
			})
			o.Expect(err).NotTo(o.HaveOccurred(), "idle annotations not removed from service")
		})
	})
})

func mustVerifyIdleAnnotationValues(annotations map[string]string) {
	o.Expect(annotations).To(o.HaveKey(unidlingapi.IdledAtAnnotation))
	o.Expect(annotations).To(o.HaveKey(unidlingapi.UnidleTargetAnnotation))

	idledAtAnnotation := annotations[unidlingapi.IdledAtAnnotation]
	idledAtTime, err := time.Parse(time.RFC3339, idledAtAnnotation)
	o.Expect(err).ToNot(o.HaveOccurred())
	o.Expect(idledAtTime).To(o.BeTemporally("~", time.Now(), 5*time.Minute))

	g.By("Checking the idle targets")
	unidleTargetAnnotation := annotations[unidlingapi.UnidleTargetAnnotation]
	var unidleTargets []unidlingapi.RecordedScaleReference
	err = json.Unmarshal([]byte(unidleTargetAnnotation), &unidleTargets)
	o.Expect(err).ToNot(o.HaveOccurred())
	o.Expect(unidleTargets).To(o.Equal([]unidlingapi.RecordedScaleReference{
		{
			Replicas: 1,
			CrossGroupObjectReference: unidlingapi.CrossGroupObjectReference{
				Kind:  "Deployment",
				Group: "apps",
				Name:  "idle-test",
			},
		},
	}))
}

// waitForRunningPods waits for podCount pods matching podSelector are
// in the running state. It retries the request every second and will
// return an error if the conditions are not met after the specified
// timeout.
func waitForRunningPods(oc *exutil.CLI, podCount int, podLabels labels.Selector, timeout time.Duration) error {
	ns := oc.KubeFramework().Namespace.Name

	if err := wait.PollImmediate(time.Second, timeout, func() (bool, error) {
		podList, err := oc.AdminKubeClient().CoreV1().Pods(ns).List(context.Background(), metav1.ListOptions{LabelSelector: podLabels.String()})
		if err != nil {
			e2e.Logf("Error listing pods: %v", err)
			return false, nil
		}
		return len(podList.Items) == podCount, nil
	}); err != nil {
		return fmt.Errorf("failed to list pods: %v", err)
	}

	e2e.Logf("Waiting for %d pods in namespace %s", podCount, ns)
	c := oc.AdminKubeClient()
	pods, err := exutil.WaitForPods(c.CoreV1().Pods(ns), podLabels, exutil.CheckPodIsRunning, podCount, timeout)
	if err != nil {
		return fmt.Errorf("error in pod wait: %v", err)
	} else if len(pods) < podCount {
		return fmt.Errorf("only got %v out of %v pods in %s (timeout)", len(pods), podCount, timeout)
	}

	e2e.Logf("All expected pods in namespace %s are running", ns)
	return nil
}

// waitHTTPGetStatus repeatedly makes a HTTP GET request to hostname
// until the GET response equals statusCode. It retries every second
// and will return an error if the conditions are not met after the
// specified timeout.
func waitHTTPGetStatus(hostname string, statusCode int, timeout time.Duration) error {
	client := makeHTTPClient(false, timeout)

	var attempt int

	return wait.Poll(time.Second, timeout, func() (bool, error) {
		attempt += 1
		url := "http://" + hostname
		resp, err := client.Get(url)
		if err != nil {
			e2e.Logf("GET#%v %q error=%v", attempt, url, err)
			return false, nil // could be 503 if service not ready
		}
		e2e.Logf("GET#%v %q status=%v", attempt, url, resp.StatusCode)
		return resp.StatusCode == statusCode, nil
	})
}
<commit_msg>test/extended/router: Fix-up Unidling test<commit_after>package router

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"time"

	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/util/wait"
	e2e "k8s.io/kubernetes/test/e2e/framework"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"

	configv1 "github.com/openshift/api/config/v1"
	unidlingapi "github.com/openshift/api/unidling/v1alpha1"
	exutil "github.com/openshift/origin/test/extended/util"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

var _ = g.Describe("[sig-network-edge][Conformance][Area:Networking][Feature:Router]", func() {
	defer g.GinkgoRecover()

	var (
		configPath = exutil.FixturePath("testdata", "router", "router-idle.yaml")
		oc         = exutil.NewCLI("router-idling")
	)

	// this hook must be registered before the framework namespace teardown
	// hook
	g.AfterEach(func() {
		if g.CurrentGinkgoTestDescription().Failed {
			exutil.DumpPodLogsStartingWithInNamespace("router", "openshift-ingress", oc.AsAdmin())
		}
	})

	g.Describe("The HAProxy router", func() {
		g.It("should be able to connect to a service that is idled because a GET on the route will unidle it", func() {
			infra, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get(context.Background(), "cluster", metav1.GetOptions{})
			o.Expect(err).NotTo(o.HaveOccurred(), "failed to get cluster-wide infrastructure")
			switch infra.Status.PlatformStatus.Type {
			case configv1.OvirtPlatformType, configv1.KubevirtPlatformType, configv1.LibvirtPlatformType, configv1.VSpherePlatformType:
				// Skip on platforms where the default
				// router is not exposed by a load
				// balancer service.
				g.Skip("https://bugzilla.redhat.com/show_bug.cgi?id=1933114")
			}

			timeout := 15 * time.Minute

			g.By(fmt.Sprintf("creating test fixture from a config file %q", configPath))
			err = oc.Run("new-app").Args("-f", configPath).Execute()
			o.Expect(err).NotTo(o.HaveOccurred(), "failed to create test fixture")

			g.By("Waiting for pods to be running")
			err = waitForRunningPods(oc, 1, exutil.ParseLabelsOrDie("app=idle-test"), timeout)
			o.Expect(err).NotTo(o.HaveOccurred(), "pods not running")

			g.By("Getting a 200 status code when accessing the route")
			hostname, err := getHostnameForRoute(oc, "idle-test")
			o.Expect(err).NotTo(o.HaveOccurred())
			err = waitHTTPGetStatus(hostname, http.StatusOK, timeout)
			o.Expect(err).NotTo(o.HaveOccurred(), "expected status 200 from the GET request")

			g.By("Idling the service")
			_, err = oc.Run("idle").Args("idle-test").Output()
			o.Expect(err).NotTo(o.HaveOccurred(), "failed to idle the service")

			var annotations map[string]string

			g.By("Fetching the endpoints and checking the idle annotations are present")
			err = wait.PollImmediate(time.Second, timeout, func() (bool, error) {
				endpoints, err := oc.KubeClient().CoreV1().Endpoints(oc.Namespace()).Get(context.Background(), "idle-test", metav1.GetOptions{})
				if err != nil {
					e2e.Logf("Error getting endpoints: %v", err)
					return false, nil
				}
				annotations = endpoints.Annotations
				_, idledAt := annotations[unidlingapi.IdledAtAnnotation]
				_, unidleTarget := annotations[unidlingapi.UnidleTargetAnnotation]
				return idledAt && unidleTarget, nil
			})
			o.Expect(err).NotTo(o.HaveOccurred(), "failed to fetch the endpoints")
			mustVerifyIdleAnnotationValues(annotations)

			g.By("Fetching the service and checking the idle annotations are present")
			err = wait.PollImmediate(time.Second, timeout, func() (bool, error) {
				service, err := oc.KubeClient().CoreV1().Services(oc.Namespace()).Get(context.Background(), "idle-test", metav1.GetOptions{})
				if err != nil {
					e2e.Logf("Error getting service: %v", err)
					return false, nil
				}
				annotations = service.Annotations
				_, idledAt := annotations[unidlingapi.IdledAtAnnotation]
				_, unidleTarget := annotations[unidlingapi.UnidleTargetAnnotation]
				return idledAt && unidleTarget, nil
			})
			o.Expect(err).NotTo(o.HaveOccurred(), "failed to fetch the service")
			mustVerifyIdleAnnotationValues(annotations)

			// wait for target deployment to actually scale down
			err = waitForRunningPods(oc, 0, exutil.ParseLabelsOrDie("app=idle-test"), timeout)
			o.Expect(err).NotTo(o.HaveOccurred())

			g.By("Unidling the service by making a GET request on the route")
			err = waitHTTPGetStatus(hostname, http.StatusOK, timeout)
			o.Expect(err).NotTo(o.HaveOccurred(), "expected status 200 from the GET request")

			g.By("Validating that the idle annotations have been removed from the endpoints")
			err = wait.PollImmediate(time.Second, timeout, func() (bool, error) {
				endpoints, err := oc.KubeClient().CoreV1().Endpoints(oc.Namespace()).Get(context.Background(), "idle-test", metav1.GetOptions{})
				if err != nil {
					e2e.Logf("Error getting endpoints: %v", err)
					return false, nil
				}
				_, idledAt := endpoints.Annotations[unidlingapi.IdledAtAnnotation]
				_, unidleTarget := endpoints.Annotations[unidlingapi.UnidleTargetAnnotation]
				return !idledAt && !unidleTarget, nil
			})
			o.Expect(err).NotTo(o.HaveOccurred(), "idle annotations not removed from endpoints")

			g.By("Validating that the idle annotations have been removed from the service")
			err = wait.PollImmediate(time.Second, timeout, func() (bool, error) {
				service, err := oc.KubeClient().CoreV1().Services(oc.Namespace()).Get(context.Background(), "idle-test", metav1.GetOptions{})
				if err != nil {
					e2e.Logf("Error getting service: %v", err)
					return false, nil
				}
				_, idledAt := service.Annotations[unidlingapi.IdledAtAnnotation]
				_, unidleTarget := service.Annotations[unidlingapi.UnidleTargetAnnotation]
				return !idledAt && !unidleTarget, nil
			})
			o.Expect(err).NotTo(o.HaveOccurred(), "idle annotations not removed from service")
		})
	})
})

func mustVerifyIdleAnnotationValues(annotations map[string]string) {
	o.Expect(annotations).To(o.HaveKey(unidlingapi.IdledAtAnnotation))
	o.Expect(annotations).To(o.HaveKey(unidlingapi.UnidleTargetAnnotation))

	idledAtAnnotation := annotations[unidlingapi.IdledAtAnnotation]
	idledAtTime, err := time.Parse(time.RFC3339, idledAtAnnotation)
	o.Expect(err).ToNot(o.HaveOccurred())
	o.Expect(idledAtTime).To(o.BeTemporally("~", time.Now(), 5*time.Minute))

	g.By("Checking the idle targets")
	unidleTargetAnnotation := annotations[unidlingapi.UnidleTargetAnnotation]
	var unidleTargets []unidlingapi.RecordedScaleReference
	err = json.Unmarshal([]byte(unidleTargetAnnotation), &unidleTargets)
	o.Expect(err).ToNot(o.HaveOccurred())
	o.Expect(unidleTargets).To(o.Equal([]unidlingapi.RecordedScaleReference{
		{
			Replicas: 1,
			CrossGroupObjectReference: unidlingapi.CrossGroupObjectReference{
				Kind:  "Deployment",
				Group: "apps",
				Name:  "idle-test",
			},
		},
	}))
}

// waitForRunningPods waits for podCount pods matching podSelector are
// in the running state. It retries the request every second and will
// return an error if the conditions are not met after the specified
// timeout.
func waitForRunningPods(oc *exutil.CLI, podCount int, podLabels labels.Selector, timeout time.Duration) error {
	ns := oc.KubeFramework().Namespace.Name

	if err := wait.PollImmediate(time.Second, timeout, func() (bool, error) {
		podList, err := oc.AdminKubeClient().CoreV1().Pods(ns).List(context.Background(), metav1.ListOptions{LabelSelector: podLabels.String()})
		if err != nil {
			e2e.Logf("Error listing pods: %v", err)
			return false, nil
		}
		return len(podList.Items) == podCount, nil
	}); err != nil {
		return fmt.Errorf("failed to list pods: %v", err)
	}

	e2e.Logf("Waiting for %d pods in namespace %s", podCount, ns)
	c := oc.AdminKubeClient()
	pods, err := exutil.WaitForPods(c.CoreV1().Pods(ns), podLabels, exutil.CheckPodIsRunning, podCount, timeout)
	if err != nil {
		return fmt.Errorf("error in pod wait: %v", err)
	} else if len(pods) < podCount {
		return fmt.Errorf("only got %v out of %v pods in %s (timeout)", len(pods), podCount, timeout)
	}

	e2e.Logf("All expected pods in namespace %s are running", ns)
	return nil
}

// waitHTTPGetStatus repeatedly makes a HTTP GET request to hostname
// until the GET response equals statusCode. It retries every second
// and will return an error if the conditions are not met after the
// specified timeout.
func waitHTTPGetStatus(hostname string, statusCode int, timeout time.Duration) error {
	client := makeHTTPClient(false, 30*time.Second)
	var attempt int

	url := "http://" + hostname

	return wait.Poll(time.Second, timeout, func() (bool, error) {
		attempt += 1
		resp, err := client.Get(url)
		if err != nil {
			e2e.Logf("GET#%v %q error=%v", attempt, url, err)
			return false, nil // could be 503 if service not ready
		}
		defer resp.Body.Close()
		io.Copy(ioutil.Discard, resp.Body)
		e2e.Logf("GET#%v %q status=%v", attempt, url, resp.StatusCode)
		return resp.StatusCode == statusCode, nil
	})
}
<|endoftext|>


<commit_before>package worker_integration_test

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"os"
	"sort"
	"strings"
	"testing"

	"github.com/streadway/amqp"
)

type LogPart struct {
	ID      int64  `json:"id"`
	Content string `json:"log"`
	Final   bool   `json:"final"`
	UUID    string `json:"uuid"`
	Number  int64  `json:"number"`
}

type LogPartSlice []LogPart

func (lps LogPartSlice) Len() int {
	return len(lps)
}

func (lps LogPartSlice) Less(i, j int) bool {
	return lps[i].Number < lps[j].Number
}

func (lps LogPartSlice) Swap(i, j int) {
	lps[i], lps[j] = lps[j], lps[i]
}

type StateUpdate struct {
	ID    int64  `json:"id"`
	State string `json:"state"`
}

func setupConn() (*amqp.Connection, error) {
	amqpConn, err := amqp.Dial(os.Getenv("AMQP_URL"))
	if err != nil {
		return nil, err
	}

	amqpChan, err := amqpConn.Channel()
	if err != nil {
		return nil, err
	}
	defer amqpChan.Close()

	_, err = amqpChan.QueueDeclare("builds.test", true, false, false, false, nil)
	if err != nil {
		return nil, err
	}

	_, err = amqpChan.QueueDeclare("reporting.jobs.logs", true, false, false, false, nil)
	if err != nil {
		return nil, err
	}

	_, err = amqpChan.QueueDeclare("reporting.jobs.builds", true, false, false, false, nil)
	if err != nil {
		return nil, err
	}

	_, err = amqpChan.QueuePurge("builds.test", false)
	if err != nil {
		return nil, err
	}

	_, err = amqpChan.QueuePurge("reporting.jobs.logs", false)
	if err != nil {
		return nil, err
	}

	_, err = amqpChan.QueuePurge("reporting.jobs.builds", false)
	if err != nil {
		return nil, err
	}

	return amqpConn, nil
}

func publishJob(amqpConn *amqp.Connection) error {
	amqpChan, err := amqpConn.Channel()
	if err != nil {
		return err
	}
	defer amqpChan.Close()

	return amqpChan.Publish("", "builds.test", false, false, amqp.Publishing{
		Body: []byte(`{
			"type": "test",
			"job": {
				"id": 3,
				"number": "1.1",
				"commit": "abcdef",
				"commit_range": "abcde...abcdef",
				"commit_message": "Hello world",
				"branch": "master",
				"ref": null,
				"state": "queued",
				"secure_env_enabled": true,
				"pull_request": false
			},
			"source": {
				"id": 2,
				"number": "1"
			},
			"repository": {
				"id": 1,
				"slug": "hello/world",
				"github_id": 1234,
				"source_url": "git://github.com/hello/world.git",
				"api_url": "https://api.github.com",
				"last_build_id": 2,
				"last_build_number": "1",
				"last_build_started_at": null,
				"last_build_finished_at": null,
				"last_build_duration": null,
				"last_build_state": "created",
				"description": "Hello world",
				"config": {},
				"queue": "builds.test",
				"uuid": "fake-uuid",
				"ssh_key": null,
				"env_vars": [],
				"timeouts": {
					"hard_limit": null,
					"log_silence": null
				}
			}`),
		DeliveryMode: amqp.Persistent,
	})
}

func TestIntegrationLogMessages(t *testing.T) {
	if os.Getenv("AMQP_URI") == "" {
		t.Skip("Skipping integration test as AMQP_URI isn't set")
	}

	buildScriptServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "Hello, world")
	}))
	defer buildScriptServer.Close()

	conn, err := setupConn()
	if err != nil {
		t.Fatal(err)
	}

	err = publishJob(conn)
	if err != nil {
		t.Fatal(err)
	}

	amqpChan, err := conn.Channel()
	if err != nil {
		t.Fatal(err)
	}

	logParts := make([]LogPart, 2)

	delivery, _, err := amqpChan.Get("reporting.jobs.logs", true)
	if err != nil {
		t.Fatal(err)
	}

	err = json.Unmarshal(delivery.Body, logParts[0])
	if err != nil {
		t.Fatal(err)
	}

	delivery, _, err = amqpChan.Get("reporting.jobs.logs", true)
	if err != nil {
		t.Fatal(err)
	}

	err = json.Unmarshal(delivery.Body, logParts[1])
	if err != nil {
		t.Fatal(err)
	}

	sort.Sort(LogPartSlice(logParts))

	if logParts[0].ID != 3 {
		t.Errorf("logParts[0].ID = %d, expected 3", logParts[0].ID)
	}
	if !strings.Contains(logParts[0].Content, "Hello to the logs") {
		t.Errorf("logParts[0].Content = %q, expected to contain %q", logParts[0].Content, "Hello to the logs")
	}
	if logParts[0].Final {
		t.Errorf("logParts[0].Final = true, expected false")
	}
	if logParts[0].UUID != "fake-uuid" {
		t.Errorf("logParts[0].UUID = %q, expected fake-uuid", logParts[0].UUID)
	}

	expected := LogPart{ID: 3, Content: "", Final: true, UUID: "fake-uuid"}
	if logParts[1] != expected {
		t.Errorf("logParts[1] = %#v, expected %#v", logParts[1], expected)
	}
}
<commit_msg>Conditionally disable integration tests until further work can be done<commit_after>package worker_integration_test

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"os"
	"sort"
	"strings"
	"testing"

	"github.com/streadway/amqp"
)

type LogPart struct {
	ID      int64  `json:"id"`
	Content string `json:"log"`
	Final   bool   `json:"final"`
	UUID    string `json:"uuid"`
	Number  int64  `json:"number"`
}

type LogPartSlice []LogPart

func (lps LogPartSlice) Len() int {
	return len(lps)
}

func (lps LogPartSlice) Less(i, j int) bool {
	return lps[i].Number < lps[j].Number
}

func (lps LogPartSlice) Swap(i, j int) {
	lps[i], lps[j] = lps[j], lps[i]
}

type StateUpdate struct {
	ID    int64  `json:"id"`
	State string `json:"state"`
}

func setupConn() (*amqp.Connection, error) {
	amqpConn, err := amqp.Dial(os.Getenv("AMQP_URL"))
	if err != nil {
		return nil, err
	}

	amqpChan, err := amqpConn.Channel()
	if err != nil {
		return nil, err
	}
	defer amqpChan.Close()

	_, err = amqpChan.QueueDeclare("builds.test", true, false, false, false, nil)
	if err != nil {
		return nil, err
	}

	_, err = amqpChan.QueueDeclare("reporting.jobs.logs", true, false, false, false, nil)
	if err != nil {
		return nil, err
	}

	_, err = amqpChan.QueueDeclare("reporting.jobs.builds", true, false, false, false, nil)
	if err != nil {
		return nil, err
	}

	_, err = amqpChan.QueuePurge("builds.test", false)
	if err != nil {
		return nil, err
	}

	_, err = amqpChan.QueuePurge("reporting.jobs.logs", false)
	if err != nil {
		return nil, err
	}

	_, err = amqpChan.QueuePurge("reporting.jobs.builds", false)
	if err != nil {
		return nil, err
	}

	return amqpConn, nil
}

func publishJob(amqpConn *amqp.Connection) error {
	amqpChan, err := amqpConn.Channel()
	if err != nil {
		return err
	}
	defer amqpChan.Close()

	return amqpChan.Publish("", "builds.test", false, false, amqp.Publishing{
		Body: []byte(`{
			"type": "test",
			"job": {
				"id": 3,
				"number": "1.1",
				"commit": "abcdef",
				"commit_range": "abcde...abcdef",
				"commit_message": "Hello world",
				"branch": "master",
				"ref": null,
				"state": "queued",
				"secure_env_enabled": true,
				"pull_request": false
			},
			"source": {
				"id": 2,
				"number": "1"
			},
			"repository": {
				"id": 1,
				"slug": "hello/world",
				"github_id": 1234,
				"source_url": "git://github.com/hello/world.git",
				"api_url": "https://api.github.com",
				"last_build_id": 2,
				"last_build_number": "1",
				"last_build_started_at": null,
				"last_build_finished_at": null,
				"last_build_duration": null,
				"last_build_state": "created",
				"description": "Hello world",
				"config": {},
				"queue": "builds.test",
				"uuid": "fake-uuid",
				"ssh_key": null,
				"env_vars": [],
				"timeouts": {
					"hard_limit": null,
					"log_silence": null
				}
			}`),
		DeliveryMode: amqp.Persistent,
	})
}

func TestIntegrationLogMessages(t *testing.T) {
	if os.Getenv("AMQP_URI") == "" {
		t.Skip("Skipping integration test as AMQP_URI isn't set")
	}

	if os.Getenv("INTEGRATION_TESTS_DISABLED") != "" {
		t.Skip("Skipping disabled integration tests")
	}

	buildScriptServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "Hello, world")
	}))
	defer buildScriptServer.Close()

	conn, err := setupConn()
	if err != nil {
		t.Fatal(err)
	}

	err = publishJob(conn)
	if err != nil {
		t.Fatal(err)
	}

	amqpChan, err := conn.Channel()
	if err != nil {
		t.Fatal(err)
	}

	logParts := make([]LogPart, 2)

	delivery, _, err := amqpChan.Get("reporting.jobs.logs", true)
	if err != nil {
		t.Fatal(err)
	}

	err = json.Unmarshal(delivery.Body, logParts[0])
	if err != nil {
		t.Fatal(err)
	}

	delivery, _, err = amqpChan.Get("reporting.jobs.logs", true)
	if err != nil {
		t.Fatal(err)
	}

	err = json.Unmarshal(delivery.Body, logParts[1])
	if err != nil {
		t.Fatal(err)
	}

	sort.Sort(LogPartSlice(logParts))

	if logParts[0].ID != 3 {
		t.Errorf("logParts[0].ID = %d, expected 3", logParts[0].ID)
	}
	if !strings.Contains(logParts[0].Content, "Hello to the logs") {
		t.Errorf("logParts[0].Content = %q, expected to contain %q", logParts[0].Content, "Hello to the logs")
	}
	if logParts[0].Final {
		t.Errorf("logParts[0].Final = true, expected false")
	}
	if logParts[0].UUID != "fake-uuid" {
		t.Errorf("logParts[0].UUID = %q, expected fake-uuid", logParts[0].UUID)
	}

	expected := LogPart{ID: 3, Content: "", Final: true, UUID: "fake-uuid"}
	if logParts[1] != expected {
		t.Errorf("logParts[1] = %#v, expected %#v", logParts[1], expected)
	}
}
<|endoftext|>