<commit_before>package transports

import (
	"fmt"
	"encoding/json"
	"net/http"
	"io/ioutil"
	"os/exec"
	"time"
	"strings"
	"bytes"
	// "errors"
)

const YowsupHttpWrapperPath = "../yowsup-http-wrapper/run.py"

var ResponseChannel chan Response

type WhatsappTransport struct {
	*Transport
	Login         string
	Password      string
	Contact				string
	YowsupWrapperPort	string
	YowsupWrapperUrl string
	Serializer		DefaultSerializer
	Messages			[]WhatsappMessage
	// ResponseChannel	chan Response
}

type WhatsappMessage struct {
	Id string	`json:"id,omitempty"`
	Body string `json:"msg,omitempty"`
	Origin string	`json:"origin,omitempty"`
	Dest string `json:"dest,omitempty"`
}

type WhatsappMessageCallback func(*WhatsappTransport)

func (t *WhatsappTransport) DaemonizeWrapper() {
	fmt.Println( "WhatsappTransport, daemonizing YowsupWrapper...")

	t.YowsupWrapperUrl = fmt.Sprintf("http://127.0.0.1:%s/", t.YowsupWrapperPort)

	cmd := exec.Command( "python3", YowsupHttpWrapperPath, t.Login, t.Password, t.YowsupWrapperPort )
	err := cmd.Run()

	if err != nil {
		panic(err)
	}
}

func( t *WhatsappTransport) GetMessageIds() []string {
	MessageIds := make( []string, 0 )
	for _, Message := range t.Messages {
		MessageIds = append( MessageIds, Message.Id )
	}
	return MessageIds
}

func( t *WhatsappTransport) PurgeMessage( Id string ) {
	messagesUrl := fmt.Sprintf("%s%s?id=%s", t.YowsupWrapperUrl, "messages", Id)
	deleteRequest, _ := http.NewRequest( "DELETE", messagesUrl, nil)
	http.DefaultClient.Do(deleteRequest)
}

func( t *WhatsappTransport) FetchMessages() {
	messagesUrl := strings.Join([]string{t.YowsupWrapperUrl, "messages"}, "")
	resp, err := http.Get(messagesUrl)

	if err != nil {
		// fmt.Println( "Wrapper error:", err)
		return
	}

	defer resp.Body.Close()

	rawBody, _ := ioutil.ReadAll( resp.Body )

	// fmt.Println(string(rawBody))

	var messageList map[string]interface{}

	jsonErr := json.Unmarshal( rawBody, &messageList)

	if jsonErr != nil {
		return
	}

	MessageIds := t.GetMessageIds()

	for Id, Values := range messageList {
		ValuesMap := Values.(map[string]interface{})
		Message := WhatsappMessage{ Id: Id, Body: ValuesMap["body"].(string), Origin: ValuesMap["origin"].(string) }
		Exists := false

		for _, ExistingId := range MessageIds {
			if ExistingId == Id {
				Exists = true
				return
			}
		}

		if !Exists {
			t.Messages = append( t.Messages, Message )
		}
	}

	return
}

func (t *WhatsappTransport) SendMessage(body string) {
	messagesUrl := strings.Join([]string{t.YowsupWrapperUrl, "messages"}, "")
	message := WhatsappMessage{Body: body, Dest: t.Contact}
	jsonBuffer, _ := json.Marshal(&message)
	http.Post(messagesUrl, "application/json", bytes.NewReader(jsonBuffer) )

	fmt.Println("<-- Sending message\n", message, "\n" )

	return
}

func (t *WhatsappTransport) Prepare() {
	// fmt.Println("WhatsappTransport, Prepare()")

	t.YowsupWrapperUrl = fmt.Sprintf("http://127.0.0.1:%s/", t.YowsupWrapperPort)

	t.Serializer = DefaultSerializer{}

	t.Messages = make([]WhatsappMessage, 0)

	ResponseChannel = make( chan Response )

	return
}

func (t *WhatsappTransport) Handler(w http.ResponseWriter, originalRequest *http.Request) {

	// client := &http.Client{}

	// request, _ := http.NewRequest(originalRequest.Method, originalRequest.URL.String(), nil)

	serializedRequest := t.Serializer.Serialize(originalRequest, true).([]byte)

	go t.SendMessage( string(serializedRequest) )

	w.Header().Set( "Via", fmt.Sprintf("WhatsappTransport/%s", t.Contact) )

	response := <- ResponseChannel

	for HeaderKey, HeaderValue := range response.Headers {
		w.Header().Set( HeaderKey, HeaderValue[0] )
	}

	fmt.Println("--> Receiving message\n", response, "\n" )
	fmt.Println("<-> Forwarding message\n", response, "\n" )

	w.Write( []byte(response.Body) )

}

func(t *WhatsappTransport) HandleClientMessages() {
	for _, Value := range t.Messages {

		response := t.Serializer.DeserializeResponse([]byte(Value.Body))

		t.PurgeMessage( Value.Id)

		go func( r Response ) {
			ResponseChannel <- response
		}(response)

	}
	t.Messages = make([]WhatsappMessage, 0)
}

func (t *WhatsappTransport) Listen( Callback WhatsappMessageCallback ) {

	// fmt.Println( "WhatsappTransport, Listen()")
	fmt.Println("Polling...")

	if Callback != nil {
		t.Prepare()
	}

	go t.DaemonizeWrapper()

	for {
		// fmt.Println( "Poll, messages:", t.Messages )
		t.FetchMessages()

		if Callback == nil {
			t.HandleClientMessages()
		} else {
			Callback( t )
		}
		time.Sleep(1 * time.Second)
	}
	return
}
<commit_msg>Don't print the raw body<commit_after>package transports

import (
	"fmt"
	"encoding/json"
	"net/http"
	"io/ioutil"
	"os/exec"
	"time"
	"strings"
	"bytes"
	// "errors"
)

const YowsupHttpWrapperPath = "../yowsup-http-wrapper/run.py"

var ResponseChannel chan Response

type WhatsappTransport struct {
	*Transport
	Login         string
	Password      string
	Contact				string
	YowsupWrapperPort	string
	YowsupWrapperUrl string
	Serializer		DefaultSerializer
	Messages			[]WhatsappMessage
	// ResponseChannel	chan Response
}

type WhatsappMessage struct {
	Id string	`json:"id,omitempty"`
	Body string `json:"msg,omitempty"`
	Origin string	`json:"origin,omitempty"`
	Dest string `json:"dest,omitempty"`
}

type WhatsappMessageCallback func(*WhatsappTransport)

func (t *WhatsappTransport) DaemonizeWrapper() {
	fmt.Println( "WhatsappTransport, daemonizing YowsupWrapper...")

	t.YowsupWrapperUrl = fmt.Sprintf("http://127.0.0.1:%s/", t.YowsupWrapperPort)

	cmd := exec.Command( "python3", YowsupHttpWrapperPath, t.Login, t.Password, t.YowsupWrapperPort )
	err := cmd.Run()

	if err != nil {
		panic(err)
	}
}

func( t *WhatsappTransport) GetMessageIds() []string {
	MessageIds := make( []string, 0 )
	for _, Message := range t.Messages {
		MessageIds = append( MessageIds, Message.Id )
	}
	return MessageIds
}

func( t *WhatsappTransport) PurgeMessage( Id string ) {
	messagesUrl := fmt.Sprintf("%s%s?id=%s", t.YowsupWrapperUrl, "messages", Id)
	deleteRequest, _ := http.NewRequest( "DELETE", messagesUrl, nil)
	http.DefaultClient.Do(deleteRequest)
}

func( t *WhatsappTransport) FetchMessages() {
	messagesUrl := strings.Join([]string{t.YowsupWrapperUrl, "messages"}, "")
	resp, err := http.Get(messagesUrl)

	if err != nil {
		// fmt.Println( "Wrapper error:", err)
		return
	}

	defer resp.Body.Close()

	rawBody, _ := ioutil.ReadAll( resp.Body )

	var messageList map[string]interface{}

	jsonErr := json.Unmarshal( rawBody, &messageList)

	if jsonErr != nil {
		return
	}

	MessageIds := t.GetMessageIds()

	for Id, Values := range messageList {
		ValuesMap := Values.(map[string]interface{})
		Message := WhatsappMessage{ Id: Id, Body: ValuesMap["body"].(string), Origin: ValuesMap["origin"].(string) }
		Exists := false

		for _, ExistingId := range MessageIds {
			if ExistingId == Id {
				Exists = true
				return
			}
		}

		if !Exists {
			t.Messages = append( t.Messages, Message )
		}
	}

	return
}

func (t *WhatsappTransport) SendMessage(body string) {
	messagesUrl := strings.Join([]string{t.YowsupWrapperUrl, "messages"}, "")
	message := WhatsappMessage{Body: body, Dest: t.Contact}
	jsonBuffer, _ := json.Marshal(&message)
	http.Post(messagesUrl, "application/json", bytes.NewReader(jsonBuffer) )

	fmt.Println("<-- Sending message\n", message, "\n" )

	return
}

func (t *WhatsappTransport) Prepare() {
	// fmt.Println("WhatsappTransport, Prepare()")

	t.YowsupWrapperUrl = fmt.Sprintf("http://127.0.0.1:%s/", t.YowsupWrapperPort)

	t.Serializer = DefaultSerializer{}

	t.Messages = make([]WhatsappMessage, 0)

	ResponseChannel = make( chan Response )

	return
}

func (t *WhatsappTransport) Handler(w http.ResponseWriter, originalRequest *http.Request) {

	// client := &http.Client{}

	// request, _ := http.NewRequest(originalRequest.Method, originalRequest.URL.String(), nil)

	serializedRequest := t.Serializer.Serialize(originalRequest, true).([]byte)

	go t.SendMessage( string(serializedRequest) )

	w.Header().Set( "Via", fmt.Sprintf("WhatsappTransport/%s", t.Contact) )

	response := <- ResponseChannel

	for HeaderKey, HeaderValue := range response.Headers {
		w.Header().Set( HeaderKey, HeaderValue[0] )
	}

	fmt.Println("--> Receiving message\n", response, "\n" )
	fmt.Println("<-> Forwarding message\n", response, "\n" )

	w.Write( []byte(response.Body) )

}

func(t *WhatsappTransport) HandleClientMessages() {
	for _, Value := range t.Messages {

		response := t.Serializer.DeserializeResponse([]byte(Value.Body))

		t.PurgeMessage( Value.Id)

		go func( r Response ) {
			ResponseChannel <- response
		}(response)

	}
	t.Messages = make([]WhatsappMessage, 0)
}

func (t *WhatsappTransport) Listen( Callback WhatsappMessageCallback ) {

	// fmt.Println( "WhatsappTransport, Listen()")
	fmt.Println("Polling...")

	if Callback != nil {
		t.Prepare()
	}

	go t.DaemonizeWrapper()

	for {
		// fmt.Println( "Poll, messages:", t.Messages )
		t.FetchMessages()

		if Callback == nil {
			t.HandleClientMessages()
		} else {
			Callback( t )
		}
		time.Sleep(1 * time.Second)
	}
	return
}
<|endoftext|>


<commit_before>package glob

import (
	"bytes"
	"fmt"
	"strings"
	"unicode/utf8"
)

const (
	char_any           = '*'
	char_separator     = ','
	char_single        = '?'
	char_escape        = '\\'
	char_range_open    = '['
	char_range_close   = ']'
	char_terms_open    = '{'
	char_terms_close   = '}'
	char_range_not     = '!'
	char_range_between = '-'
)

var specials = []byte{
	char_any,
	char_separator,
	char_single,
	char_escape,
	char_range_open,
	char_range_close,
	char_terms_open,
	char_terms_close,
	char_range_not,
	char_range_between,
}

func special(c byte) bool {
	return bytes.IndexByte(specials, c) != -1
}

var eof rune = 0

type stateFn func(*lexer) stateFn

type itemType int

const (
	item_eof itemType = iota
	item_error
	item_text
	item_char
	item_any
	item_super
	item_single
	item_not
	item_separator
	item_range_open
	item_range_close
	item_range_lo
	item_range_hi
	item_range_between
	item_terms_open
	item_terms_close
)

func (i itemType) String() string {
	switch i {
	case item_eof:
		return "eof"

	case item_error:
		return "error"

	case item_text:
		return "text"

	case item_char:
		return "char"

	case item_any:
		return "any"

	case item_super:
		return "super"

	case item_single:
		return "single"

	case item_not:
		return "not"

	case item_separator:
		return "separator"

	case item_range_open:
		return "range_open"

	case item_range_close:
		return "range_close"

	case item_range_lo:
		return "range_lo"

	case item_range_hi:
		return "range_hi"

	case item_range_between:
		return "range_between"

	case item_terms_open:
		return "terms_open"

	case item_terms_close:
		return "terms_close"

	default:
		return "undef"
	}
}

type item struct {
	t itemType
	s string
}

func (i item) String() string {
	return fmt.Sprintf("%v<%s>", i.t, i.s)
}

type lexer struct {
	input       string
	start       int
	pos         int
	width       int
	runes       int
	termScopes  []int
	termPhrases map[int]int
	state       stateFn
	items       chan item
}

func newLexer(source string) *lexer {
	l := &lexer{
		input:       source,
		state:       lexText,
		items:       make(chan item, len(source)),
		termPhrases: make(map[int]int),
	}
	return l
}

func (l *lexer) run() {
	for state := lexText; state != nil; {
		state = state(l)
	}
	close(l.items)
}

func (l *lexer) nextItem() item {
	for {
		select {
		case item := <-l.items:
			return item
		default:
			if l.state == nil {
				return item{t: item_eof}
			}

			l.state = l.state(l)
		}
	}

	panic("something went wrong")
}

func (l *lexer) read() (r rune) {
	if l.pos >= len(l.input) {
		return eof
	}

	r, l.width = utf8.DecodeRuneInString(l.input[l.pos:])
	l.pos += l.width
	l.runes++

	return
}

func (l *lexer) unread() {
	l.pos -= l.width
	l.runes--
}

func (l *lexer) reset() {
	l.pos = l.start
	l.runes = 0
}

func (l *lexer) ignore() {
	l.start = l.pos
	l.runes = 0
}

func (l *lexer) lookahead() rune {
	r := l.read()
	if r != eof {
		l.unread()
	}
	return r
}

func (l *lexer) accept(valid string) bool {
	if strings.IndexRune(valid, l.read()) != -1 {
		return true
	}
	l.unread()
	return false
}

func (l *lexer) acceptAll(valid string) {
	for strings.IndexRune(valid, l.read()) != -1 {
	}
	l.unread()
}

func (l *lexer) emit(t itemType) {
	if l.pos == len(l.input) {
		l.items <- item{t, l.input[l.start:]}
	} else {
		l.items <- item{t, l.input[l.start:l.pos]}
	}

	l.start = l.pos
	l.runes = 0
	l.width = 0
}

func (l *lexer) emitMaybe(t itemType) {
	if l.pos > l.start {
		l.emit(t)
	}
}

func (l *lexer) errorf(format string, args ...interface{}) {
	l.items <- item{item_error, fmt.Sprintf(format, args...)}
}

func lexText(l *lexer) stateFn {
	for {
		c := l.read()
		if c == eof {
			break
		}

		switch c {
		case char_escape:
			l.unread()
			l.emitMaybe(item_text)

			l.read()
			l.ignore()

			if l.read() == eof {
				l.errorf("unclosed '%s' character", string(char_escape))
				return nil
			}

		case char_single:
			l.unread()
			l.emitMaybe(item_text)
			return lexSingle

		case char_any:
			var n stateFn
			if l.lookahead() == char_any {
				n = lexSuper
			} else {
				n = lexAny
			}

			l.unread()
			l.emitMaybe(item_text)
			return n

		case char_range_open:
			l.unread()
			l.emitMaybe(item_text)
			return lexRangeOpen

		case char_terms_open:
			l.unread()
			l.emitMaybe(item_text)
			return lexTermsOpen

		case char_terms_close:
			l.unread()
			l.emitMaybe(item_text)
			return lexTermsClose

		case char_separator:
			l.unread()
			l.emitMaybe(item_text)
			return lexSeparator
		}
	}

	if l.pos > l.start {
		l.emit(item_text)
	}

	if len(l.termScopes) != 0 {
		l.errorf("invalid pattern syntax: unclosed terms")
		return nil
	}

	l.emit(item_eof)

	return nil
}

func lexInsideRange(l *lexer) stateFn {
	for {
		c := l.read()
		if c == eof {
			l.errorf("unclosed range construction")
			return nil
		}

		switch c {
		case char_range_not:
			// only first char makes sense
			if l.pos-l.width == l.start {
				l.emit(item_not)
			}

		case char_range_between:
			if l.runes != 2 {
				l.errorf("unexpected length of lo char inside range")
				return nil
			}

			l.reset()
			return lexRangeHiLo

		case char_range_close:
			l.unread()
			l.emitMaybe(item_text)
			return lexRangeClose
		}
	}
}

func lexRangeHiLo(l *lexer) stateFn {
	start := l.start

	for {
		c := l.read()
		if c == eof {
			l.errorf("unexpected end of input")
			return nil
		}

		switch c {
		case char_range_between:
			if l.runes != 1 {
				l.errorf("unexpected length of range: single character expected before minus")
				return nil
			}

			l.emit(item_range_between)

		case char_range_close:
			l.unread()

			if l.runes != 1 {
				l.errorf("unexpected length of range: single character expected before close")
				return nil
			}

			l.emit(item_range_hi)
			return lexRangeClose

		default:
			if start != l.start {
				continue
			}

			if l.runes != 1 {
				l.errorf("unexpected length of range: single character expected at the begining")
				return nil
			}

			l.emit(item_range_lo)
		}
	}
}

func lexAny(l *lexer) stateFn {
	l.pos += 1
	l.emit(item_any)
	return lexText
}

func lexSuper(l *lexer) stateFn {
	l.pos += 2
	l.emit(item_super)
	return lexText
}

func lexSingle(l *lexer) stateFn {
	l.pos += 1
	l.emit(item_single)
	return lexText
}

func lexSeparator(l *lexer) stateFn {
	if len(l.termScopes) == 0 {
		l.errorf("syntax error: separator not inside terms list")
		return nil
	}

	posOpen := l.termScopes[len(l.termScopes)-1]

	if l.pos-posOpen == 1 {
		l.errorf("syntax error: empty term before separator")
		return nil
	}

	l.termPhrases[posOpen] += 1
	l.pos += 1
	l.emit(item_separator)
	return lexText
}

func lexTermsOpen(l *lexer) stateFn {
	l.termScopes = append(l.termScopes, l.pos)
	l.pos += 1
	l.emit(item_terms_open)

	return lexText
}

func lexTermsClose(l *lexer) stateFn {
	if len(l.termScopes) == 0 {
		l.errorf("unexpected closing of terms: there is no opened terms")
		return nil
	}

	lastOpen := len(l.termScopes) - 1
	posOpen := l.termScopes[lastOpen]

	// if it is empty term
	if posOpen == l.pos-1 {
		l.errorf("term could not be empty")
		return nil
	}

	if l.termPhrases[posOpen] == 0 {
		l.errorf("term must contain >1 phrases")
		return nil
	}

	// cleanup
	l.termScopes = l.termScopes[:lastOpen]
	delete(l.termPhrases, posOpen)

	l.pos += 1
	l.emit(item_terms_close)

	return lexText
}

func lexRangeOpen(l *lexer) stateFn {
	l.pos += 1
	l.emit(item_range_open)
	return lexInsideRange
}

func lexRangeClose(l *lexer) stateFn {
	l.pos += 1
	l.emit(item_range_close)
	return lexText
}
<commit_msg>cleanup specials<commit_after>package glob

import (
	"bytes"
	"fmt"
	"strings"
	"unicode/utf8"
)

const (
	char_any           = '*'
	char_separator     = ','
	char_single        = '?'
	char_escape        = '\\'
	char_range_open    = '['
	char_range_close   = ']'
	char_terms_open    = '{'
	char_terms_close   = '}'
	char_range_not     = '!'
	char_range_between = '-'
)

var specials = []byte{
	char_any,
	char_single,
	char_escape,
	char_range_open,
	char_range_close,
	char_terms_open,
	char_terms_close,
}

func special(c byte) bool {
	return bytes.IndexByte(specials, c) != -1
}

var eof rune = 0

type stateFn func(*lexer) stateFn

type itemType int

const (
	item_eof itemType = iota
	item_error
	item_text
	item_char
	item_any
	item_super
	item_single
	item_not
	item_separator
	item_range_open
	item_range_close
	item_range_lo
	item_range_hi
	item_range_between
	item_terms_open
	item_terms_close
)

func (i itemType) String() string {
	switch i {
	case item_eof:
		return "eof"

	case item_error:
		return "error"

	case item_text:
		return "text"

	case item_char:
		return "char"

	case item_any:
		return "any"

	case item_super:
		return "super"

	case item_single:
		return "single"

	case item_not:
		return "not"

	case item_separator:
		return "separator"

	case item_range_open:
		return "range_open"

	case item_range_close:
		return "range_close"

	case item_range_lo:
		return "range_lo"

	case item_range_hi:
		return "range_hi"

	case item_range_between:
		return "range_between"

	case item_terms_open:
		return "terms_open"

	case item_terms_close:
		return "terms_close"

	default:
		return "undef"
	}
}

type item struct {
	t itemType
	s string
}

func (i item) String() string {
	return fmt.Sprintf("%v<%s>", i.t, i.s)
}

type lexer struct {
	input       string
	start       int
	pos         int
	width       int
	runes       int
	termScopes  []int
	termPhrases map[int]int
	state       stateFn
	items       chan item
}

func newLexer(source string) *lexer {
	l := &lexer{
		input:       source,
		state:       lexText,
		items:       make(chan item, len(source)),
		termPhrases: make(map[int]int),
	}
	return l
}

func (l *lexer) run() {
	for state := lexText; state != nil; {
		state = state(l)
	}
	close(l.items)
}

func (l *lexer) nextItem() item {
	for {
		select {
		case item := <-l.items:
			return item
		default:
			if l.state == nil {
				return item{t: item_eof}
			}

			l.state = l.state(l)
		}
	}

	panic("something went wrong")
}

func (l *lexer) read() (r rune) {
	if l.pos >= len(l.input) {
		return eof
	}

	r, l.width = utf8.DecodeRuneInString(l.input[l.pos:])
	l.pos += l.width
	l.runes++

	return
}

func (l *lexer) unread() {
	l.pos -= l.width
	l.runes--
}

func (l *lexer) reset() {
	l.pos = l.start
	l.runes = 0
}

func (l *lexer) ignore() {
	l.start = l.pos
	l.runes = 0
}

func (l *lexer) lookahead() rune {
	r := l.read()
	if r != eof {
		l.unread()
	}
	return r
}

func (l *lexer) accept(valid string) bool {
	if strings.IndexRune(valid, l.read()) != -1 {
		return true
	}
	l.unread()
	return false
}

func (l *lexer) acceptAll(valid string) {
	for strings.IndexRune(valid, l.read()) != -1 {
	}
	l.unread()
}

func (l *lexer) emit(t itemType) {
	if l.pos == len(l.input) {
		l.items <- item{t, l.input[l.start:]}
	} else {
		l.items <- item{t, l.input[l.start:l.pos]}
	}

	l.start = l.pos
	l.runes = 0
	l.width = 0
}

func (l *lexer) emitMaybe(t itemType) {
	if l.pos > l.start {
		l.emit(t)
	}
}

func (l *lexer) errorf(format string, args ...interface{}) {
	l.items <- item{item_error, fmt.Sprintf(format, args...)}
}

func lexText(l *lexer) stateFn {
	for {
		c := l.read()
		if c == eof {
			break
		}

		switch c {
		case char_escape:
			l.unread()
			l.emitMaybe(item_text)

			l.read()
			l.ignore()

			if l.read() == eof {
				l.errorf("unclosed '%s' character", string(char_escape))
				return nil
			}

		case char_single:
			l.unread()
			l.emitMaybe(item_text)
			return lexSingle

		case char_any:
			var n stateFn
			if l.lookahead() == char_any {
				n = lexSuper
			} else {
				n = lexAny
			}

			l.unread()
			l.emitMaybe(item_text)
			return n

		case char_range_open:
			l.unread()
			l.emitMaybe(item_text)
			return lexRangeOpen

		case char_terms_open:
			l.unread()
			l.emitMaybe(item_text)
			return lexTermsOpen

		case char_terms_close:
			l.unread()
			l.emitMaybe(item_text)
			return lexTermsClose

		case char_separator:
			l.unread()
			l.emitMaybe(item_text)
			return lexSeparator
		}
	}

	if l.pos > l.start {
		l.emit(item_text)
	}

	if len(l.termScopes) != 0 {
		l.errorf("invalid pattern syntax: unclosed terms")
		return nil
	}

	l.emit(item_eof)

	return nil
}

func lexInsideRange(l *lexer) stateFn {
	for {
		c := l.read()
		if c == eof {
			l.errorf("unclosed range construction")
			return nil
		}

		switch c {
		case char_range_not:
			// only first char makes sense
			if l.pos-l.width == l.start {
				l.emit(item_not)
			}

		case char_range_between:
			if l.runes != 2 {
				l.errorf("unexpected length of lo char inside range")
				return nil
			}

			l.reset()
			return lexRangeHiLo

		case char_range_close:
			l.unread()
			l.emitMaybe(item_text)
			return lexRangeClose
		}
	}
}

func lexRangeHiLo(l *lexer) stateFn {
	start := l.start

	for {
		c := l.read()
		if c == eof {
			l.errorf("unexpected end of input")
			return nil
		}

		switch c {
		case char_range_between:
			if l.runes != 1 {
				l.errorf("unexpected length of range: single character expected before minus")
				return nil
			}

			l.emit(item_range_between)

		case char_range_close:
			l.unread()

			if l.runes != 1 {
				l.errorf("unexpected length of range: single character expected before close")
				return nil
			}

			l.emit(item_range_hi)
			return lexRangeClose

		default:
			if start != l.start {
				continue
			}

			if l.runes != 1 {
				l.errorf("unexpected length of range: single character expected at the begining")
				return nil
			}

			l.emit(item_range_lo)
		}
	}
}

func lexAny(l *lexer) stateFn {
	l.pos += 1
	l.emit(item_any)
	return lexText
}

func lexSuper(l *lexer) stateFn {
	l.pos += 2
	l.emit(item_super)
	return lexText
}

func lexSingle(l *lexer) stateFn {
	l.pos += 1
	l.emit(item_single)
	return lexText
}

func lexSeparator(l *lexer) stateFn {
	if len(l.termScopes) == 0 {
		l.errorf("syntax error: separator not inside terms list")
		return nil
	}

	posOpen := l.termScopes[len(l.termScopes)-1]

	if l.pos-posOpen == 1 {
		l.errorf("syntax error: empty term before separator")
		return nil
	}

	l.termPhrases[posOpen] += 1
	l.pos += 1
	l.emit(item_separator)
	return lexText
}

func lexTermsOpen(l *lexer) stateFn {
	l.termScopes = append(l.termScopes, l.pos)
	l.pos += 1
	l.emit(item_terms_open)

	return lexText
}

func lexTermsClose(l *lexer) stateFn {
	if len(l.termScopes) == 0 {
		l.errorf("unexpected closing of terms: there is no opened terms")
		return nil
	}

	lastOpen := len(l.termScopes) - 1
	posOpen := l.termScopes[lastOpen]

	// if it is empty term
	if posOpen == l.pos-1 {
		l.errorf("term could not be empty")
		return nil
	}

	if l.termPhrases[posOpen] == 0 {
		l.errorf("term must contain >1 phrases")
		return nil
	}

	// cleanup
	l.termScopes = l.termScopes[:lastOpen]
	delete(l.termPhrases, posOpen)

	l.pos += 1
	l.emit(item_terms_close)

	return lexText
}

func lexRangeOpen(l *lexer) stateFn {
	l.pos += 1
	l.emit(item_range_open)
	return lexInsideRange
}

func lexRangeClose(l *lexer) stateFn {
	l.pos += 1
	l.emit(item_range_close)
	return lexText
}
<|endoftext|>


<commit_before>package codeship

import (
	"encoding/json"
	"fmt"

	log "github.com/Sirupsen/logrus"

	cc "github.com/commonchat/commonchat-go"
	"github.com/grokify/webhook-proxy-go/src/adapters"
	"github.com/grokify/webhook-proxy-go/src/config"
	"github.com/grokify/webhook-proxy-go/src/util"
	"github.com/valyala/fasthttp"
)

const (
	DisplayName      = "Codeship"
	HandlerKey       = "codeship"
	IconURL          = "https://res.cloudinary.com/crunchbase-production/image/upload/v1408220431/yr2j599ysmtkfevjpvll.png"
	DocumentationURL = "https://documentation.codeship.com/basic/getting-started/webhooks/"
)

// FastHttp request handler for Semaphore CI outbound webhook
type CodeshipOutToGlipHandler struct {
	Config  config.Configuration
	Adapter adapters.Adapter
}

// FastHttp request handler constructor for Semaphore CI outbound webhook
func NewCodeshipOutToGlipHandler(cfg config.Configuration, adapter adapters.Adapter) CodeshipOutToGlipHandler {
	return CodeshipOutToGlipHandler{Config: cfg, Adapter: adapter}
}

// HandleFastHTTP is the method to respond to a fasthttp request.
func (h *CodeshipOutToGlipHandler) HandleFastHTTP(ctx *fasthttp.RequestCtx) {
	ccMsg, err := Normalize(ctx.PostBody())

	if err != nil {
		ctx.SetStatusCode(fasthttp.StatusNotAcceptable)
		log.WithFields(log.Fields{
			"type":   "http.response",
			"status": fasthttp.StatusNotAcceptable,
		}).Info(fmt.Sprintf("%v request is not acceptable.", DisplayName))
		return
	}

	util.SendWebhook(ctx, h.Adapter, ccMsg)
}

func Normalize(bytes []byte) (cc.Message, error) {
	message := cc.NewMessage()
	message.IconURL = IconURL

	src, err := CodeshipOutMessageFromBytes(bytes)
	if err != nil {
		return message, err
	}

	build := src.Build

	status := build.Status
	if status == "infrastructure_failure" {
		status = "failed due to infrastructure error"
	}

	message.Activity = fmt.Sprintf("Build %v", status)
	message.Title = fmt.Sprintf("[Build #%v](%s) for **%s** %s ([%s](%s))",
		build.BuildId,
		build.BuildURL,
		build.ProjectName,
		status,
		build.ShortCommitId,
		build.CommitURL)

	attachment := cc.NewAttachment()

	if len(build.Message) > 0 {
		if len(build.CommitURL) > 0 {
			attachment.AddField(cc.Field{
				Title: "Message",
				Value: fmt.Sprintf("[%v](%v)", build.Message, build.CommitURL)})
		} else {
			attachment.AddField(cc.Field{
				Title: "Message",
				Value: build.Message})
		}
	}
	if len(build.Branch) > 0 {
		attachment.AddField(cc.Field{
			Title: "Branch",
			Value: build.Branch,
			Short: true})
	}
	if len(build.Committer) > 0 {
		attachment.AddField(cc.Field{
			Title: "Committer",
			Value: build.Committer,
			Short: true})
	}

	message.AddAttachment(attachment)
	return message, nil
}

type CodeshipOutMessage struct {
	Build CodeshipOutBuild `json:"build,omitempty"`
}

type CodeshipOutBuild struct {
	BuildURL        string `json:"build_url,omitempty"`
	CommitURL       string `json:"commit_url,omitempty"`
	ProjectId       int64  `json:"project_id,omitempty"`
	BuildId         int64  `json:"build_id,omitempty"`
	Status          string `json:"status,omitempty"`
	ProjectFullName string `json:"project_full_name,omitempty"`
	ProjectName     string `json:"project_name,omitempty"`
	CommitId        string `json:"commit_id,omitempty"`
	ShortCommitId   string `json:"short_commit_id,omitempty"`
	Message         string `json:"message,omitempty"`
	Committer       string `json:"committer,omitempty"`
	Branch          string `json:"branch,omitempty"`
}

func CodeshipOutMessageFromBytes(bytes []byte) (CodeshipOutMessage, error) {
	msg := CodeshipOutMessage{}
	err := json.Unmarshal(bytes, &msg)
	return msg, err
}

/*
{
  "build": {
    "build_url":"https://www.codeship.com/projects/10213/builds/973711",
    "commit_url":"https://github.com/codeship/docs/
                  commit/96943dc5269634c211b6fbb18896ecdcbd40a047",
    "project_id":10213,
    "build_id":973711,
    "status":"testing",
    # PROJECT_FULL_NAME IS DEPRECATED AND WILL BE REMOVED IN THE FUTURE
    "project_full_name":"codeship/docs",
    "project_name":"codeship/docs",
    "commit_id":"96943dc5269634c211b6fbb18896ecdcbd40a047",
    "short_commit_id":"96943",
    "message":"Merge pull request #34 from codeship/feature/shallow-clone",
    "committer":"beanieboi",
    "branch":"master"
  }
}

The status field can have one of the following values:
testing for newly started build

error for failed builds
success for passed builds
stopped for stopped builds
waiting for waiting builds
ignored for builds ignored because the account is over the monthly build limit
blocked for builds blocked because of excessive resource consumption
infrastructure_failure for builds which failed because of an internal error on the build VM
*/
<commit_msg>update codeship<commit_after>package codeship

import (
	"encoding/json"
	"fmt"

	log "github.com/Sirupsen/logrus"

	cc "github.com/commonchat/commonchat-go"
	"github.com/grokify/webhook-proxy-go/src/adapters"
	"github.com/grokify/webhook-proxy-go/src/config"
	"github.com/grokify/webhook-proxy-go/src/util"
	"github.com/valyala/fasthttp"
)

const (
	DisplayName      = "Codeship"
	HandlerKey       = "codeship"
	IconURL          = "http://chaindock.com/wp-content/uploads/2016/10/codeship.png"
	DocumentationURL = "https://documentation.codeship.com/basic/getting-started/webhooks/"
)

// FastHttp request handler for Semaphore CI outbound webhook
type CodeshipOutToGlipHandler struct {
	Config  config.Configuration
	Adapter adapters.Adapter
}

// FastHttp request handler constructor for Semaphore CI outbound webhook
func NewCodeshipOutToGlipHandler(cfg config.Configuration, adapter adapters.Adapter) CodeshipOutToGlipHandler {
	return CodeshipOutToGlipHandler{Config: cfg, Adapter: adapter}
}

// HandleFastHTTP is the method to respond to a fasthttp request.
func (h *CodeshipOutToGlipHandler) HandleFastHTTP(ctx *fasthttp.RequestCtx) {
	ccMsg, err := Normalize(ctx.PostBody())

	if err != nil {
		ctx.SetStatusCode(fasthttp.StatusNotAcceptable)
		log.WithFields(log.Fields{
			"type":   "http.response",
			"status": fasthttp.StatusNotAcceptable,
		}).Info(fmt.Sprintf("%v request is not acceptable.", DisplayName))
		return
	}

	util.SendWebhook(ctx, h.Adapter, ccMsg)
}

func Normalize(bytes []byte) (cc.Message, error) {
	message := cc.NewMessage()
	message.IconURL = IconURL

	src, err := CodeshipOutMessageFromBytes(bytes)
	if err != nil {
		return message, err
	}

	build := src.Build

	status := build.Status
	if status == "infrastructure_failure" {
		status = "failed due to infrastructure error"
	}

	message.Activity = fmt.Sprintf("Build %v", status)
	message.Title = fmt.Sprintf("[Build #%v](%s) for **%s** %s ([%s](%s))",
		build.BuildId,
		build.BuildURL,
		build.ProjectName,
		status,
		build.ShortCommitId,
		build.CommitURL)

	attachment := cc.NewAttachment()

	if len(build.Message) > 0 {
		if len(build.CommitURL) > 0 {
			attachment.AddField(cc.Field{
				Title: "Message",
				Value: fmt.Sprintf("[%v](%v)", build.Message, build.CommitURL)})
		} else {
			attachment.AddField(cc.Field{
				Title: "Message",
				Value: build.Message})
		}
	}
	if len(build.Branch) > 0 {
		attachment.AddField(cc.Field{
			Title: "Branch",
			Value: build.Branch,
			Short: true})
	}
	if len(build.Committer) > 0 {
		attachment.AddField(cc.Field{
			Title: "Committer",
			Value: build.Committer,
			Short: true})
	}

	message.AddAttachment(attachment)
	return message, nil
}

type CodeshipOutMessage struct {
	Build CodeshipOutBuild `json:"build,omitempty"`
}

type CodeshipOutBuild struct {
	BuildURL        string `json:"build_url,omitempty"`
	CommitURL       string `json:"commit_url,omitempty"`
	ProjectId       int64  `json:"project_id,omitempty"`
	BuildId         int64  `json:"build_id,omitempty"`
	Status          string `json:"status,omitempty"`
	ProjectFullName string `json:"project_full_name,omitempty"`
	ProjectName     string `json:"project_name,omitempty"`
	CommitId        string `json:"commit_id,omitempty"`
	ShortCommitId   string `json:"short_commit_id,omitempty"`
	Message         string `json:"message,omitempty"`
	Committer       string `json:"committer,omitempty"`
	Branch          string `json:"branch,omitempty"`
}

func CodeshipOutMessageFromBytes(bytes []byte) (CodeshipOutMessage, error) {
	msg := CodeshipOutMessage{}
	err := json.Unmarshal(bytes, &msg)
	return msg, err
}

/*
{
  "build": {
    "build_url":"https://www.codeship.com/projects/10213/builds/973711",
    "commit_url":"https://github.com/codeship/docs/
                  commit/96943dc5269634c211b6fbb18896ecdcbd40a047",
    "project_id":10213,
    "build_id":973711,
    "status":"testing",
    # PROJECT_FULL_NAME IS DEPRECATED AND WILL BE REMOVED IN THE FUTURE
    "project_full_name":"codeship/docs",
    "project_name":"codeship/docs",
    "commit_id":"96943dc5269634c211b6fbb18896ecdcbd40a047",
    "short_commit_id":"96943",
    "message":"Merge pull request #34 from codeship/feature/shallow-clone",
    "committer":"beanieboi",
    "branch":"master"
  }
}

The status field can have one of the following values:
testing for newly started build

error for failed builds
success for passed builds
stopped for stopped builds
waiting for waiting builds
ignored for builds ignored because the account is over the monthly build limit
blocked for builds blocked because of excessive resource consumption
infrastructure_failure for builds which failed because of an internal error on the build VM
*/
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/gorilla/mux"
	tid "github.com/Financial-Times/transactionid-utils-go"
	"golang.org/x/net/context"
	"io"
	"net/http"
)

const uuidKey = "uuid"

type Handlers struct {
	serviceConfig *ServiceConfig
	log           *AppLogger
}

func pingHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "pong")
}

func buildInfoHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "build-info")
}

func (h Handlers) contentPreviewHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	uuid := vars["uuid"]

	h.log.TransactionStartedEvent(r.RequestURI, tid.GetTransactionIDFromRequest(r), uuid)

	ctx := tid.TransactionAwareContext(context.Background(), r.Header.Get(tid.TransactionIDHeader))
	ctx = context.WithValue(ctx, uuidKey, uuid)
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	success, nativeContentSourceAppResponse := h.getNativeContent(ctx, w)
	if !success { return }
	success, transformAppResponse := h.getTransformedContent(ctx, *nativeContentSourceAppResponse, w)
	if(!success) {
		nativeContentSourceAppResponse.Body.Close()

		return
	}

	io.Copy(w, transformAppResponse.Body)
	transformAppResponse.Body.Close()
}

func ( h Handlers) getNativeContent(ctx context.Context, w http.ResponseWriter) (ok bool, resp *http.Response) {
	uuid := ctx.Value(uuidKey).(string)
	requestUrl := fmt.Sprintf("%s%s", h.serviceConfig.nativeContentAppUri, uuid)

	transactionId, _ := tid.GetTransactionIDFromContext(ctx)
	h.log.RequestEvent(h.serviceConfig.sourceAppName, requestUrl, transactionId, uuid)

	req, err := http.NewRequest("GET", requestUrl, nil)

	req.Header.Set(tid.TransactionIDHeader, transactionId)
	req.Header.Set("Authorization", "Basic " + h.serviceConfig.nativeContentAppAuth)
	req.Header.Set("Content-Type", "application/json")
	resp, err = client.Do(req)

	//this happens when hostname cannot be resolved or host is not accessible
	if err !=nil {
		h.log.ErrorEvent(h.serviceConfig.sourceAppName, req.URL.String(), req.Header.Get(tid.TransactionIDHeader), err, uuid)
		w.WriteHeader(http.StatusServiceUnavailable)
		return false, nil
	}
	if resp.StatusCode != http.StatusOK {
		w.WriteHeader(http.StatusNotFound);
		h.log.RequestFailedEvent(h.serviceConfig.sourceAppName, req.URL.String(), resp, uuid)
		return false, nil
	}
	h.log.ResponseEvent(h.serviceConfig.sourceAppName, req.URL.String(), resp, uuid)
	return true, resp
}

func ( h Handlers) getTransformedContent(ctx context.Context, nativeContentSourceAppResponse http.Response, w http.ResponseWriter) (ok bool, resp *http.Response) {
	uuid := ctx.Value(uuidKey).(string)
	requestUrl := fmt.Sprintf("%s%s?preview=true", h.serviceConfig.transformAppUri, uuid)
	transactionId, _ := tid.GetTransactionIDFromContext(ctx)

	//TODO we need to assert that resp.Header.Get(tid.TransactionIDHeader) ==  transactionId
	//to ensure that we are logging exactly what is actually passed around in the headers

	h.log.RequestEvent(h.serviceConfig.transformAppName, requestUrl, transactionId, uuid)

	req, err := http.NewRequest("POST", requestUrl, nativeContentSourceAppResponse.Body)
	req.Host = h.serviceConfig.transformAppHostHeader
	req.Header.Set(tid.TransactionIDHeader, transactionId)
	req.Header.Set("Content-Type", "application/json")

	resp, err = client.Do(req)

	//this happens when hostname cannot be resolved or host is not accessible

	if err !=nil {
		h.log.ErrorEvent(h.serviceConfig.transformAppName, req.URL.String(), req.Header.Get(tid.TransactionIDHeader), err, uuid)
		return false, nil
	}

	if resp.StatusCode != http.StatusOK {

		w.WriteHeader(http.StatusNotFound);
		h.log.RequestFailedEvent(h.serviceConfig.transformAppName, req.URL.String(), resp, uuid)
		return false, nil
	}

	h.log.ResponseEvent(h.serviceConfig.transformAppName, req.URL.String(), resp, uuid)
	return true, resp
}

<commit_msg>removed more references to MAT & MAPI<commit_after>package main

import (
	"fmt"
	"github.com/gorilla/mux"
	tid "github.com/Financial-Times/transactionid-utils-go"
	"golang.org/x/net/context"
	"io"
	"net/http"
)

const uuidKey = "uuid"

type Handlers struct {
	serviceConfig *ServiceConfig
	log           *AppLogger
}

func pingHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "pong")
}

func buildInfoHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "build-info")
}

func (h Handlers) contentPreviewHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	uuid := vars["uuid"]

	h.log.TransactionStartedEvent(r.RequestURI, tid.GetTransactionIDFromRequest(r), uuid)

	ctx := tid.TransactionAwareContext(context.Background(), r.Header.Get(tid.TransactionIDHeader))
	ctx = context.WithValue(ctx, uuidKey, uuid)
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	success, nativeContentSourceAppResponse := h.getNativeContent(ctx, w)
	if !success { return }
	success, transformAppResponse := h.getTransformedContent(ctx, *nativeContentSourceAppResponse, w)
	if(!success) {
		nativeContentSourceAppResponse.Body.Close()
		return
	}

	io.Copy(w, transformAppResponse.Body)
	transformAppResponse.Body.Close()
}

func ( h Handlers) getNativeContent(ctx context.Context, w http.ResponseWriter) (ok bool, resp *http.Response) {
	uuid := ctx.Value(uuidKey).(string)
	requestUrl := fmt.Sprintf("%s%s", h.serviceConfig.nativeContentAppUri, uuid)

	transactionId, _ := tid.GetTransactionIDFromContext(ctx)
	h.log.RequestEvent(h.serviceConfig.sourceAppName, requestUrl, transactionId, uuid)

	req, err := http.NewRequest("GET", requestUrl, nil)

	req.Header.Set(tid.TransactionIDHeader, transactionId)
	req.Header.Set("Authorization", "Basic " + h.serviceConfig.nativeContentAppAuth)
	req.Header.Set("Content-Type", "application/json")
	resp, err = client.Do(req)

	//this happens when hostname cannot be resolved or host is not accessible
	if err !=nil {
		h.log.ErrorEvent(h.serviceConfig.sourceAppName, req.URL.String(), req.Header.Get(tid.TransactionIDHeader), err, uuid)
		w.WriteHeader(http.StatusServiceUnavailable)
		return false, nil
	}
	if resp.StatusCode != http.StatusOK {
		w.WriteHeader(http.StatusNotFound);
		h.log.RequestFailedEvent(h.serviceConfig.sourceAppName, req.URL.String(), resp, uuid)
		return false, nil
	}
	h.log.ResponseEvent(h.serviceConfig.sourceAppName, req.URL.String(), resp, uuid)
	return true, resp
}

func ( h Handlers) getTransformedContent(ctx context.Context, nativeContentSourceAppResponse http.Response, w http.ResponseWriter) (ok bool, resp *http.Response) {
	uuid := ctx.Value(uuidKey).(string)
	requestUrl := fmt.Sprintf("%s%s?preview=true", h.serviceConfig.transformAppUri, uuid)
	transactionId, _ := tid.GetTransactionIDFromContext(ctx)

	//TODO we need to assert that resp.Header.Get(tid.TransactionIDHeader) ==  transactionId
	//to ensure that we are logging exactly what is actually passed around in the headers
	h.log.RequestEvent(h.serviceConfig.transformAppName, requestUrl, transactionId, uuid)

	req, err := http.NewRequest("POST", requestUrl, nativeContentSourceAppResponse.Body)
	req.Host = h.serviceConfig.transformAppHostHeader
	req.Header.Set(tid.TransactionIDHeader, transactionId)
	req.Header.Set("Content-Type", "application/json")

	resp, err = client.Do(req)

	//this happens when hostname cannot be resolved or host is not accessible
	if err !=nil {
		h.log.ErrorEvent(h.serviceConfig.transformAppName, req.URL.String(), req.Header.Get(tid.TransactionIDHeader), err, uuid)
		return false, nil
	}

	if resp.StatusCode != http.StatusOK {

		w.WriteHeader(http.StatusNotFound);
		h.log.RequestFailedEvent(h.serviceConfig.transformAppName, req.URL.String(), resp, uuid)
		return false, nil
	}

	h.log.ResponseEvent(h.serviceConfig.transformAppName, req.URL.String(), resp, uuid)
	return true, resp
}

<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strconv"
	"time"

	"github.com/dustin/go-humanize"
)

func serverInfo(parts []string, w http.ResponseWriter, req *http.Request) {
	sinfo := map[string]string{
		"seriesly": "Why so series?", "version": "seriesly 0.0",
	}
	mustEncode(200, w, sinfo)
}

func listDatabases(parts []string, w http.ResponseWriter, req *http.Request) {
	mustEncode(200, w, dblist(*dbRoot))
}

func notImplemented(parts []string, w http.ResponseWriter, req *http.Request) {
	emitError(501, w, "not implemented", "TODO")
}

func createDB(parts []string, w http.ResponseWriter, req *http.Request) {
	path := dbPath(parts[0])
	err := dbcreate(path)
	if err == nil {
		w.WriteHeader(201)
	} else {
		emitError(500, w, "Server Error", err.Error())
	}
}

func checkDB(args []string, w http.ResponseWriter, req *http.Request) {
	dbname := args[0]
	if db, err := dbopen(dbname); err == nil {
		db.Close()
		w.WriteHeader(200)
	} else {
		w.WriteHeader(404)
	}
}

func newDocument(args []string, w http.ResponseWriter, req *http.Request) {
	var k, fk string
	form, err := url.ParseQuery(req.URL.RawQuery)
	if err == nil {
		fk = form.Get("ts")
	}

	if fk == "" {
		k = time.Now().UTC().Format(time.RFC3339Nano)
	} else {
		t, err := parseTime(fk)
		if err != nil {
			emitError(400, w, "Bad time format", err.Error())
			return
		}
		k = t.UTC().Format(time.RFC3339Nano)
	}
	putDocument([]string{args[0], k}, w, req)
}

func putDocument(args []string, w http.ResponseWriter, req *http.Request) {
	dbname := args[0]
	k := args[1]
	defer req.Body.Close()
	body, err := ioutil.ReadAll(req.Body)
	if err != nil {
		emitError(400, w, "Bad Request",
			fmt.Sprintf("Error reading body: %v", err))
		return
	}

	err = dbstore(dbname, k, body)

	if err == nil {
		w.WriteHeader(201)
	} else {
		emitError(500, w, "Error storing data", err.Error())
	}
}

func cleanupRangeParam(in, def string) (string, error) {
	if in == "" {
		return def, nil
	}
	t, err := parseTime(in)
	if err != nil {
		return in, err
	}
	return t.UTC().Format(time.RFC3339Nano), nil
}

func query(args []string, w http.ResponseWriter, req *http.Request) {
	// Parse the params

	req.ParseForm()

	group, err := strconv.Atoi(req.FormValue("group"))
	if err != nil {
		emitError(400, w, "Bad group value", err.Error())
		return
	}

	from, err := cleanupRangeParam(req.FormValue("from"), "")
	if err != nil {
		emitError(400, w, "Bad from value: %v", err.Error())
		return
	}
	to, err := cleanupRangeParam(req.FormValue("to"), "")
	if err != nil {
		emitError(400, w, "Bad to value: %v", err.Error())
		return
	}

	ptrs := req.Form["ptr"]
	reds := make([]Reducer, 0, len(ptrs))
	for _, r := range req.Form["reducer"] {
		f, ok := reducers[r]
		if !ok {
			emitError(400, w, "No such reducer", r)
			return
		}
		reds = append(reds, f)
	}
	if len(ptrs) != len(reds) {
		emitError(400, w, "Parameter mismatch",
			"Must supply the same number of pointers and reducers")
		return
	}

	q := executeQuery(args[0], from, to, group, ptrs, reds)
	defer close(q.out)
	defer close(q.cherr)

	// Open the DB and do the work.

	output := map[string]interface{}{}
	going := true
	finished := int32(0)

	for going || (q.started-finished) > 0 {
		select {
		case po := <-q.out:
			finished++
			if po.err != nil {
				err = po.err
			}
			output[strconv.FormatInt(po.key/1e6, 10)] = po.value
		case err = <-q.cherr:
			going = false
		}
	}

	log.Printf("Completed query processing in %v, %v keys, %v chunks",
		time.Since(q.start), humanize.Comma(int64(q.totalKeys)),
		humanize.Comma(int64(q.started)))

	if err != nil {
		log.Printf("Error processing query: %v", err)
		emitError(500, w, "Error traversing DB", err.Error())
	} else {
		e := json.NewEncoder(w)
		err := e.Encode(output)
		if err != nil {
			emitError(500, w, "Error encoding output", err.Error())
		}
	}
}

func deleteDB(parts []string, w http.ResponseWriter, req *http.Request) {
	err := dbdelete(parts[0])
	if err == nil {
		mustEncode(200, w, map[string]interface{}{"ok": true})
	} else {
		emitError(500, w, "Error deleting DB", err.Error())
	}
}

// TODO:

func dbInfo(parts []string, w http.ResponseWriter, req *http.Request) {
	notImplemented(parts, w, req)
}

func dbChanges(parts []string, w http.ResponseWriter, req *http.Request) {
	notImplemented(parts, w, req)
}

func getDocument(parts []string, w http.ResponseWriter, req *http.Request) {
	notImplemented(parts, w, req)
}

func rmDocument(parts []string, w http.ResponseWriter, req *http.Request) {
	notImplemented(parts, w, req)
}
<commit_msg>Compress query results if allowed.<commit_after>package main

import (
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/dustin/go-humanize"
)

func serverInfo(parts []string, w http.ResponseWriter, req *http.Request) {
	sinfo := map[string]string{
		"seriesly": "Why so series?", "version": "seriesly 0.0",
	}
	mustEncode(200, w, sinfo)
}

func listDatabases(parts []string, w http.ResponseWriter, req *http.Request) {
	mustEncode(200, w, dblist(*dbRoot))
}

func notImplemented(parts []string, w http.ResponseWriter, req *http.Request) {
	emitError(501, w, "not implemented", "TODO")
}

func createDB(parts []string, w http.ResponseWriter, req *http.Request) {
	path := dbPath(parts[0])
	err := dbcreate(path)
	if err == nil {
		w.WriteHeader(201)
	} else {
		emitError(500, w, "Server Error", err.Error())
	}
}

func checkDB(args []string, w http.ResponseWriter, req *http.Request) {
	dbname := args[0]
	if db, err := dbopen(dbname); err == nil {
		db.Close()
		w.WriteHeader(200)
	} else {
		w.WriteHeader(404)
	}
}

func newDocument(args []string, w http.ResponseWriter, req *http.Request) {
	var k, fk string
	form, err := url.ParseQuery(req.URL.RawQuery)
	if err == nil {
		fk = form.Get("ts")
	}

	if fk == "" {
		k = time.Now().UTC().Format(time.RFC3339Nano)
	} else {
		t, err := parseTime(fk)
		if err != nil {
			emitError(400, w, "Bad time format", err.Error())
			return
		}
		k = t.UTC().Format(time.RFC3339Nano)
	}
	putDocument([]string{args[0], k}, w, req)
}

func putDocument(args []string, w http.ResponseWriter, req *http.Request) {
	dbname := args[0]
	k := args[1]
	defer req.Body.Close()
	body, err := ioutil.ReadAll(req.Body)
	if err != nil {
		emitError(400, w, "Bad Request",
			fmt.Sprintf("Error reading body: %v", err))
		return
	}

	err = dbstore(dbname, k, body)

	if err == nil {
		w.WriteHeader(201)
	} else {
		emitError(500, w, "Error storing data", err.Error())
	}
}

func cleanupRangeParam(in, def string) (string, error) {
	if in == "" {
		return def, nil
	}
	t, err := parseTime(in)
	if err != nil {
		return in, err
	}
	return t.UTC().Format(time.RFC3339Nano), nil
}

func canGzip(req *http.Request) bool {
	acceptable := req.Header.Get("accept-encoding")
	return strings.Contains(acceptable, "gzip")
}

func query(args []string, w http.ResponseWriter, req *http.Request) {
	// Parse the params

	req.ParseForm()

	group, err := strconv.Atoi(req.FormValue("group"))
	if err != nil {
		emitError(400, w, "Bad group value", err.Error())
		return
	}

	from, err := cleanupRangeParam(req.FormValue("from"), "")
	if err != nil {
		emitError(400, w, "Bad from value: %v", err.Error())
		return
	}
	to, err := cleanupRangeParam(req.FormValue("to"), "")
	if err != nil {
		emitError(400, w, "Bad to value: %v", err.Error())
		return
	}

	ptrs := req.Form["ptr"]
	reds := make([]Reducer, 0, len(ptrs))
	for _, r := range req.Form["reducer"] {
		f, ok := reducers[r]
		if !ok {
			emitError(400, w, "No such reducer", r)
			return
		}
		reds = append(reds, f)
	}
	if len(ptrs) != len(reds) {
		emitError(400, w, "Parameter mismatch",
			"Must supply the same number of pointers and reducers")
		return
	}

	q := executeQuery(args[0], from, to, group, ptrs, reds)
	defer close(q.out)
	defer close(q.cherr)

	// Open the DB and do the work.

	output := map[string]interface{}{}
	going := true
	finished := int32(0)

	for going || (q.started-finished) > 0 {
		select {
		case po := <-q.out:
			finished++
			if po.err != nil {
				err = po.err
			}
			output[strconv.FormatInt(po.key/1e6, 10)] = po.value
		case err = <-q.cherr:
			going = false
		}
	}

	log.Printf("Completed query processing in %v, %v keys, %v chunks",
		time.Since(q.start), humanize.Comma(int64(q.totalKeys)),
		humanize.Comma(int64(q.started)))

	if err != nil {
		log.Printf("Error processing query: %v", err)
		emitError(500, w, "Error traversing DB", err.Error())
	} else {
		z := canGzip(req)

		w.Header().Set("Content-type", "text/html")
		if z {
			w.Header().Set("Content-Encoding", "gzip")
		}
		w.WriteHeader(200)

		var e *json.Encoder

		if z {
			gz := gzip.NewWriter(w)
			defer gz.Close()

			e = json.NewEncoder(gz)
		} else {
			e = json.NewEncoder(w)
		}

		err := e.Encode(output)
		if err != nil {
			emitError(500, w, "Error encoding output", err.Error())
		}
	}
}

func deleteDB(parts []string, w http.ResponseWriter, req *http.Request) {
	err := dbdelete(parts[0])
	if err == nil {
		mustEncode(200, w, map[string]interface{}{"ok": true})
	} else {
		emitError(500, w, "Error deleting DB", err.Error())
	}
}

// TODO:

func dbInfo(parts []string, w http.ResponseWriter, req *http.Request) {
	notImplemented(parts, w, req)
}

func dbChanges(parts []string, w http.ResponseWriter, req *http.Request) {
	notImplemented(parts, w, req)
}

func getDocument(parts []string, w http.ResponseWriter, req *http.Request) {
	notImplemented(parts, w, req)
}

func rmDocument(parts []string, w http.ResponseWriter, req *http.Request) {
	notImplemented(parts, w, req)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"log"
	"strings"
	"sync"
	"time"

	"github.com/inominate/apicache"
)

// Prototype for page specific handlers.
type APIHandler func(url string, params map[string]string) *apicache.Response

// Bug Correcting Handler
// API occasionally returns 221s for no reason, retry automatically when we
// run into one of them.
func defaultHandler(url string, params map[string]string) *apicache.Response {
	resp, err := APIReq(url, params)

	// :ccp: 221's come up for no apparent reason and need to be ignored
	if err == nil && resp.Error.ErrorCode == 221 {
		params["force"] = "true"

		for i := 0; i < conf.Retries; i++ {
			resp, err = APIReq(url, params)
			if resp.Error.ErrorCode != 221 || err != nil {
				break
			}
			time.Sleep(2 * time.Second)
		}
		if resp.Error.ErrorCode == 221 {
			keyid, _ := params["keyid"]
			log.Printf("Failed to recover from 221 at %s for keyid %s: %s", url, keyid, resp.Error)
		}
	}
	if err != nil {
		debugLog.Printf("API Error %s: %s - %+v", err, url, params)
	}
	return resp
}

// Defines valid API pages and what special handler they should use.
// nil handlers will attempt to use defaultHandler which is a straight
// passthrough.
var validPages = map[string]APIHandler{
	//	"/control/":                             controlHandler,
	"/account/accountstatus.xml.aspx":       nil,
	"/account/apikeyinfo.xml.aspx":          nil,
	"/account/characters.xml.aspx":          nil,
	"/char/accountbalance.xml.aspx":         nil,
	"/char/assetlist.xml.aspx":              nil,
	"/char/calendareventattendees.xml.aspx": nil,
	"/char/charactersheet.xml.aspx":         nil,
	"/char/contactlist.xml.aspx":            nil,
	"/char/contactnotifications.xml.aspx":   nil,
	"/char/contracts.xml.aspx":              nil,
	"/char/contractitems.xml.aspx":          nil,
	"/char/contractbids.xml.aspx":           nil,
	"/char/facwarstats.xml.aspx":            nil,
	"/char/industryjobs.xml.aspx":           nil,
	"/char/killlog.xml.aspx":                nil,
	"/char/locations.xml.aspx":              idsListHandler,
	"/char/mailbodies.xml.aspx":             idsListHandler,
	"/char/mailinglists.xml.aspx":           nil,
	"/char/mailmessages.xml.aspx":           nil,
	"/char/marketorders.xml.aspx":           nil,
	"/char/medals.xml.aspx":                 nil,
	"/char/notifications.xml.aspx":          nil,
	"/char/notificationtexts.xml.aspx":      idsListHandler,
	"/char/research.xml.aspx":               nil,
	"/char/skillintraining.xml.aspx":        nil,
	"/char/skillqueue.xml.aspx":             nil,
	"/char/standings.xml.aspx":              nil,
	"/char/upcomingcalendarevents.xml.aspx": nil,
	"/char/walletjournal.xml.aspx":          nil,
	"/char/wallettransactions.xml.aspx":     nil,
	"/corp/accountbalance.xml.aspx":         nil,
	"/corp/assetlist.xml.aspx":              nil,
	"/corp/contactlist.xml.aspx":            nil,
	"/corp/containerlog.xml.aspx":           nil,
	"/corp/contracts.xml.aspx":              nil,
	"/corp/contractitems.xml.aspx":          nil,
	"/corp/contractbids.xml.aspx":           nil,
	"/corp/corporationsheet.xml.aspx":       nil,
	"/corp/facwarstats.xml.aspx":            nil,
	"/corp/industryjobs.xml.aspx":           nil,
	"/corp/killlog.xml.aspx":                nil,
	"/corp/locations.xml.aspx":              idsListHandler,
	"/corp/marketorders.xml.aspx":           nil,
	"/corp/medals.xml.aspx":                 nil,
	"/corp/membermedals.xml.aspx":           nil,
	"/corp/membersecurity.xml.aspx":         nil,
	"/corp/membersecuritylog.xml.aspx":      nil,
	"/corp/membertracking.xml.aspx":         nil,
	"/corp/outpostlist.xml.aspx":            nil,
	"/corp/outpostservicedetail.xml.aspx":   nil,
	"/corp/shareholders.xml.aspx":           nil,
	"/corp/standings.xml.aspx":              nil,
	"/corp/starbasedetail.xml.aspx":         nil,
	"/corp/starbaselist.xml.aspx":           nil,
	"/corp/titles.xml.aspx":                 nil,
	"/corp/walletjournal.xml.aspx":          nil,
	"/corp/wallettransactions.xml.aspx":     nil,
	"/eve/alliancelist.xml.aspx":            nil,
	"/eve/certificatetree.xml.aspx":         nil,
	"/eve/characterid.xml.aspx":             nil,
	"/eve/characterinfo.xml.aspx":           nil,
	"/eve/charactername.xml.aspx":           nil,
	"/eve/conquerablestationlist.xml.aspx":  nil,
	"/eve/errorlist.xml.aspx":               nil,
	"/eve/facwarstats.xml.aspx":             nil,
	"/eve/facwartopstats.xml.aspx":          nil,
	"/eve/reftypes.xml.aspx":                nil,
	"/eve/skilltree.xml.aspx":               nil,
	"/eve/typename.xml.aspx":                nil,
	"/map/facwarsystems.xml.aspx":           nil,
	"/map/jumps.xml.aspx":                   nil,
	"/map/kills.xml.aspx":                   nil,
	"/map/sovereignty.xml.aspx":             nil,
	"/map/sovereigntystatus.xml.aspx":       nil,
	"/server/serverstatus.xml.aspx":         nil,
	"/api/calllist.xml.aspx":                nil,
}

/*
Note that this is a best-attempt number only, actual error count can go
significantly higher as massed concurrent requests run. This isn't to prevent
errors being sent to the API so much as to prevent things from getting out of
control in response to a pathlogical request.
*/
const maxIDErrors = 16

// Bug Correcting Handler for endpoints using comma separated ID lists which
// will fail entirely in case of a single invalid ID.
//
// Note: Can generate many errors so should only be used with applications
// that know to behave themselves. Add a form value of fix with any content
// to enable the correction.
func idsListHandler(url string, params map[string]string) *apicache.Response {
	var runFixer bool
	if _, ok := params["fix"]; ok {
		delete(params, "fix")
		runFixer = true
	}

	resp, err := APIReq(url, params)
	if err != nil {
		debugLog.Printf("API Error %s: %s - %+v", err, url, params)
	}
	if !runFixer {
		return resp
	}

	var ids []string
	if idsParam, ok := params["ids"]; ok {
		ids = strings.Split(idsParam, ",")
	}

	// If we have no ids or just one, we're not doing anything special.
	// If there's more than 250 ids, that's beyond the API limit so we won't
	// touch that either.
	if len(ids) == 0 || len(ids) == 1 || len(ids) > 250 {
		return resp
	}
	// If the request didn't have an invalid id, errorcode 135, there's nothing
	// we can do to help.
	if resp.Error.ErrorCode != 135 {
		return resp
	}

	// If we got this far there's more than one ID, at least one of which is
	// invalid.
	debugLog.Printf("idsListHandler going into action for %d ids: %s", len(ids), params["ids"])

	var errCount errCount
	delete(params, "ids")

	validIDs, err := findValidIDs(url, params, ids, &errCount)
	if err != nil {
		debugLog.Printf("findValidIDs failed: %s", err)
		return resp
	}

	idsBuf := &bytes.Buffer{}
	fmt.Fprintf(idsBuf, "%s", validIDs[0])
	for i := 1; i < len(validIDs); i++ {
		fmt.Fprintf(idsBuf, ",%s", validIDs[i])
	}
	idsParam := idsBuf.String()
	params["ids"] = idsParam

	resp, err = APIReq(url, params)
	if err != nil {
		debugLog.Printf("API Error %s: %s - %+v", err, url, params)
	}
	debugLog.Printf("Completed with: %d errors.", errCount.Get())
	return resp
}

type errCount struct {
	count int
	sync.Mutex
}

func (e *errCount) Get() int {
	e.Lock()
	defer e.Unlock()

	count := e.count
	return count
}

func (e *errCount) Add() int {
	e.Lock()
	defer e.Unlock()

	e.count++
	count := e.count
	return count
}

func findValidIDs(url string, params map[string]string, ids []string, errCount *errCount) ([]string, error) {
	if false && len(ids) == 1 {
		valid, err := isValidIDList(url, params, ids, errCount)
		if valid {
			return ids, err
		} else {
			return nil, err
		}
	}

	if count := errCount.Get(); count >= maxIDErrors {
		return nil, fmt.Errorf("failed to get ids, hit %d errors limit", count)
	}

	var leftIDs, rightIDs []string
	var leftErr, rightErr error

	left := ids[0 : len(ids)/2]
	leftValid, leftErr := isValidIDList(url, params, left, errCount)
	if leftErr != nil {
		return nil, leftErr
	}
	if leftValid {
		leftIDs = left
	} else {
		if len(left) > 1 {
			leftIDs, leftErr = findValidIDs(url, params, left, errCount)
			if rightErr != nil {
				return nil, leftErr
			}
		}
	}

	right := ids[len(ids)/2:]
	rightValid, rightErr := isValidIDList(url, params, right, errCount)
	if rightErr != nil {
		return nil, rightErr
	}
	if rightValid {
		rightIDs = right
	} else {
		if len(right) > 1 {
			rightIDs, rightErr = findValidIDs(url, params, right, errCount)
			if rightErr != nil {
				return nil, rightErr
			}
		}
	}

	validIDs := append(leftIDs, rightIDs...)
	return validIDs, nil
}

func isValidIDList(url string, params map[string]string, ids []string, errCount *errCount) (bool, error) {
	if count := errCount.Get(); count >= maxIDErrors {
		return false, fmt.Errorf("failed to get ids, hit %d errors limit", count)
	}

	idsBuf := &bytes.Buffer{}
	fmt.Fprintf(idsBuf, "%s", ids[0])
	for i := 1; i < len(ids); i++ {
		fmt.Fprintf(idsBuf, ",%s", ids[i])
	}
	idsParam := idsBuf.String()

	var newParams = make(map[string]string)
	for k, v := range params {
		newParams[k] = v
	}
	newParams["ids"] = idsParam

	resp, err := APIReq(url, newParams)
	// Bail completely if the API itself fails for any reason.
	if err != nil {
		return false, err
	}
	// If there is no error then this batch is okay.
	if resp.Error.ErrorCode == 0 {
		return true, nil
	}
	// Bail if we got a non-api failure error other than invalid ID
	if resp.Error.ErrorCode != 135 {
		return false, resp.Error
	}

	debugLog.Printf("Adding Error %d for: %v", errCount.Get(), ids)
	errCount.Add()

	return false, nil
}
<commit_msg>log successful recoveries<commit_after>package main

import (
	"bytes"
	"fmt"
	"log"
	"strings"
	"sync"
	"time"

	"github.com/inominate/apicache"
)

// Prototype for page specific handlers.
type APIHandler func(url string, params map[string]string) *apicache.Response

// Bug Correcting Handler
// API occasionally returns 221s for no reason, retry automatically when we
// run into one of them.
func defaultHandler(url string, params map[string]string) *apicache.Response {
	resp, err := APIReq(url, params)

	// :ccp: 221's come up for no apparent reason and need to be ignored
	if err == nil && resp.Error.ErrorCode == 221 {
		params["force"] = "true"

		for i := 0; i < conf.Retries; i++ {
			resp, err = APIReq(url, params)
			if resp.Error.ErrorCode != 221 || err != nil {
				break
			}
			time.Sleep(2 * time.Second)
		}
		if resp.Error.ErrorCode == 221 {
			keyid, _ := params["keyid"]
			log.Printf("Failed to recover from 221 at %s for keyid %s: %s", url, keyid, resp.Error)
		} else if err == nil {
			log.Printf("Successfully recovered from 221 at %s for keyid %s: %s", url, params["keyid"], resp.Error)
		}
	}
	if err != nil {
		debugLog.Printf("API Error %s: %s - %+v", err, url, params)
	}
	return resp
}

// Defines valid API pages and what special handler they should use.
// nil handlers will attempt to use defaultHandler which is a straight
// passthrough.
var validPages = map[string]APIHandler{
	//	"/control/":                             controlHandler,
	"/account/accountstatus.xml.aspx":       nil,
	"/account/apikeyinfo.xml.aspx":          nil,
	"/account/characters.xml.aspx":          nil,
	"/char/accountbalance.xml.aspx":         nil,
	"/char/assetlist.xml.aspx":              nil,
	"/char/calendareventattendees.xml.aspx": nil,
	"/char/charactersheet.xml.aspx":         nil,
	"/char/contactlist.xml.aspx":            nil,
	"/char/contactnotifications.xml.aspx":   nil,
	"/char/contracts.xml.aspx":              nil,
	"/char/contractitems.xml.aspx":          nil,
	"/char/contractbids.xml.aspx":           nil,
	"/char/facwarstats.xml.aspx":            nil,
	"/char/industryjobs.xml.aspx":           nil,
	"/char/killlog.xml.aspx":                nil,
	"/char/locations.xml.aspx":              idsListHandler,
	"/char/mailbodies.xml.aspx":             idsListHandler,
	"/char/mailinglists.xml.aspx":           nil,
	"/char/mailmessages.xml.aspx":           nil,
	"/char/marketorders.xml.aspx":           nil,
	"/char/medals.xml.aspx":                 nil,
	"/char/notifications.xml.aspx":          nil,
	"/char/notificationtexts.xml.aspx":      idsListHandler,
	"/char/research.xml.aspx":               nil,
	"/char/skillintraining.xml.aspx":        nil,
	"/char/skillqueue.xml.aspx":             nil,
	"/char/standings.xml.aspx":              nil,
	"/char/upcomingcalendarevents.xml.aspx": nil,
	"/char/walletjournal.xml.aspx":          nil,
	"/char/wallettransactions.xml.aspx":     nil,
	"/corp/accountbalance.xml.aspx":         nil,
	"/corp/assetlist.xml.aspx":              nil,
	"/corp/contactlist.xml.aspx":            nil,
	"/corp/containerlog.xml.aspx":           nil,
	"/corp/contracts.xml.aspx":              nil,
	"/corp/contractitems.xml.aspx":          nil,
	"/corp/contractbids.xml.aspx":           nil,
	"/corp/corporationsheet.xml.aspx":       nil,
	"/corp/facwarstats.xml.aspx":            nil,
	"/corp/industryjobs.xml.aspx":           nil,
	"/corp/killlog.xml.aspx":                nil,
	"/corp/locations.xml.aspx":              idsListHandler,
	"/corp/marketorders.xml.aspx":           nil,
	"/corp/medals.xml.aspx":                 nil,
	"/corp/membermedals.xml.aspx":           nil,
	"/corp/membersecurity.xml.aspx":         nil,
	"/corp/membersecuritylog.xml.aspx":      nil,
	"/corp/membertracking.xml.aspx":         nil,
	"/corp/outpostlist.xml.aspx":            nil,
	"/corp/outpostservicedetail.xml.aspx":   nil,
	"/corp/shareholders.xml.aspx":           nil,
	"/corp/standings.xml.aspx":              nil,
	"/corp/starbasedetail.xml.aspx":         nil,
	"/corp/starbaselist.xml.aspx":           nil,
	"/corp/titles.xml.aspx":                 nil,
	"/corp/walletjournal.xml.aspx":          nil,
	"/corp/wallettransactions.xml.aspx":     nil,
	"/eve/alliancelist.xml.aspx":            nil,
	"/eve/certificatetree.xml.aspx":         nil,
	"/eve/characterid.xml.aspx":             nil,
	"/eve/characterinfo.xml.aspx":           nil,
	"/eve/charactername.xml.aspx":           nil,
	"/eve/conquerablestationlist.xml.aspx":  nil,
	"/eve/errorlist.xml.aspx":               nil,
	"/eve/facwarstats.xml.aspx":             nil,
	"/eve/facwartopstats.xml.aspx":          nil,
	"/eve/reftypes.xml.aspx":                nil,
	"/eve/skilltree.xml.aspx":               nil,
	"/eve/typename.xml.aspx":                nil,
	"/map/facwarsystems.xml.aspx":           nil,
	"/map/jumps.xml.aspx":                   nil,
	"/map/kills.xml.aspx":                   nil,
	"/map/sovereignty.xml.aspx":             nil,
	"/map/sovereigntystatus.xml.aspx":       nil,
	"/server/serverstatus.xml.aspx":         nil,
	"/api/calllist.xml.aspx":                nil,
}

/*
Note that this is a best-attempt number only, actual error count can go
significantly higher as massed concurrent requests run. This isn't to prevent
errors being sent to the API so much as to prevent things from getting out of
control in response to a pathlogical request.
*/
const maxIDErrors = 16

// Bug Correcting Handler for endpoints using comma separated ID lists which
// will fail entirely in case of a single invalid ID.
//
// Note: Can generate many errors so should only be used with applications
// that know to behave themselves. Add a form value of fix with any content
// to enable the correction.
func idsListHandler(url string, params map[string]string) *apicache.Response {
	var runFixer bool
	if _, ok := params["fix"]; ok {
		delete(params, "fix")
		runFixer = true
	}

	resp, err := APIReq(url, params)
	if err != nil {
		debugLog.Printf("API Error %s: %s - %+v", err, url, params)
	}
	if !runFixer {
		return resp
	}

	var ids []string
	if idsParam, ok := params["ids"]; ok {
		ids = strings.Split(idsParam, ",")
	}

	// If we have no ids or just one, we're not doing anything special.
	// If there's more than 250 ids, that's beyond the API limit so we won't
	// touch that either.
	if len(ids) == 0 || len(ids) == 1 || len(ids) > 250 {
		return resp
	}
	// If the request didn't have an invalid id, errorcode 135, there's nothing
	// we can do to help.
	if resp.Error.ErrorCode != 135 {
		return resp
	}

	// If we got this far there's more than one ID, at least one of which is
	// invalid.
	debugLog.Printf("idsListHandler going into action for %d ids: %s", len(ids), params["ids"])

	var errCount errCount
	delete(params, "ids")

	validIDs, err := findValidIDs(url, params, ids, &errCount)
	if err != nil {
		debugLog.Printf("findValidIDs failed: %s", err)
		return resp
	}

	idsBuf := &bytes.Buffer{}
	fmt.Fprintf(idsBuf, "%s", validIDs[0])
	for i := 1; i < len(validIDs); i++ {
		fmt.Fprintf(idsBuf, ",%s", validIDs[i])
	}
	idsParam := idsBuf.String()
	params["ids"] = idsParam

	resp, err = APIReq(url, params)
	if err != nil {
		debugLog.Printf("API Error %s: %s - %+v", err, url, params)
	}
	debugLog.Printf("Completed with: %d errors.", errCount.Get())
	return resp
}

type errCount struct {
	count int
	sync.Mutex
}

func (e *errCount) Get() int {
	e.Lock()
	defer e.Unlock()

	count := e.count
	return count
}

func (e *errCount) Add() int {
	e.Lock()
	defer e.Unlock()

	e.count++
	count := e.count
	return count
}

func findValidIDs(url string, params map[string]string, ids []string, errCount *errCount) ([]string, error) {
	if false && len(ids) == 1 {
		valid, err := isValidIDList(url, params, ids, errCount)
		if valid {
			return ids, err
		} else {
			return nil, err
		}
	}

	if count := errCount.Get(); count >= maxIDErrors {
		return nil, fmt.Errorf("failed to get ids, hit %d errors limit", count)
	}

	var leftIDs, rightIDs []string
	var leftErr, rightErr error

	left := ids[0 : len(ids)/2]
	leftValid, leftErr := isValidIDList(url, params, left, errCount)
	if leftErr != nil {
		return nil, leftErr
	}
	if leftValid {
		leftIDs = left
	} else {
		if len(left) > 1 {
			leftIDs, leftErr = findValidIDs(url, params, left, errCount)
			if rightErr != nil {
				return nil, leftErr
			}
		}
	}

	right := ids[len(ids)/2:]
	rightValid, rightErr := isValidIDList(url, params, right, errCount)
	if rightErr != nil {
		return nil, rightErr
	}
	if rightValid {
		rightIDs = right
	} else {
		if len(right) > 1 {
			rightIDs, rightErr = findValidIDs(url, params, right, errCount)
			if rightErr != nil {
				return nil, rightErr
			}
		}
	}

	validIDs := append(leftIDs, rightIDs...)
	return validIDs, nil
}

func isValidIDList(url string, params map[string]string, ids []string, errCount *errCount) (bool, error) {
	if count := errCount.Get(); count >= maxIDErrors {
		return false, fmt.Errorf("failed to get ids, hit %d errors limit", count)
	}

	idsBuf := &bytes.Buffer{}
	fmt.Fprintf(idsBuf, "%s", ids[0])
	for i := 1; i < len(ids); i++ {
		fmt.Fprintf(idsBuf, ",%s", ids[i])
	}
	idsParam := idsBuf.String()

	var newParams = make(map[string]string)
	for k, v := range params {
		newParams[k] = v
	}
	newParams["ids"] = idsParam

	resp, err := APIReq(url, newParams)
	// Bail completely if the API itself fails for any reason.
	if err != nil {
		return false, err
	}
	// If there is no error then this batch is okay.
	if resp.Error.ErrorCode == 0 {
		return true, nil
	}
	// Bail if we got a non-api failure error other than invalid ID
	if resp.Error.ErrorCode != 135 {
		return false, resp.Error
	}

	debugLog.Printf("Adding Error %d for: %v", errCount.Get(), ids)
	errCount.Add()

	return false, nil
}
<|endoftext|>


<commit_before>package rest

import (
	"net/http"
	"path/filepath"
)

// Redirect возвращает Handler, который осуществляет постоянное перенаправление
// на указанный в параметрах URL.
func Redirect(url string) Handler {
	return func(c *Context) error {
		http.Redirect(c, c.Request, url, http.StatusMovedPermanently)
		return nil
	}
}

// File отдает на запрос содержимое файла с указанным именем.
func File(name string) Handler {
	return func(c *Context) error {
		http.ServeFile(c, c.Request, name)
		return nil
	}
}

// Files отдает файлы по имени из указанного каталога. Имя файла задается
// в пути в виде последнего именованного параметра.
func Files(dir string) Handler {
	return func(c *Context) error {
		filename := filepath.Join(dir, c.params[len(c.params)-1].Value)
		http.ServeFile(c, c.Request, filename)
		return nil
	}
}

// Data постоянно отдает указанные в параметрах данные в виде ответа на запрос.
func Data(data interface{}, contentType string) Handler {
	return func(c *Context) error {
		c.ContentType = contentType
		return c.Send(data)
	}
}

// NotImplemented возвращает ошибку ErrNotImplemented.
//
// Иногда при разработке руки сразу не доходят до того, чтобы написать
// полноценный обработчик какого нибудь запроса. В этом случае очень выручает
// данная функция, которую можно использовать вместо временной "заплатки".
func NotImplemented(*Context) error {
	return ErrNotImplemented
}
<commit_msg>BasicAuth Handler<commit_after>package rest

import (
	"fmt"
	"net/http"
	"path/filepath"
)

// Redirect возвращает Handler, который осуществляет постоянное перенаправление
// на указанный в параметрах URL.
func Redirect(url string) Handler {
	return func(c *Context) error {
		http.Redirect(c, c.Request, url, http.StatusMovedPermanently)
		return nil
	}
}

// File отдает на запрос содержимое файла с указанным именем.
func File(name string) Handler {
	return func(c *Context) error {
		http.ServeFile(c, c.Request, name)
		return nil
	}
}

// Files отдает файлы по имени из указанного каталога. Имя файла задается
// в пути в виде последнего именованного параметра.
func Files(dir string) Handler {
	return func(c *Context) error {
		filename := filepath.Join(dir, c.params[len(c.params)-1].Value)
		http.ServeFile(c, c.Request, filename)
		return nil
	}
}

// Data постоянно отдает указанные в параметрах данные в виде ответа на запрос.
func Data(data interface{}, contentType string) Handler {
	return func(c *Context) error {
		c.ContentType = contentType
		return c.Send(data)
	}
}

// NotImplemented возвращает ошибку ErrNotImplemented.
//
// Иногда при разработке руки сразу не доходят до того, чтобы написать
// полноценный обработчик какого нибудь запроса. В этом случае очень выручает
// данная функция, которую можно использовать вместо временной "заплатки".
func NotImplemented(*Context) error {
	return ErrNotImplemented
}

// BasicAuth проверяет HTTP Basic авторизацию пользователя. В качестве
// аргумента передается функция, принимающая значения логина и пароля
// пользователя, и возвращающая true, если пользователь успешно авторизован.
// Вторым параметром передается строка, которая будет использоваться в
// заголовке авторизации для обозначения раздела.
func BasicAuth(auth func(login, password string) bool, realm string) Handler {
	return func(c *Context) error {
		login, password, ok := c.BasicAuth()
		if auth(login, password) {
			return nil
		}
		if ok {
			return c.Send(ErrForbidden)
		}
		if realm == "" {
			realm = "Restricted"
		}
		c.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", realm))
		return c.Send(ErrUnauthorized)
	}
}
<|endoftext|>


<commit_before>package hbot

import (
	"fmt"
	"net"
	"time"
	"bufio"
	"crypto/tls"

	"io"
	"os"
)

type IrcCon struct {
	// Channel for user to read incoming messages
	Incoming chan *Message

	// Map of irc channels this bot is joined to
	Channels map[string]*IrcChannel

	//Server password (optional) only used if set
	Password string

	// SSL
	UseSSL bool

	con net.Conn
	outgoing chan string
	tr []*Trigger

	// This bots nick
	nick string

	// Unix domain socket address for reconnects (linux only)
	unixastr string

	// Whether or not this is a reconnect instance
	reconnect bool
}

// Connect to an irc server
func NewIrcConnection(host, nick string, ssl bool) (*IrcCon, error) {
	irc := new(IrcCon)

	irc.Incoming = make(chan *Message, 16)
	irc.outgoing = make(chan string, 16)
	irc.Channels = make(map[string]*IrcChannel)
	irc.nick = nick
	irc.unixastr = fmt.Sprintf("@%s/irc", nick)
	irc.UseSSL = ssl

	// Attempt reconnection
	if !irc.HijackSession() {
		err := irc.Connect(host)
		if err != nil {
			return nil,err
		}
	}

	irc.AddTrigger(pingPong)
	return irc, nil
}

func (irc *IrcCon) Connect(host string) (err error) {
	if irc.UseSSL {
		irc.con,err = tls.Dial("tcp", host, &tls.Config{})
	} else {
		irc.con,err = net.Dial("tcp", host)
	}
	return
}

// Incoming message gathering routine
func (irc *IrcCon) handleIncomingMessages() {
	io.Copy(os.Stdout, irc.con)
	scan := bufio.NewScanner(irc.con)
	for scan.Scan() {
		mes := ParseMessage(scan.Text())
		consumed := false
		for _,t := range irc.tr {
			if t.Condition(mes) {
				consumed = t.Action(irc,mes)
			}
			if consumed {
				break
			}
		}
		if !consumed {
			irc.Incoming <- mes
		}
	}
}

// Handles message speed throtling
func (irc *IrcCon) handleOutgoingMessages() {
	for s := range irc.outgoing {
		_,err := fmt.Fprint(irc.con, s + "\r\n")
		if err != nil {
			panic(err)
		}
		time.Sleep(time.Millisecond * 200)
	}
}

// Start up servers various running methods
func (irc *IrcCon) Start() {
	go irc.handleIncomingMessages()
	go irc.handleOutgoingMessages()

	go irc.StartUnixListener()

	// Only register on an initial connection
	if !irc.reconnect {
		//Server registration
		if irc.Password != "" {
			irc.Send("PASS " + irc.Password)
		}
		irc.Send(fmt.Sprintf("USER %s 8 * :%s", irc.nick, irc.nick))
		irc.Send(fmt.Sprintf("NICK %s", irc.nick))
	}
}

// Send a message to 'who' (user or channel)
func (irc *IrcCon) Msg(who, text string) {
	irc.Send("PRIVMSG " + who + " :" + text)
}

// Send any command to the server
func (irc *IrcCon) Send(command string) {
	irc.outgoing <- command
}

// Used to change users modes in a channel
// operator = "+o" deop = "-o"
// ban = "+b"
func (irc *IrcCon) ChMode(user, channel, mode string) {
	irc.Send("MODE " + channel + " " + mode + " " + user)
}

// Join a channel and register its struct in the IrcCons channel map
func (irc *IrcCon) Join(ch string) *IrcChannel {
	irc.Send("JOIN " + ch)
	ichan := &IrcChannel{Name: ch, con: irc, Counts: make(map[string]int)}

	irc.Channels[ch] = ichan
	ichan.TryLoadStats(ch[1:] + ".stats")
	return ichan
}

func (irc *IrcCon) AddTrigger(t *Trigger) {
	irc.tr = append(irc.tr, t)
}

// A trigger is used to subscribe and react to events on the Irc Server
type Trigger struct {
	// Returns true if this trigger applies to the passed in message
	Condition func (*Message) bool

	// The action to perform if Condition is true
	// return true if the message was 'consumed'
	Action func (*IrcCon,*Message) bool
}

// A trigger to respond to the servers ping pong messages
// If PingPong messages are not responded to, the server assumes the
// client has timed out and will close the connection.
// Note: this is automatically added in the IrcCon constructor
var pingPong = &Trigger{
	func (m *Message) bool {
		return m.Command == "PING"
	},
	func (irc *IrcCon, m *Message) bool {
		irc.Send("PONG :" + m.Content)
		return true
	},
}
<commit_msg>remove bad debug code<commit_after>package hbot

import (
	"fmt"
	"net"
	"time"
	"bufio"
	"crypto/tls"
)

type IrcCon struct {
	// Channel for user to read incoming messages
	Incoming chan *Message

	// Map of irc channels this bot is joined to
	Channels map[string]*IrcChannel

	//Server password (optional) only used if set
	Password string

	// SSL
	UseSSL bool

	con net.Conn
	outgoing chan string
	tr []*Trigger

	// This bots nick
	nick string

	// Unix domain socket address for reconnects (linux only)
	unixastr string

	// Whether or not this is a reconnect instance
	reconnect bool
}

// Connect to an irc server
func NewIrcConnection(host, nick string, ssl bool) (*IrcCon, error) {
	irc := new(IrcCon)

	irc.Incoming = make(chan *Message, 16)
	irc.outgoing = make(chan string, 16)
	irc.Channels = make(map[string]*IrcChannel)
	irc.nick = nick
	irc.unixastr = fmt.Sprintf("@%s/irc", nick)
	irc.UseSSL = ssl

	// Attempt reconnection
	if !irc.HijackSession() {
		err := irc.Connect(host)
		if err != nil {
			return nil,err
		}
	}

	irc.AddTrigger(pingPong)
	return irc, nil
}

func (irc *IrcCon) Connect(host string) (err error) {
	if irc.UseSSL {
		irc.con,err = tls.Dial("tcp", host, &tls.Config{})
	} else {
		irc.con,err = net.Dial("tcp", host)
	}
	return
}

// Incoming message gathering routine
func (irc *IrcCon) handleIncomingMessages() {
	scan := bufio.NewScanner(irc.con)
	for scan.Scan() {
		mes := ParseMessage(scan.Text())
		consumed := false
		for _,t := range irc.tr {
			if t.Condition(mes) {
				consumed = t.Action(irc,mes)
			}
			if consumed {
				break
			}
		}
		if !consumed {
			irc.Incoming <- mes
		}
	}
}

// Handles message speed throtling
func (irc *IrcCon) handleOutgoingMessages() {
	for s := range irc.outgoing {
		_,err := fmt.Fprint(irc.con, s + "\r\n")
		if err != nil {
			panic(err)
		}
		time.Sleep(time.Millisecond * 200)
	}
}

// Start up servers various running methods
func (irc *IrcCon) Start() {
	go irc.handleIncomingMessages()
	go irc.handleOutgoingMessages()

	go irc.StartUnixListener()

	// Only register on an initial connection
	if !irc.reconnect {
		//Server registration
		if irc.Password != "" {
			irc.Send("PASS " + irc.Password)
		}
		irc.Send(fmt.Sprintf("USER %s 8 * :%s", irc.nick, irc.nick))
		irc.Send(fmt.Sprintf("NICK %s", irc.nick))
	}
}

// Send a message to 'who' (user or channel)
func (irc *IrcCon) Msg(who, text string) {
	irc.Send("PRIVMSG " + who + " :" + text)
}

// Send any command to the server
func (irc *IrcCon) Send(command string) {
	irc.outgoing <- command
}

// Used to change users modes in a channel
// operator = "+o" deop = "-o"
// ban = "+b"
func (irc *IrcCon) ChMode(user, channel, mode string) {
	irc.Send("MODE " + channel + " " + mode + " " + user)
}

// Join a channel and register its struct in the IrcCons channel map
func (irc *IrcCon) Join(ch string) *IrcChannel {
	irc.Send("JOIN " + ch)
	ichan := &IrcChannel{Name: ch, con: irc, Counts: make(map[string]int)}

	irc.Channels[ch] = ichan
	ichan.TryLoadStats(ch[1:] + ".stats")
	return ichan
}

func (irc *IrcCon) AddTrigger(t *Trigger) {
	irc.tr = append(irc.tr, t)
}

// A trigger is used to subscribe and react to events on the Irc Server
type Trigger struct {
	// Returns true if this trigger applies to the passed in message
	Condition func (*Message) bool

	// The action to perform if Condition is true
	// return true if the message was 'consumed'
	Action func (*IrcCon,*Message) bool
}

// A trigger to respond to the servers ping pong messages
// If PingPong messages are not responded to, the server assumes the
// client has timed out and will close the connection.
// Note: this is automatically added in the IrcCon constructor
var pingPong = &Trigger{
	func (m *Message) bool {
		return m.Command == "PING"
	},
	func (irc *IrcCon, m *Message) bool {
		irc.Send("PONG :" + m.Content)
		return true
	},
}
<|endoftext|>