<commit_before>package server

// Error represents an error that occured during an IRMA sessions.
type Error struct {
	Type        ErrorType `json:"error"`
	Status      int       `json:"status"`
	Description string    `json:"description"`
}

type ErrorType string

var (
	ErrorInvalidTimestamp          Error = Error{Type: "INVALID_TIMESTAMP", Status: 400, Description: "Timestamp was not an epoch boundary"}
	ErrorIssuingDisabled           Error = Error{Type: "ISSUING_DISABLED", Status: 403, Description: "This server does not support issuing"}
	ErrorMalformedVerifierRequest  Error = Error{Type: "MALFORMED_VERIFIER_REQUEST", Status: 400, Description: "Malformed verification request"}
	ErrorMalformedSignatureRequest Error = Error{Type: "MALFORMED_SIGNATURE_REQUEST", Status: 400, Description: "Malformed signature request"}
	ErrorMalformedIssuerRequest    Error = Error{Type: "MALFORMED_ISSUER_REQUEST", Status: 400, Description: "Malformed issuer request"}
	ErrorUnauthorized              Error = Error{Type: "UNAUTHORIZED", Status: 403, Description: "You are not authorized to issue or verify this attribute"}
	ErrorAttributesWrong           Error = Error{Type: "ATTRIBUTES_WRONG", Status: 400, Description: "Specified attribute(s) do not belong to this credential type or missing attributes"}
	ErrorCannotIssue               Error = Error{Type: "CANNOT_ISSUE", Status: 500, Description: "Cannot issue this credential"}

	ErrorIssuanceFailed       Error = Error{Type: "ISSUANCE_FAILED", Status: 500, Description: "Failed to create credential(s)"}
	ErrorInvalidProofs        Error = Error{Type: "INVALID_PROOFS", Status: 400, Description: "Invalid secret key commitments and/or disclosure proofs"}
	ErrorAttributesMissing    Error = Error{Type: "ATTRIBUTES_MISSING", Status: 400, Description: "Not all requested-for attributes were present"}
	ErrorAttributesExpired    Error = Error{Type: "ATTRIBUTES_EXPIRED", Status: 400, Description: "Disclosed attributes were expired"}
	ErrorUnexpectedRequest    Error = Error{Type: "UNEXPECTED_REQUEST", Status: 403, Description: "Unexpected request in this state"}
	ErrorUnknownPublicKey     Error = Error{Type: "UNKNOWN_PUBLIC_KEY", Status: 403, Description: "Attributes were not valid against a known public key"}
	ErrorKeyshareProofMissing Error = Error{Type: "KEYSHARE_PROOF_MISSING", Status: 403, Description: "ProofP object from a keyshare server missing"}
	ErrorSessionUnknown       Error = Error{Type: "SESSION_UNKNOWN", Status: 400, Description: "Unknown or expired session"}
	ErrorMalformedInput       Error = Error{Type: "MALFORMED_INPUT", Status: 400, Description: "Input could not be parsed"}
	ErrorUnknown              Error = Error{Type: "EXCEPTION", Status: 500, Description: "Encountered unexpected problem"}
	ErrorInternal             Error = Error{Type: "EXCEPTION", Status: 500, Description: "Encountered unexpected problem"}
	ErrorNextSession          Error = Error{Type: "NEXT_SESSION", Status: 500, Description: "Error starting next session"}
	ErrorRevocation           Error = Error{Type: "REVOCATION", Status: 500, Description: "Revocation error"}
	ErrorUnknownRevocationKey Error = Error{Type: "UNKNOWN_REVOCATION_KEY", Status: 404, Description: "No issuance records correspond to the given revocationKey"}

	ErrorUnsupported     Error = Error{Type: "UNSUPPORTED", Status: 501, Description: "Unsupported by this server"}
	ErrorInvalidRequest  Error = Error{Type: "INVALID_REQUEST", Status: 400, Description: "Invalid HTTP request"}
	ErrorProtocolVersion Error = Error{Type: "PROTOCOL_VERSION", Status: 400, Description: "Protocol version negotiation failed"}
)
<commit_msg>Differentiate between ErrorUnknown and ErrorInternal error message.<commit_after>package server

// Error represents an error that occured during an IRMA sessions.
type Error struct {
	Type        ErrorType `json:"error"`
	Status      int       `json:"status"`
	Description string    `json:"description"`
}

type ErrorType string

var (
	ErrorInvalidTimestamp          Error = Error{Type: "INVALID_TIMESTAMP", Status: 400, Description: "Timestamp was not an epoch boundary"}
	ErrorIssuingDisabled           Error = Error{Type: "ISSUING_DISABLED", Status: 403, Description: "This server does not support issuing"}
	ErrorMalformedVerifierRequest  Error = Error{Type: "MALFORMED_VERIFIER_REQUEST", Status: 400, Description: "Malformed verification request"}
	ErrorMalformedSignatureRequest Error = Error{Type: "MALFORMED_SIGNATURE_REQUEST", Status: 400, Description: "Malformed signature request"}
	ErrorMalformedIssuerRequest    Error = Error{Type: "MALFORMED_ISSUER_REQUEST", Status: 400, Description: "Malformed issuer request"}
	ErrorUnauthorized              Error = Error{Type: "UNAUTHORIZED", Status: 403, Description: "You are not authorized to issue or verify this attribute"}
	ErrorAttributesWrong           Error = Error{Type: "ATTRIBUTES_WRONG", Status: 400, Description: "Specified attribute(s) do not belong to this credential type or missing attributes"}
	ErrorCannotIssue               Error = Error{Type: "CANNOT_ISSUE", Status: 500, Description: "Cannot issue this credential"}

	ErrorIssuanceFailed       Error = Error{Type: "ISSUANCE_FAILED", Status: 500, Description: "Failed to create credential(s)"}
	ErrorInvalidProofs        Error = Error{Type: "INVALID_PROOFS", Status: 400, Description: "Invalid secret key commitments and/or disclosure proofs"}
	ErrorAttributesMissing    Error = Error{Type: "ATTRIBUTES_MISSING", Status: 400, Description: "Not all requested-for attributes were present"}
	ErrorAttributesExpired    Error = Error{Type: "ATTRIBUTES_EXPIRED", Status: 400, Description: "Disclosed attributes were expired"}
	ErrorUnexpectedRequest    Error = Error{Type: "UNEXPECTED_REQUEST", Status: 403, Description: "Unexpected request in this state"}
	ErrorUnknownPublicKey     Error = Error{Type: "UNKNOWN_PUBLIC_KEY", Status: 403, Description: "Attributes were not valid against a known public key"}
	ErrorKeyshareProofMissing Error = Error{Type: "KEYSHARE_PROOF_MISSING", Status: 403, Description: "ProofP object from a keyshare server missing"}
	ErrorSessionUnknown       Error = Error{Type: "SESSION_UNKNOWN", Status: 400, Description: "Unknown or expired session"}
	ErrorMalformedInput       Error = Error{Type: "MALFORMED_INPUT", Status: 400, Description: "Input could not be parsed"}
	ErrorUnknown              Error = Error{Type: "EXCEPTION", Status: 500, Description: "Encountered unexpected problem"}
	ErrorInternal             Error = Error{Type: "INTERNAL_ERROR", Status: 500, Description: "Internal server error"}
	ErrorNextSession          Error = Error{Type: "NEXT_SESSION", Status: 500, Description: "Error starting next session"}
	ErrorRevocation           Error = Error{Type: "REVOCATION", Status: 500, Description: "Revocation error"}
	ErrorUnknownRevocationKey Error = Error{Type: "UNKNOWN_REVOCATION_KEY", Status: 404, Description: "No issuance records correspond to the given revocationKey"}

	ErrorUnsupported     Error = Error{Type: "UNSUPPORTED", Status: 501, Description: "Unsupported by this server"}
	ErrorInvalidRequest  Error = Error{Type: "INVALID_REQUEST", Status: 400, Description: "Invalid HTTP request"}
	ErrorProtocolVersion Error = Error{Type: "PROTOCOL_VERSION", Status: 400, Description: "Protocol version negotiation failed"}
)
<|endoftext|>


<commit_before>// Copyright 2012 Apcera Inc. All rights reserved.

package server

import (
	"fmt"
)

type pubArg struct {
	subject []byte
	reply   []byte
	szb     []byte
	size    int
}

type parseState struct {
	state   int
	as      int
	drop    int
	pa      pubArg
	argBuf  []byte
	msgBuf  []byte
	scratch [MAX_CONTROL_LINE_SIZE]byte
}

const (
	OP_START = iota
	OP_C
	OP_CO
	OP_CON
	OP_CONN
	OP_CONNE
	OP_CONNEC
	OP_CONNECT
	CONNECT_ARG
	OP_P
	OP_PU
	OP_PUB
	PUB_ARG
	OP_PI
	OP_PIN
	OP_PING
	MSG_PAYLOAD
	MSG_END
	OP_S
	OP_SU
	OP_SUB
	SUB_ARG
	OP_U
	OP_UN
	OP_UNS
	OP_UNSU
	OP_UNSUB
	UNSUB_ARG
)

func (c *client) parse(buf []byte) error {
	var i int
	var b byte

	c.nr++
	c.nb += len(buf)

	for i, b = range buf {
		switch c.state {
		case OP_START:
			switch b {
			case 'C', 'c':
				c.state = OP_C
			case 'P', 'p':
				c.state = OP_P
			case 'S', 's':
				c.state = OP_S
			case 'U', 'u':
				c.state = OP_U
			default:
				goto parseErr
			}
		case OP_P:
			switch b {
			case 'U', 'u':
				c.state = OP_PU
			case 'I', 'i':
				c.state = OP_PI
			default:
				goto parseErr
			}
		case OP_PU:
			switch b {
			case 'B', 'b':
				c.state = OP_PUB
			default:
				goto parseErr
			}
		case OP_PUB:
			switch b {
			case ' ', '\t':
				continue
			default:
				c.state = PUB_ARG
				c.as = i
			}
		case PUB_ARG:
			switch b {
			case '\r':
				c.drop = 1
			case '\n':
				var arg []byte
				if c.argBuf != nil {
					arg = c.argBuf
				} else {
					arg = buf[c.as : i-c.drop]
				}
				c.drop, c.as, c.state = 0, i+1, MSG_PAYLOAD
				if err := c.processPub(arg); err != nil {
					return err
				}
			default:
				if c.argBuf != nil {
					c.argBuf = append(c.argBuf, b)
				}
			}
		case MSG_PAYLOAD:
			if c.msgBuf != nil {
				if len(c.msgBuf) >= c.pa.size {
					c.processMsg(c.msgBuf)
					c.argBuf, c.msgBuf, c.state = nil, nil, MSG_END
				} else {
					c.msgBuf = append(c.msgBuf, b)
				}
			} else if i-c.as >= c.pa.size {
				c.processMsg(buf[c.as:i])
				c.state = MSG_END
			}
		case MSG_END:
			switch b {
			case '\n':
				c.drop, c.as, c.state = 0, i+1, OP_START
			default:
				continue
			}
		case OP_S:
			switch b {
			case 'U', 'u':
				c.state = OP_SU
			default:
				goto parseErr
			}
		case OP_SU:
			switch b {
			case 'B', 'b':
				c.state = OP_SUB
			default:
				goto parseErr
			}
		case OP_SUB:
			switch b {
			case ' ', '\t':
				continue
			default:
				c.state = SUB_ARG
				c.as = i
			}
		case SUB_ARG:
			switch b {
			case '\r':
				c.drop = 1
			case '\n':
				var arg []byte
				if c.argBuf != nil {
					arg = c.argBuf
					c.argBuf = nil
				} else {
					arg = buf[c.as : i-c.drop]
				}
				if err := c.processSub(arg); err != nil {
					return err
				}
				c.drop, c.as, c.state = 0, i+1, OP_START
			default:
				if c.argBuf != nil {
					c.argBuf = append(c.argBuf, b)
				}
			}
		case OP_U:
			switch b {
			case 'N', 'n':
				c.state = OP_UN
			default:
				goto parseErr
			}
		case OP_UN:
			switch b {
			case 'S', 's':
				c.state = OP_UNS
			default:
				goto parseErr
			}
		case OP_UNS:
			switch b {
			case 'U', 'u':
				c.state = OP_UNSU
			default:
				goto parseErr
			}
		case OP_UNSU:
			switch b {
			case 'B', 'b':
				c.state = OP_UNSUB
			default:
				goto parseErr
			}
		case OP_UNSUB:
			switch b {
			case ' ', '\t':
				continue
			default:
				c.state = UNSUB_ARG
				c.as = i
			}
		case UNSUB_ARG:
			switch b {
			case '\r':
				c.drop = 1
			case '\n':
				var arg []byte
				if c.argBuf != nil {
					arg = c.argBuf
					c.argBuf = nil
				} else {
					arg = buf[c.as : i-c.drop]
				}
				if err := c.processUnsub(arg); err != nil {
					return err
				}
				c.drop, c.as, c.state = 0, i+1, OP_START
			default:
				if c.argBuf != nil {
					c.argBuf = append(c.argBuf, b)
				}
			}
		case OP_PI:
			switch b {
			case 'N', 'n':
				c.state = OP_PIN
			default:
				goto parseErr
			}
		case OP_PIN:
			switch b {
			case 'G', 'g':
				c.state = OP_PING
			default:
				goto parseErr
			}
		case OP_PING:
			switch b {
			case '\n':
				c.processPing()
				c.drop, c.state = 0, OP_START
			}
		case OP_C:
			switch b {
			case 'O', 'o':
				c.state = OP_CO
			default:
				goto parseErr
			}
		case OP_CO:
			switch b {
			case 'N', 'n':
				c.state = OP_CON
			default:
				goto parseErr
			}
		case OP_CON:
			switch b {
			case 'N', 'n':
				c.state = OP_CONN
			default:
				goto parseErr
			}
		case OP_CONN:
			switch b {
			case 'E', 'e':
				c.state = OP_CONNE
			default:
				goto parseErr
			}
		case OP_CONNE:
			switch b {
			case 'C', 'c':
				c.state = OP_CONNEC
			default:
				goto parseErr
			}
		case OP_CONNEC:
			switch b {
			case 'T', 't':
				c.state = OP_CONNECT
			default:
				goto parseErr
			}
		case OP_CONNECT:
			switch b {
			case ' ', '\t':
				continue
			default:
				c.state = CONNECT_ARG
				c.as = i
			}
		case CONNECT_ARG:
			switch b {
			case '\r':
				c.drop = 1
			case '\n':
				if err := c.processConnect(buf[c.as : i-c.drop]); err != nil {
					return err
				}
				c.drop, c.state = 0, OP_START
			}
		default:
			goto parseErr
		}
	}
	// Check for split buffer scenarios for SUB and UNSUB and PUB
	if (c.state == SUB_ARG || c.state == UNSUB_ARG || c.state == PUB_ARG) && c.argBuf == nil {
		c.argBuf = c.scratch[:0]
		c.argBuf = append(c.argBuf, buf[c.as:(i+1)-c.drop]...)
		// FIXME, check max len
	}
	// Check for split msg
	if c.state == MSG_PAYLOAD && c.msgBuf == nil {
		// We need to clone the pubArg if it is still referencing the
		// read buffer and we are not able to process the msg.
		if c.argBuf == nil {
			c.clonePubArg()
		}
		// FIXME: copy better here? Make whole buf if large?
		//c.msgBuf = c.scratch[:0]
		c.msgBuf = c.scratch[len(c.argBuf):len(c.argBuf)]
		c.msgBuf = append(c.msgBuf, (buf[c.as:])...)
	}
	return nil

parseErr:
	return fmt.Errorf("Parse Error [%d]: '%s'", c.state, buf[i:])
}

// clonePubArg is used when the split buffer scenario has the pubArg in the existing read buffer, but
// we need to hold onto it into the next read.
func (c *client) clonePubArg() {
	c.argBuf = c.scratch[:0]
	c.argBuf = append(c.argBuf, c.pa.subject...)
	c.argBuf = append(c.argBuf, c.pa.reply...)
	c.argBuf = append(c.argBuf, c.pa.szb...)
	c.pa.subject = c.argBuf[:len(c.pa.subject)]
	c.pa.reply = c.argBuf[len(c.pa.subject) : len(c.pa.subject)+len(c.pa.reply)]
	c.pa.szb = c.argBuf[len(c.pa.subject)+len(c.pa.reply):]
}
<commit_msg>Add in clone pub args, fix state cleanup<commit_after>// Copyright 2012 Apcera Inc. All rights reserved.

package server

import (
	"fmt"
)

type pubArg struct {
	subject []byte
	reply   []byte
	szb     []byte
	size    int
}

type parseState struct {
	state   int
	as      int
	drop    int
	pa      pubArg
	argBuf  []byte
	msgBuf  []byte
	scratch [MAX_CONTROL_LINE_SIZE]byte
}

const (
	OP_START = iota
	OP_C
	OP_CO
	OP_CON
	OP_CONN
	OP_CONNE
	OP_CONNEC
	OP_CONNECT
	CONNECT_ARG
	OP_P
	OP_PU
	OP_PUB
	PUB_ARG
	OP_PI
	OP_PIN
	OP_PING
	MSG_PAYLOAD
	MSG_END
	OP_S
	OP_SU
	OP_SUB
	SUB_ARG
	OP_U
	OP_UN
	OP_UNS
	OP_UNSU
	OP_UNSUB
	UNSUB_ARG
)

func (c *client) parse(buf []byte) error {
	var i int
	var b byte

	c.nr++
	c.nb += len(buf)

	for i, b = range buf {
		switch c.state {
		case OP_START:
			switch b {
			case 'C', 'c':
				c.state = OP_C
			case 'P', 'p':
				c.state = OP_P
			case 'S', 's':
				c.state = OP_S
			case 'U', 'u':
				c.state = OP_U
			default:
				goto parseErr
			}
		case OP_P:
			switch b {
			case 'U', 'u':
				c.state = OP_PU
			case 'I', 'i':
				c.state = OP_PI
			default:
				goto parseErr
			}
		case OP_PU:
			switch b {
			case 'B', 'b':
				c.state = OP_PUB
			default:
				goto parseErr
			}
		case OP_PUB:
			switch b {
			case ' ', '\t':
				continue
			default:
				c.state = PUB_ARG
				c.as = i
			}
		case PUB_ARG:
			switch b {
			case '\r':
				c.drop = 1
			case '\n':
				var arg []byte
				if c.argBuf != nil {
					arg = c.argBuf
				} else {
					arg = buf[c.as : i-c.drop]
				}
				if err := c.processPub(arg); err != nil {
					return err
				}
				c.drop, c.as, c.state = 0, i+1, MSG_PAYLOAD
			default:
				if c.argBuf != nil {
					c.argBuf = append(c.argBuf, b)
				}
			}
		case MSG_PAYLOAD:
			if c.msgBuf != nil {
				if len(c.msgBuf) >= c.pa.size {
					c.processMsg(c.msgBuf)
					c.argBuf, c.msgBuf, c.state = nil, nil, MSG_END
				} else {
					c.msgBuf = append(c.msgBuf, b)
				}
			} else if i-c.as >= c.pa.size {
				c.processMsg(buf[c.as:i])
				c.argBuf, c.msgBuf, c.state = nil, nil, MSG_END
			}
		case MSG_END:
			switch b {
			case '\n':
				c.drop, c.as, c.state = 0, i+1, OP_START
			default:
				continue
			}
		case OP_S:
			switch b {
			case 'U', 'u':
				c.state = OP_SU
			default:
				goto parseErr
			}
		case OP_SU:
			switch b {
			case 'B', 'b':
				c.state = OP_SUB
			default:
				goto parseErr
			}
		case OP_SUB:
			switch b {
			case ' ', '\t':
				continue
			default:
				c.state = SUB_ARG
				c.as = i
			}
		case SUB_ARG:
			switch b {
			case '\r':
				c.drop = 1
			case '\n':
				var arg []byte
				if c.argBuf != nil {
					arg = c.argBuf
					c.argBuf = nil
				} else {
					arg = buf[c.as : i-c.drop]
				}
				if err := c.processSub(arg); err != nil {
					return err
				}
				c.drop, c.as, c.state = 0, i+1, OP_START
			default:
				if c.argBuf != nil {
					c.argBuf = append(c.argBuf, b)
				}
			}
		case OP_U:
			switch b {
			case 'N', 'n':
				c.state = OP_UN
			default:
				goto parseErr
			}
		case OP_UN:
			switch b {
			case 'S', 's':
				c.state = OP_UNS
			default:
				goto parseErr
			}
		case OP_UNS:
			switch b {
			case 'U', 'u':
				c.state = OP_UNSU
			default:
				goto parseErr
			}
		case OP_UNSU:
			switch b {
			case 'B', 'b':
				c.state = OP_UNSUB
			default:
				goto parseErr
			}
		case OP_UNSUB:
			switch b {
			case ' ', '\t':
				continue
			default:
				c.state = UNSUB_ARG
				c.as = i
			}
		case UNSUB_ARG:
			switch b {
			case '\r':
				c.drop = 1
			case '\n':
				var arg []byte
				if c.argBuf != nil {
					arg = c.argBuf
					c.argBuf = nil
				} else {
					arg = buf[c.as : i-c.drop]
				}
				if err := c.processUnsub(arg); err != nil {
					return err
				}
				c.drop, c.as, c.state = 0, i+1, OP_START
			default:
				if c.argBuf != nil {
					c.argBuf = append(c.argBuf, b)
				}
			}
		case OP_PI:
			switch b {
			case 'N', 'n':
				c.state = OP_PIN
			default:
				goto parseErr
			}
		case OP_PIN:
			switch b {
			case 'G', 'g':
				c.state = OP_PING
			default:
				goto parseErr
			}
		case OP_PING:
			switch b {
			case '\n':
				c.processPing()
				c.drop, c.state = 0, OP_START
			}
		case OP_C:
			switch b {
			case 'O', 'o':
				c.state = OP_CO
			default:
				goto parseErr
			}
		case OP_CO:
			switch b {
			case 'N', 'n':
				c.state = OP_CON
			default:
				goto parseErr
			}
		case OP_CON:
			switch b {
			case 'N', 'n':
				c.state = OP_CONN
			default:
				goto parseErr
			}
		case OP_CONN:
			switch b {
			case 'E', 'e':
				c.state = OP_CONNE
			default:
				goto parseErr
			}
		case OP_CONNE:
			switch b {
			case 'C', 'c':
				c.state = OP_CONNEC
			default:
				goto parseErr
			}
		case OP_CONNEC:
			switch b {
			case 'T', 't':
				c.state = OP_CONNECT
			default:
				goto parseErr
			}
		case OP_CONNECT:
			switch b {
			case ' ', '\t':
				continue
			default:
				c.state = CONNECT_ARG
				c.as = i
			}
		case CONNECT_ARG:
			switch b {
			case '\r':
				c.drop = 1
			case '\n':
				if err := c.processConnect(buf[c.as : i-c.drop]); err != nil {
					return err
				}
				c.drop, c.state = 0, OP_START
			}
		default:
			goto parseErr
		}
	}
	// Check for split buffer scenarios for SUB and UNSUB and PUB
	if (c.state == SUB_ARG || c.state == UNSUB_ARG || c.state == PUB_ARG) && c.argBuf == nil {
		c.argBuf = c.scratch[:0]
		c.argBuf = append(c.argBuf, buf[c.as:(i+1)-c.drop]...)
		// FIXME, check max len
	}
	// Check for split msg
	if c.state == MSG_PAYLOAD && c.msgBuf == nil {
		// We need to clone the pubArg if it is still referencing the
		// read buffer and we are not able to process the msg.
		if c.argBuf == nil {
			c.clonePubArg()
		}
		// FIXME: copy better here? Make whole buf if large?
		//c.msgBuf = c.scratch[:0]
		c.msgBuf = c.scratch[len(c.argBuf):len(c.argBuf)]
		c.msgBuf = append(c.msgBuf, (buf[c.as:])...)
	}
	return nil

parseErr:
	return fmt.Errorf("Parse Error [%d]: '%s'", c.state, buf[i:])
}

// clonePubArg is used when the split buffer scenario has the pubArg in the existing read buffer, but
// we need to hold onto it into the next read.
func (c *client) clonePubArg() {
	c.argBuf = c.scratch[:0]
	c.argBuf = append(c.argBuf, c.pa.subject...)
	c.argBuf = append(c.argBuf, c.pa.reply...)
	c.argBuf = append(c.argBuf, c.pa.szb...)
	c.pa.subject = c.argBuf[:len(c.pa.subject)]
	if c.pa.reply != nil {
		c.pa.reply = c.argBuf[len(c.pa.subject) : len(c.pa.subject)+len(c.pa.reply)]
	}
	c.pa.szb = c.argBuf[len(c.pa.subject)+len(c.pa.reply):]
}
<|endoftext|>


<commit_before>package server

var routerTemplateStr = `
package server

// Code auto-generated. Do not edit.

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"strconv"
	"time"

	// register pprof listener
	_ "net/http/pprof"

	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	opentracing "github.com/opentracing/opentracing-go"
	"gopkg.in/Clever/kayvee-go.v6/logger"
	kvMiddleware "gopkg.in/Clever/kayvee-go.v6/middleware"
	"gopkg.in/tylerb/graceful.v1"
	"github.com/Clever/go-process-metrics/metrics"
	"github.com/kardianos/osext"
	jaeger "github.com/uber/jaeger-client-go"
	jaegercfg "github.com/uber/jaeger-client-go/config"
	"github.com/uber/jaeger-client-go/transport"
)

const (
	// lowerBoundRateLimiter determines the lower bound interval that we sample every operation.
	// https://godoc.org/github.com/uber/jaeger-client-go#GuaranteedThroughputProbabilisticSampler
	lowerBoundRateLimiter = 1.0 / 60 // 1 request/minute/operation
)

type contextKey struct{}

// Server defines a HTTP server that implements the Controller interface.
type Server struct {
	// Handler should generally not be changed. It exposed to make testing easier.
	Handler http.Handler
	addr string
	l logger.KayveeLogger
}

// Serve starts the server. It will return if an error occurs.
func (s *Server) Serve() error {

	go func() {
		metrics.Log("{{.Title}}", 1*time.Minute)
	}()

	go func() {
		// This should never return. Listen on the pprof port
		log.Printf("PProf server crashed: %%s", http.ListenAndServe("localhost:6060", nil))
	}()

	dir, err := osext.ExecutableFolder()
	if err != nil {
		log.Fatal(err)
	}
	if err := logger.SetGlobalRouting(path.Join(dir, "kvconfig.yml")); err != nil {
		s.l.Info("please provide a kvconfig.yml file to enable app log routing")
	}

	tracingToken := os.Getenv("TRACING_ACCESS_TOKEN")
	ingestURL := os.Getenv("TRACING_INGEST_URL")
	isLocal := os.Getenv("_IS_LOCAL") == "true"
	if (tracingToken != "" && ingestURL != "") || isLocal {
		samplingRate := .01 // 1%% of requests

		if samplingRateStr := os.Getenv("TRACING_SAMPLING_RATE_PERCENT"); samplingRateStr != "" {
			samplingRateP, err := strconv.ParseFloat(samplingRateStr, 64)
			if err != nil {
				s.l.ErrorD("tracing-sampling-override-failed", logger.M{
					"msg": fmt.Sprintf("could not parse '%%s' to integer", samplingRateStr),
				})
			} else {
				samplingRate = samplingRateP
			}

			s.l.InfoD("tracing-sampling-rate", logger.M{
				"msg": fmt.Sprintf("sampling rate will be %%.3f", samplingRate),
			})
		}

		sampler, err := jaeger.NewGuaranteedThroughputProbabilisticSampler(lowerBoundRateLimiter, samplingRate)
		if err != nil {
			return fmt.Errorf("failed to build jaeger sampler: %%s", err)
		}

		cfg := &jaegercfg.Configuration{
			ServiceName: os.Getenv("_APP_NAME"),
			Tags:        []opentracing.Tag{
				opentracing.Tag{Key: "app_name", Value: os.Getenv("_APP_NAME")},
				opentracing.Tag{Key: "build_id", Value: os.Getenv("_BUILD_ID")},
				opentracing.Tag{Key: "deploy_env", Value: os.Getenv("_DEPLOY_ENV")},
				opentracing.Tag{Key: "team_owner", Value: os.Getenv("_TEAM_OWNER")},
				opentracing.Tag{Key: "pod_id", Value: os.Getenv("_POD_ID")},
				opentracing.Tag{Key: "pod_shortname", Value: os.Getenv("_POD_SHORTNAME")},
				opentracing.Tag{Key: "pod_account", Value: os.Getenv("_POD_ACCOUNT")},
				opentracing.Tag{Key: "pod_region", Value: os.Getenv("_POD_REGION")},
			},
		}

		var tracer opentracing.Tracer
		var closer io.Closer
		if isLocal {
			// when local, send everything and use the default params for the Jaeger collector
			cfg.Sampler = &jaegercfg.SamplerConfig{
				Type:  "const",
				Param: 1.0,
			}
			tracer, closer, err = cfg.NewTracer()
			s.l.InfoD("local-tracing", logger.M{"msg": "sending traces to default localhost jaeger address"})
		} else {
			// Create a Jaeger HTTP Thrift transport
			transport := transport.NewHTTPTransport(ingestURL, transport.HTTPBasicAuth("auth", tracingToken))
			tracer, closer, err = cfg.NewTracer(
				jaegercfg.Reporter(jaeger.NewRemoteReporter(transport)),
				jaegercfg.Sampler(sampler))
		}
		if err != nil {
			log.Fatalf("Could not initialize jaeger tracer: %%s", err)
		}
		defer closer.Close()

		opentracing.SetGlobalTracer(tracer)
	} else {
		s.l.Error("please set TRACING_ACCESS_TOKEN & TRACING_INGEST_URL to enable tracing")
	}

	s.l.Counter("server-started")

	// Give the sever 30 seconds to shut down
	return graceful.RunWithErr(s.addr,30*time.Second,s.Handler)
}

type handler struct {
	Controller
}

func withMiddleware(serviceName string, router http.Handler, m []func(http.Handler) http.Handler) http.Handler {
	handler := router

	// compress everything
	handler = handlers.CompressHandler(handler)

	// Wrap the middleware in the opposite order specified so that when called then run
	// in the order specified
	for i := len(m) - 1; i >= 0; i-- {
		handler = m[i](handler)
	}
	handler = TracingMiddleware(handler)
	handler = PanicMiddleware(handler)
	// Logging middleware comes last, i.e. will be run first.
	// This makes it so that other middleware has access to the logger
	// that kvMiddleware injects into the request context.
	handler = kvMiddleware.New(handler, serviceName)
	return handler
}

// New returns a Server that implements the Controller interface. It will start when "Serve" is called.
func New(c Controller, addr string) *Server {
	return NewWithMiddleware(c, addr, []func(http.Handler) http.Handler{})
}

// NewRouter returns a mux.Router with no middleware. This is so we can attach additional routes to the
// router if necessary
func NewRouter(c Controller) *mux.Router {
	return newRouter(c)
}

func newRouter(c Controller) *mux.Router {
	router := mux.NewRouter()
	h := handler{Controller: c}

	{{range $index, $val := .Functions}}
	router.Methods("{{$val.Method}}").Path("{{$val.Path}}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		logger.FromContext(r.Context()).AddContext("op", "{{$val.OpID}}")
		h.{{$val.HandlerName}}Handler(r.Context(), w, r)
		ctx := WithTracingOpName(r.Context(), "{{$val.OpID}}")
		r = r.WithContext(ctx)
	})
	{{end}}
	return router
}

// NewWithMiddleware returns a Server that implemenets the Controller interface. It runs the
// middleware after the built-in middleware (e.g. logging), but before the controller methods.
// The middleware is executed in the order specified. The server will start when "Serve" is called.
func NewWithMiddleware(c Controller, addr string, m []func(http.Handler) http.Handler) *Server {
	router := newRouter(c)

	return AttachMiddleware(router, addr, m)
}

// AttachMiddleware attaches the given middleware to the router; this is to be used in conjunction with
// NewServer. It attaches custom middleware passed as arguments as well as the built-in middleware for
// logging, tracing, and handling panics. It should be noted that the built-in middleware executes first
// followed by the passed in middleware (in the order specified).
func AttachMiddleware(router *mux.Router, addr string, m []func(http.Handler) http.Handler) *Server {
	l := logger.New("{{.Title}}")

	handler := withMiddleware("{{.Title}}", router, m)
	return &Server{Handler: handler, addr: addr, l: l}
}`
<commit_msg>Do not log process metrics when running locally<commit_after>package server

var routerTemplateStr = `
package server

// Code auto-generated. Do not edit.

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"strconv"
	"time"

	// register pprof listener
	_ "net/http/pprof"

	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	opentracing "github.com/opentracing/opentracing-go"
	"gopkg.in/Clever/kayvee-go.v6/logger"
	kvMiddleware "gopkg.in/Clever/kayvee-go.v6/middleware"
	"gopkg.in/tylerb/graceful.v1"
	"github.com/Clever/go-process-metrics/metrics"
	"github.com/kardianos/osext"
	jaeger "github.com/uber/jaeger-client-go"
	jaegercfg "github.com/uber/jaeger-client-go/config"
	"github.com/uber/jaeger-client-go/transport"
)

const (
	// lowerBoundRateLimiter determines the lower bound interval that we sample every operation.
	// https://godoc.org/github.com/uber/jaeger-client-go#GuaranteedThroughputProbabilisticSampler
	lowerBoundRateLimiter = 1.0 / 60 // 1 request/minute/operation
)

type contextKey struct{}

// Server defines a HTTP server that implements the Controller interface.
type Server struct {
	// Handler should generally not be changed. It exposed to make testing easier.
	Handler http.Handler
	addr string
	l logger.KayveeLogger
}

// Serve starts the server. It will return if an error occurs.
func (s *Server) Serve() error {
	tracingToken := os.Getenv("TRACING_ACCESS_TOKEN")
	ingestURL := os.Getenv("TRACING_INGEST_URL")
	isLocal := os.Getenv("_IS_LOCAL") == "true"

	if !isLocal {
		startLoggingProcessMetrics()
	}

	go func() {
		// This should never return. Listen on the pprof port
		log.Printf("PProf server crashed: %%s", http.ListenAndServe("localhost:6060", nil))
	}()

	dir, err := osext.ExecutableFolder()
	if err != nil {
		log.Fatal(err)
	}
	if err := logger.SetGlobalRouting(path.Join(dir, "kvconfig.yml")); err != nil {
		s.l.Info("please provide a kvconfig.yml file to enable app log routing")
	}

	if (tracingToken != "" && ingestURL != "") || isLocal {
		samplingRate := .01 // 1%% of requests

		if samplingRateStr := os.Getenv("TRACING_SAMPLING_RATE_PERCENT"); samplingRateStr != "" {
			samplingRateP, err := strconv.ParseFloat(samplingRateStr, 64)
			if err != nil {
				s.l.ErrorD("tracing-sampling-override-failed", logger.M{
					"msg": fmt.Sprintf("could not parse '%%s' to integer", samplingRateStr),
				})
			} else {
				samplingRate = samplingRateP
			}

			s.l.InfoD("tracing-sampling-rate", logger.M{
				"msg": fmt.Sprintf("sampling rate will be %%.3f", samplingRate),
			})
		}

		sampler, err := jaeger.NewGuaranteedThroughputProbabilisticSampler(lowerBoundRateLimiter, samplingRate)
		if err != nil {
			return fmt.Errorf("failed to build jaeger sampler: %%s", err)
		}

		cfg := &jaegercfg.Configuration{
			ServiceName: os.Getenv("_APP_NAME"),
			Tags:        []opentracing.Tag{
				opentracing.Tag{Key: "app_name", Value: os.Getenv("_APP_NAME")},
				opentracing.Tag{Key: "build_id", Value: os.Getenv("_BUILD_ID")},
				opentracing.Tag{Key: "deploy_env", Value: os.Getenv("_DEPLOY_ENV")},
				opentracing.Tag{Key: "team_owner", Value: os.Getenv("_TEAM_OWNER")},
				opentracing.Tag{Key: "pod_id", Value: os.Getenv("_POD_ID")},
				opentracing.Tag{Key: "pod_shortname", Value: os.Getenv("_POD_SHORTNAME")},
				opentracing.Tag{Key: "pod_account", Value: os.Getenv("_POD_ACCOUNT")},
				opentracing.Tag{Key: "pod_region", Value: os.Getenv("_POD_REGION")},
			},
		}

		var tracer opentracing.Tracer
		var closer io.Closer
		if isLocal {
			// when local, send everything and use the default params for the Jaeger collector
			cfg.Sampler = &jaegercfg.SamplerConfig{
				Type:  "const",
				Param: 1.0,
			}
			tracer, closer, err = cfg.NewTracer()
			s.l.InfoD("local-tracing", logger.M{"msg": "sending traces to default localhost jaeger address"})
		} else {
			// Create a Jaeger HTTP Thrift transport
			transport := transport.NewHTTPTransport(ingestURL, transport.HTTPBasicAuth("auth", tracingToken))
			tracer, closer, err = cfg.NewTracer(
				jaegercfg.Reporter(jaeger.NewRemoteReporter(transport)),
				jaegercfg.Sampler(sampler))
		}
		if err != nil {
			log.Fatalf("Could not initialize jaeger tracer: %%s", err)
		}
		defer closer.Close()

		opentracing.SetGlobalTracer(tracer)
	} else {
		s.l.Error("please set TRACING_ACCESS_TOKEN & TRACING_INGEST_URL to enable tracing")
	}

	s.l.Counter("server-started")

	// Give the sever 30 seconds to shut down
	return graceful.RunWithErr(s.addr,30*time.Second,s.Handler)
}

type handler struct {
	Controller
}

func startLoggingProcessMetrics() {
	metrics.Log("{{.Title}}", 1*time.Minute)
}

func withMiddleware(serviceName string, router http.Handler, m []func(http.Handler) http.Handler) http.Handler {
	handler := router

	// compress everything
	handler = handlers.CompressHandler(handler)

	// Wrap the middleware in the opposite order specified so that when called then run
	// in the order specified
	for i := len(m) - 1; i >= 0; i-- {
		handler = m[i](handler)
	}
	handler = TracingMiddleware(handler)
	handler = PanicMiddleware(handler)
	// Logging middleware comes last, i.e. will be run first.
	// This makes it so that other middleware has access to the logger
	// that kvMiddleware injects into the request context.
	handler = kvMiddleware.New(handler, serviceName)
	return handler
}

// New returns a Server that implements the Controller interface. It will start when "Serve" is called.
func New(c Controller, addr string) *Server {
	return NewWithMiddleware(c, addr, []func(http.Handler) http.Handler{})
}

// NewRouter returns a mux.Router with no middleware. This is so we can attach additional routes to the
// router if necessary
func NewRouter(c Controller) *mux.Router {
	return newRouter(c)
}

func newRouter(c Controller) *mux.Router {
	router := mux.NewRouter()
	h := handler{Controller: c}

	{{range $index, $val := .Functions}}
	router.Methods("{{$val.Method}}").Path("{{$val.Path}}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		logger.FromContext(r.Context()).AddContext("op", "{{$val.OpID}}")
		h.{{$val.HandlerName}}Handler(r.Context(), w, r)
		ctx := WithTracingOpName(r.Context(), "{{$val.OpID}}")
		r = r.WithContext(ctx)
	})
	{{end}}
	return router
}

// NewWithMiddleware returns a Server that implemenets the Controller interface. It runs the
// middleware after the built-in middleware (e.g. logging), but before the controller methods.
// The middleware is executed in the order specified. The server will start when "Serve" is called.
func NewWithMiddleware(c Controller, addr string, m []func(http.Handler) http.Handler) *Server {
	router := newRouter(c)

	return AttachMiddleware(router, addr, m)
}

// AttachMiddleware attaches the given middleware to the router; this is to be used in conjunction with
// NewServer. It attaches custom middleware passed as arguments as well as the built-in middleware for
// logging, tracing, and handling panics. It should be noted that the built-in middleware executes first
// followed by the passed in middleware (in the order specified).
func AttachMiddleware(router *mux.Router, addr string, m []func(http.Handler) http.Handler) *Server {
	l := logger.New("{{.Title}}")

	handler := withMiddleware("{{.Title}}", router, m)
	return &Server{Handler: handler, addr: addr, l: l}
}`
<|endoftext|>


<commit_before>// Copyright 2016 Qubit Digital Ltd.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package logspray is a collection of tools for streaming and indexing
// large volumes of dynamic logs.

package server

import (
	"errors"
	"fmt"
	"time"

	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	"github.com/QubitProducts/logspray/common"
	"github.com/QubitProducts/logspray/indexer"
	"github.com/QubitProducts/logspray/proto/logspray"
	"github.com/QubitProducts/logspray/ql"
	"github.com/QubitProducts/logspray/sinks"
	"github.com/golang/glog"
	"github.com/golang/protobuf/ptypes"
	"github.com/golang/protobuf/ptypes/timestamp"
	"github.com/prometheus/client_golang/prometheus"
	"golang.org/x/net/context"
)

var (
	lineRxCount = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "logspray_server_received_lines_total",
		Help: "Counter of total lines received since process start.",
	})
	lineTxCount = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "logspray_server_transmit_lines_total",
		Help: "Counter of total lines sent to clients since process start.",
	})
	subscribersGauge = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "logspray_server_active_subscribers",
		Help: "Gauge of number of active subscribers.",
	})
	sourcesGauge = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "logspray_server_active_sources",
		Help: "Gauge of number of active sources.",
	})
	lagTime = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name:    "logspray_server_message_lag_time_seconds",
		Help:    "Histogram of he difference between wall clock time and the message time.",
		Buckets: prometheus.ExponentialBuckets(0.001, 10, 5),
	})
)

type dateRanger interface {
	GetFrom() *timestamp.Timestamp
	GetTo() *timestamp.Timestamp
}

func getRange(dr dateRanger) (time.Time, time.Time, error) {
	from, err := ptypes.Timestamp(dr.GetFrom())
	if err != nil {
		return time.Time{}, time.Time{}, err
	}

	to, err := ptypes.Timestamp(dr.GetTo())
	if err != nil {
		return time.Time{}, time.Time{}, err
	}

	return from, to, nil
}

type serverOpt func(*logServer) error
type logServer struct {
	checkClaims bool
	indx        *indexer.Indexer

	subs                     *subscriberSet
	grafanaUser, grafanaPass string
}

// RegisterStats explicitly register the prometheus metrics, this prevents them
// showing up in the client
func RegisterStats() {
	prometheus.MustRegister(lineRxCount)
	prometheus.MustRegister(lineTxCount)
	prometheus.MustRegister(subscribersGauge)
	prometheus.MustRegister(sourcesGauge)
	prometheus.MustRegister(lagTime)
}

func new(opts ...serverOpt) *logServer {
	lsrv := &logServer{
		checkClaims: true,
		indx:        nil,
		subs:        newSubsSet(),
	}

	for _, o := range opts {
		if err := o(lsrv); err != nil {
			panic(err)
		}
	}

	glog.Info("Creating a new server")
	return lsrv
}

func WithCheckClaims(check bool) serverOpt {
	return func(srv *logServer) error {
		srv.checkClaims = check
		return nil
	}
}

func WithIndex(index *indexer.Indexer) serverOpt {
	return func(srv *logServer) error {
		srv.indx = index
		return nil
	}
}

func WithGrafanaBasicAuth(user, pass string) serverOpt {
	return func(srv *logServer) error {
		srv.grafanaUser, srv.grafanaPass = user, pass
		return nil
	}
}

func (l *logServer) Log(ctx context.Context, r *logspray.Message) (*logspray.LogSummary, error) {
	if err := l.ensureScope(ctx, common.WriteScope); err != nil {
		return nil, err
	}
	l.subs.publish(nil, r)

	if glog.V(1) {
		glog.Info("New Log event arriving")
	}

	return &logspray.LogSummary{}, nil
}

func (l *logServer) LogStream(s logspray.LogService_LogStreamServer) error {
	sourcesGauge.Add(1.0)
	defer sourcesGauge.Sub(1.0)

	var err error
	if err := l.ensureScope(s.Context(), common.WriteScope); err != nil {
		return err
	}

	var hdr *logspray.Message

	if glog.V(1) {
		glog.Info("New Log stream arriving")
	}
	defer func() {
		if err != nil && err != context.Canceled {
			glog.Info("Log stream ended: err = %v", err)
		}
	}()

	var iw sinks.MessageWriter
	for {
		m, err := s.Recv()
		if err != nil {
			return err
		}
		if glog.V(3) {
			glog.Infof("Incoming message: %#v", *m)
		}
		if m.Setheader || m.ControlMessage == logspray.Message_SETHEADER {
			if hdr != nil {
				return errors.New("Multiple headers in one steram are not allowed")
			}
			hdr = m
			if l.indx != nil {
				iw, err = l.indx.AddSource(s.Context(), m.StreamID, m.Labels)
				if err != nil {
					glog.Errorf("Error adding index source, err = %v\n", err)
				}
			}

			defer func() {
				if iw != nil {
					iw.Close()
				}

				l.subs.publish(
					hdr,
					&logspray.Message{
						StreamID:       hdr.StreamID,
						ControlMessage: logspray.Message_STREAMEND,
					})
			}()
			continue
		}

		if hdr == nil {
			return errors.New("Message data sent before header")
		}

		// We'll set the StreamID here
		m.StreamID = hdr.StreamID

		lineRxCount.Inc()
		if mtime, err := ptypes.Timestamp(m.Time); err == nil {
			lagTime.Observe(float64(time.Since(mtime)) / float64(time.Second))
		}

		if m.Labels == nil {
			m.Labels = map[string]string{}
		}

		l.subs.publish(hdr, m)

		err = iw.WriteMessage(s.Context(), m)
		if err != nil {
			glog.Errorf("Error adding index source, err = %v\n", err)
		}
	}
}

func (l *logServer) Tail(r *logspray.TailRequest, s logspray.LogService_TailServer) error {
	ctx := s.Context()

	if err := l.ensureScope(ctx, common.ReadScope); err != nil {
		return err
	}

	mc := l.subs.subscribe()
	if glog.V(1) {
		glog.Info("Subscriber added")
	}
	subscribersGauge.Add(1.0)

	defer l.subs.unsubscribe(mc)
	defer subscribersGauge.Sub(1.0)
	defer glog.Info("Subscriber gone")

	tick := time.NewTicker(5 * time.Second)
	defer tick.Stop()

	matcher, err := ql.Compile(r.Query)
	if err != nil {
		return status.Errorf(codes.InvalidArgument, err.Error())
	}

	headers := map[string]*logspray.Message{}
	sentHeaders := map[*logspray.Message]struct{}{}

	for {
		select {
		case m := <-mc:
			if m.ControlMessage == logspray.Message_SETHEADER {
				headers[m.StreamID] = m
				continue
			}

			if m.ControlMessage == logspray.Message_STREAMEND {
				hm, ok := headers[m.StreamID]
				if !ok {
					glog.Errorf("Got close for untracked stream")
					continue
				}
				delete(headers, hm.StreamID)
				delete(sentHeaders, hm)
				if err := s.Send(m); err != nil {
					glog.Errorf("Error sending stream end to subscribe err = %v", err)
					return err
				}
				continue
			}

			hdr, ok := headers[m.StreamID]
			if !ok {
				glog.Info("Error no known header for Stream %s", fmt.Sprintf("%s", m.StreamID))
			}

			if !matcher(hdr, m, false) {
				continue
			}
			if hdr != nil {
				if _, ok := sentHeaders[hdr]; !ok {
					if err := s.Send(hdr); err != nil {
						glog.Info("Error sending to subscribe err = %v", err)
						return err
					}
					sentHeaders[hdr] = struct{}{}
				}
			}
			if err := s.Send(m); err != nil {
				glog.Errorf("Error sending to subscribe err = %v", err)
				return err
			}
			lineTxCount.Inc()
		case <-tick.C:
			err := s.Send(&logspray.Message{
				ControlMessage: logspray.Message_OK,
			})
			if err != nil {
				glog.Errorf("Error sending heartbeat to subscribe err = %v", err)
				return err
			}
		case <-ctx.Done():
			if err != nil && err != context.Canceled {
				glog.Errorf("Tail Context closed = %v", ctx.Err())
				return err
			}
			return nil
		}
	}
}

func (l *logServer) Labels(ctx context.Context, r *logspray.LabelsRequest) (*logspray.LabelsResponse, error) {
	var err error
	if err = l.ensureScope(ctx, common.ReadScope); err != nil {
		return nil, err
	}

	from, to, err := getRange(r)
	if err != nil {
		return nil, err
	}

	res := &logspray.LabelsResponse{Names: []string{}}
	res.Names, err = l.indx.Labels(from, to)

	return res, err
}

func (l *logServer) LabelValues(ctx context.Context, r *logspray.LabelValuesRequest) (*logspray.LabelValuesResponse, error) {
	var err error
	if err = l.ensureScope(ctx, common.ReadScope); err != nil {
		return nil, err
	}

	from, to, err := getRange(r)
	if err != nil {
		return nil, err
	}
	vs, hitcount, err := l.indx.LabelValues(r.Name, from, to, int(r.Count))
	if err != nil {
		return nil, err
	}

	res := &logspray.LabelValuesResponse{Values: vs, TotalHitCount: uint64(hitcount)}

	return res, nil
}

func (l *logServer) Search(ctx context.Context, r *logspray.SearchRequest) (*logspray.SearchResponse, error) {
	ctx, cancel := context.WithCancel(ctx)

	var err error
	if err = l.ensureScope(ctx, common.ReadScope); err != nil {
		return nil, err
	}
	from, to, err := getRange(r)
	if err != nil {
		return nil, err
	}

	matcher, err := ql.Compile(r.Query)
	if err != nil {
		return nil, status.Errorf(codes.InvalidArgument, err.Error())
	}

	offset := r.Offset
	count := r.Count
	res := &logspray.SearchResponse{}
	msgFunc := logspray.MakeFlattenStreamFunc(func(m *logspray.Message) error {
		t, _ := ptypes.Timestamp(m.Time)
		if m.ControlMessage == 0 {
			if t.Before(from) || t.After(to) {
				return nil
			}
			if offset != 0 {
				offset--
				return nil
			}
		}
		res.TotalHitCount++
		res.Messages = append(res.Messages, m)
		if m.ControlMessage == 0 {
			count--
			if count == 0 {
				cancel()
			}
		}
		return nil
	})
	err = l.indx.Search(ctx, msgFunc, matcher, from, to, r.Reverse)
	if err != nil && err != context.Canceled {
		return res, err
	}

	return res, nil
}

func (l *logServer) SearchStream(r *logspray.SearchRequest, s logspray.LogService_SearchStreamServer) error {
	ctx := s.Context()
	ctx, cancel := context.WithCancel(ctx)

	if err := l.ensureScope(ctx, common.ReadScope); err != nil {
		return err
	}

	from, to, err := getRange(r)
	if err != nil {
		return err
	}

	matcher, err := ql.Compile(r.Query)
	if err != nil {
		return status.Errorf(codes.InvalidArgument, err.Error())
	}

	count := r.Count
	offset := r.Offset
	msgFunc := logspray.MakeInjectStreamHeadersFunc(func(m *logspray.Message) error {
		t, _ := ptypes.Timestamp(m.Time)
		if m.ControlMessage == 0 {
			if t.Before(from) || t.After(to) {
				return nil
			}
			if offset != 0 {
				offset--
				return nil
			}
		}

		if err := s.Send(m); err != nil {
			return err
		}

		if m.ControlMessage == 0 {
			count--
			if count == 0 {
				cancel()
			}
		}
		return nil
	})

	err = l.indx.Search(ctx, msgFunc, matcher, from, to, r.Reverse)
	if err != nil && err != context.Canceled {
		return err
	}

	return nil
}
<commit_msg>Allow a count of 0 on streaming results<commit_after>// Copyright 2016 Qubit Digital Ltd.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package logspray is a collection of tools for streaming and indexing
// large volumes of dynamic logs.

package server

import (
	"errors"
	"fmt"
	"time"

	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	"github.com/QubitProducts/logspray/common"
	"github.com/QubitProducts/logspray/indexer"
	"github.com/QubitProducts/logspray/proto/logspray"
	"github.com/QubitProducts/logspray/ql"
	"github.com/QubitProducts/logspray/sinks"
	"github.com/golang/glog"
	"github.com/golang/protobuf/ptypes"
	"github.com/golang/protobuf/ptypes/timestamp"
	"github.com/prometheus/client_golang/prometheus"
	"golang.org/x/net/context"
)

var (
	lineRxCount = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "logspray_server_received_lines_total",
		Help: "Counter of total lines received since process start.",
	})
	lineTxCount = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "logspray_server_transmit_lines_total",
		Help: "Counter of total lines sent to clients since process start.",
	})
	subscribersGauge = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "logspray_server_active_subscribers",
		Help: "Gauge of number of active subscribers.",
	})
	sourcesGauge = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "logspray_server_active_sources",
		Help: "Gauge of number of active sources.",
	})
	lagTime = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name:    "logspray_server_message_lag_time_seconds",
		Help:    "Histogram of he difference between wall clock time and the message time.",
		Buckets: prometheus.ExponentialBuckets(0.001, 10, 5),
	})
)

type dateRanger interface {
	GetFrom() *timestamp.Timestamp
	GetTo() *timestamp.Timestamp
}

func getRange(dr dateRanger) (time.Time, time.Time, error) {
	from, err := ptypes.Timestamp(dr.GetFrom())
	if err != nil {
		return time.Time{}, time.Time{}, err
	}

	to, err := ptypes.Timestamp(dr.GetTo())
	if err != nil {
		return time.Time{}, time.Time{}, err
	}

	return from, to, nil
}

type serverOpt func(*logServer) error
type logServer struct {
	checkClaims bool
	indx        *indexer.Indexer

	subs                     *subscriberSet
	grafanaUser, grafanaPass string
}

// RegisterStats explicitly register the prometheus metrics, this prevents them
// showing up in the client
func RegisterStats() {
	prometheus.MustRegister(lineRxCount)
	prometheus.MustRegister(lineTxCount)
	prometheus.MustRegister(subscribersGauge)
	prometheus.MustRegister(sourcesGauge)
	prometheus.MustRegister(lagTime)
}

func new(opts ...serverOpt) *logServer {
	lsrv := &logServer{
		checkClaims: true,
		indx:        nil,
		subs:        newSubsSet(),
	}

	for _, o := range opts {
		if err := o(lsrv); err != nil {
			panic(err)
		}
	}

	glog.Info("Creating a new server")
	return lsrv
}

func WithCheckClaims(check bool) serverOpt {
	return func(srv *logServer) error {
		srv.checkClaims = check
		return nil
	}
}

func WithIndex(index *indexer.Indexer) serverOpt {
	return func(srv *logServer) error {
		srv.indx = index
		return nil
	}
}

func WithGrafanaBasicAuth(user, pass string) serverOpt {
	return func(srv *logServer) error {
		srv.grafanaUser, srv.grafanaPass = user, pass
		return nil
	}
}

func (l *logServer) Log(ctx context.Context, r *logspray.Message) (*logspray.LogSummary, error) {
	if err := l.ensureScope(ctx, common.WriteScope); err != nil {
		return nil, err
	}
	l.subs.publish(nil, r)

	if glog.V(1) {
		glog.Info("New Log event arriving")
	}

	return &logspray.LogSummary{}, nil
}

func (l *logServer) LogStream(s logspray.LogService_LogStreamServer) error {
	sourcesGauge.Add(1.0)
	defer sourcesGauge.Sub(1.0)

	var err error
	if err := l.ensureScope(s.Context(), common.WriteScope); err != nil {
		return err
	}

	var hdr *logspray.Message

	if glog.V(1) {
		glog.Info("New Log stream arriving")
	}
	defer func() {
		if err != nil && err != context.Canceled {
			glog.Info("Log stream ended: err = %v", err)
		}
	}()

	var iw sinks.MessageWriter
	for {
		m, err := s.Recv()
		if err != nil {
			return err
		}
		if glog.V(3) {
			glog.Infof("Incoming message: %#v", *m)
		}
		if m.Setheader || m.ControlMessage == logspray.Message_SETHEADER {
			if hdr != nil {
				return errors.New("Multiple headers in one steram are not allowed")
			}
			hdr = m
			if l.indx != nil {
				iw, err = l.indx.AddSource(s.Context(), m.StreamID, m.Labels)
				if err != nil {
					glog.Errorf("Error adding index source, err = %v\n", err)
				}
			}

			defer func() {
				if iw != nil {
					iw.Close()
				}

				l.subs.publish(
					hdr,
					&logspray.Message{
						StreamID:       hdr.StreamID,
						ControlMessage: logspray.Message_STREAMEND,
					})
			}()
			continue
		}

		if hdr == nil {
			return errors.New("Message data sent before header")
		}

		// We'll set the StreamID here
		m.StreamID = hdr.StreamID

		lineRxCount.Inc()
		if mtime, err := ptypes.Timestamp(m.Time); err == nil {
			lagTime.Observe(float64(time.Since(mtime)) / float64(time.Second))
		}

		if m.Labels == nil {
			m.Labels = map[string]string{}
		}

		l.subs.publish(hdr, m)

		err = iw.WriteMessage(s.Context(), m)
		if err != nil {
			glog.Errorf("Error adding index source, err = %v\n", err)
		}
	}
}

func (l *logServer) Tail(r *logspray.TailRequest, s logspray.LogService_TailServer) error {
	ctx := s.Context()

	if err := l.ensureScope(ctx, common.ReadScope); err != nil {
		return err
	}

	mc := l.subs.subscribe()
	if glog.V(1) {
		glog.Info("Subscriber added")
	}
	subscribersGauge.Add(1.0)

	defer l.subs.unsubscribe(mc)
	defer subscribersGauge.Sub(1.0)
	defer glog.Info("Subscriber gone")

	tick := time.NewTicker(5 * time.Second)
	defer tick.Stop()

	matcher, err := ql.Compile(r.Query)
	if err != nil {
		return status.Errorf(codes.InvalidArgument, err.Error())
	}

	headers := map[string]*logspray.Message{}
	sentHeaders := map[*logspray.Message]struct{}{}

	for {
		select {
		case m := <-mc:
			if m.ControlMessage == logspray.Message_SETHEADER {
				headers[m.StreamID] = m
				continue
			}

			if m.ControlMessage == logspray.Message_STREAMEND {
				hm, ok := headers[m.StreamID]
				if !ok {
					glog.Errorf("Got close for untracked stream")
					continue
				}
				delete(headers, hm.StreamID)
				delete(sentHeaders, hm)
				if err := s.Send(m); err != nil {
					glog.Errorf("Error sending stream end to subscribe err = %v", err)
					return err
				}
				continue
			}

			hdr, ok := headers[m.StreamID]
			if !ok {
				glog.Info("Error no known header for Stream %s", fmt.Sprintf("%s", m.StreamID))
			}

			if !matcher(hdr, m, false) {
				continue
			}
			if hdr != nil {
				if _, ok := sentHeaders[hdr]; !ok {
					if err := s.Send(hdr); err != nil {
						glog.Info("Error sending to subscribe err = %v", err)
						return err
					}
					sentHeaders[hdr] = struct{}{}
				}
			}
			if err := s.Send(m); err != nil {
				glog.Errorf("Error sending to subscribe err = %v", err)
				return err
			}
			lineTxCount.Inc()
		case <-tick.C:
			err := s.Send(&logspray.Message{
				ControlMessage: logspray.Message_OK,
			})
			if err != nil {
				glog.Errorf("Error sending heartbeat to subscribe err = %v", err)
				return err
			}
		case <-ctx.Done():
			if err != nil && err != context.Canceled {
				glog.Errorf("Tail Context closed = %v", ctx.Err())
				return err
			}
			return nil
		}
	}
}

func (l *logServer) Labels(ctx context.Context, r *logspray.LabelsRequest) (*logspray.LabelsResponse, error) {
	var err error
	if err = l.ensureScope(ctx, common.ReadScope); err != nil {
		return nil, err
	}

	from, to, err := getRange(r)
	if err != nil {
		return nil, err
	}

	res := &logspray.LabelsResponse{Names: []string{}}
	res.Names, err = l.indx.Labels(from, to)

	return res, err
}

func (l *logServer) LabelValues(ctx context.Context, r *logspray.LabelValuesRequest) (*logspray.LabelValuesResponse, error) {
	var err error
	if err = l.ensureScope(ctx, common.ReadScope); err != nil {
		return nil, err
	}

	from, to, err := getRange(r)
	if err != nil {
		return nil, err
	}
	vs, hitcount, err := l.indx.LabelValues(r.Name, from, to, int(r.Count))
	if err != nil {
		return nil, err
	}

	res := &logspray.LabelValuesResponse{Values: vs, TotalHitCount: uint64(hitcount)}

	return res, nil
}

func (l *logServer) Search(ctx context.Context, r *logspray.SearchRequest) (*logspray.SearchResponse, error) {
	ctx, cancel := context.WithCancel(ctx)

	var err error
	if err = l.ensureScope(ctx, common.ReadScope); err != nil {
		return nil, err
	}
	from, to, err := getRange(r)
	if err != nil {
		return nil, err
	}

	if r.Count == 0 {
		return nil, status.Errorf(codes.InvalidArgument, "count must be non-zero")
	}

	matcher, err := ql.Compile(r.Query)
	if err != nil {
		return nil, status.Errorf(codes.InvalidArgument, err.Error())
	}

	offset := r.Offset
	count := r.Count
	res := &logspray.SearchResponse{}
	msgFunc := logspray.MakeFlattenStreamFunc(func(m *logspray.Message) error {
		t, _ := ptypes.Timestamp(m.Time)
		if m.ControlMessage == 0 {
			if t.Before(from) || t.After(to) {
				return nil
			}
			if offset != 0 {
				offset--
				return nil
			}
		}
		res.TotalHitCount++
		res.Messages = append(res.Messages, m)
		if m.ControlMessage == 0 {
			count--
			if count == 0 {
				cancel()
			}
		}
		return nil
	})
	err = l.indx.Search(ctx, msgFunc, matcher, from, to, r.Reverse)
	if err != nil && err != context.Canceled {
		return res, err
	}

	return res, nil
}

func (l *logServer) SearchStream(r *logspray.SearchRequest, s logspray.LogService_SearchStreamServer) error {
	ctx := s.Context()
	ctx, cancel := context.WithCancel(ctx)

	if err := l.ensureScope(ctx, common.ReadScope); err != nil {
		return err
	}

	from, to, err := getRange(r)
	if err != nil {
		return err
	}

	matcher, err := ql.Compile(r.Query)
	if err != nil {
		return status.Errorf(codes.InvalidArgument, err.Error())
	}

	enforceCount := r.Count != 0
	count := r.Count
	offset := r.Offset
	msgFunc := logspray.MakeInjectStreamHeadersFunc(func(m *logspray.Message) error {
		t, _ := ptypes.Timestamp(m.Time)
		if m.ControlMessage == 0 {
			if t.Before(from) || t.After(to) {
				return nil
			}
			if offset != 0 {
				offset--
				return nil
			}
		}

		if err := s.Send(m); err != nil {
			return err
		}

		if enforceCount && m.ControlMessage == 0 {
			count--
			if count == 0 {
				cancel()
			}
		}
		return nil
	})

	err = l.indx.Search(ctx, msgFunc, matcher, from, to, r.Reverse)
	if err != nil && err != context.Canceled {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"net/http"
	"os"

	"github.com/cardigann/cardigann/config"
	"github.com/cardigann/cardigann/indexer"
	"github.com/cardigann/cardigann/logger"
)

// Server is an http server which wraps the Handler
type Server struct {
	Bind, Port, Passphrase string
	version                string
	config                 config.Config
}

func New(conf config.Config, version string) (*Server, error) {
	bind, err := config.GetGlobalConfig("bind", "0.0.0.0", conf)
	if err != nil {
		return nil, err
	}

	port, err := config.GetGlobalConfig("port", "5060", conf)
	if err != nil {
		return nil, err
	}

	passphrase, err := config.GetGlobalConfig("passphrase", "", conf)
	if err != nil {
		return nil, err
	}

	if envPort := os.Getenv("PORT"); envPort != "" {
		port = envPort
	}

	if version == "" {
		version = "dev"
	}

	return &Server{
		Bind:       bind,
		Port:       port,
		Passphrase: passphrase,
		config:     conf,
		version:    version,
	}, nil
}

func (s *Server) Listen() error {
	logger.Logger.Infof("Cardigann %s", s.version)

	path, err := config.GetConfigPath()
	if err != nil {
		return err
	}

	logger.Logger.Debugf("Config path is %s", path)
	logger.Logger.Debugf("Cache dir is %s", config.GetCachePath("/"))

	for _, dir := range config.GetDefinitionDirs() {
		if _, err := os.Stat(dir); os.IsExist(err) {
			logger.Logger.Debugf("Searching %s for definitions", dir)
		}
	}

	builtins, err := indexer.ListBuiltins()
	if err != nil {
		return err
	}

	logger.Logger.Debugf("Found %d built-in definitions", len(builtins))

	listenOn := fmt.Sprintf("%s:%s", s.Bind, s.Port)
	logger.Logger.Infof("Listening on %s", listenOn)

	h, err := NewHandler(Params{
		Passphrase: s.Passphrase,
		Config:     s.config,
		Version:    s.version,
	})
	if err != nil {
		return err
	}

	return http.ListenAndServe(listenOn, h)
}
<commit_msg>Add more debugging about server startup<commit_after>package server

import (
	"fmt"
	"net/http"
	"os"

	"github.com/cardigann/cardigann/config"
	"github.com/cardigann/cardigann/indexer"
	"github.com/cardigann/cardigann/logger"
)

// Server is an http server which wraps the Handler
type Server struct {
	Bind, Port, Passphrase string
	version                string
	config                 config.Config
}

func New(conf config.Config, version string) (*Server, error) {
	bind, err := config.GetGlobalConfig("bind", "0.0.0.0", conf)
	if err != nil {
		return nil, err
	}

	port, err := config.GetGlobalConfig("port", "5060", conf)
	if err != nil {
		return nil, err
	}

	passphrase, err := config.GetGlobalConfig("passphrase", "", conf)
	if err != nil {
		return nil, err
	}

	if envPort := os.Getenv("PORT"); envPort != "" {
		port = envPort
	}

	if version == "" {
		version = "dev"
	}

	return &Server{
		Bind:       bind,
		Port:       port,
		Passphrase: passphrase,
		config:     conf,
		version:    version,
	}, nil
}

func (s *Server) Listen() error {
	logger.Logger.Infof("Cardigann %s", s.version)

	path, err := config.GetConfigPath()
	if err != nil {
		return err
	}

	logger.Logger.Debugf("Config path is %s", path)
	logger.Logger.Debugf("Cache dir is %s", config.GetCachePath("/"))

	for _, dir := range config.GetDefinitionDirs() {
		if _, err := os.Stat(dir); os.IsExist(err) {
			logger.Logger.Debugf("Searching %s for definitions", dir)
		}
	}

	builtins, err := indexer.ListBuiltins()
	if err != nil {
		return err
	}

	logger.Logger.Debugf("Found %d built-in definitions", len(builtins))

	defs, err := indexer.DefaultDefinitionLoader.List()
	if err != nil {
		return err
	}

	active := 0
	for _, key := range defs {
		if config.IsSectionEnabled(key, s.config) {
			active++
		}
	}

	logger.Logger.Debugf("Found %d indexers enabled via config", active)

	listenOn := fmt.Sprintf("%s:%s", s.Bind, s.Port)
	logger.Logger.Infof("Listening on %s", listenOn)

	h, err := NewHandler(Params{
		Passphrase: s.Passphrase,
		Config:     s.config,
		Version:    s.version,
	})
	if err != nil {
		return err
	}

	return http.ListenAndServe(listenOn, h)
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"log"
	"net/http"

	"github.com/codegangsta/negroni"
	"github.com/gorilla/mux"
	"github.com/unrolled/render"
	"github.com/wanelo/image-server/core"
	"github.com/wanelo/image-server/fetcher"
	"github.com/wanelo/image-server/info"
	"github.com/wanelo/image-server/parser"
	"github.com/wanelo/image-server/processor"
	"github.com/wanelo/image-server/uploader"
)

func InitializeRouter(sc *core.ServerConfiguration, port string) {
	log.Println("starting server on http://0.0.0.0:" + port)

	router := mux.NewRouter()
	router.HandleFunc("/{namespace:[a-z0-9]+}", func(wr http.ResponseWriter, req *http.Request) {
		NewImageHandler(wr, req, sc)
	}).Methods("POST").Name("newImage")

	router.HandleFunc("/{namespace:[a-z0-9]+}/{id1:[a-f0-9]{3}}/{id2:[a-f0-9]{3}}/{id3:[a-f0-9]{3}}/{id4:[a-f0-9]{23}}/{filename}", func(wr http.ResponseWriter, req *http.Request) {
		ResizeHandler(wr, req, sc)
	}).Methods("GET").Name("resizeImage")

	// n := negroni.New()
	n := negroni.Classic()
	n.UseHandler(router)

	n.Run(":" + port)
}

func NewImageHandler(w http.ResponseWriter, req *http.Request, sc *core.ServerConfiguration) {
	r := render.New(render.Options{
		IndentJSON: true,
	})

	qs := req.URL.Query()
	vars := mux.Vars(req)
	errorStr := ""

	source := qs.Get("source")
	namespace := vars["namespace"]

	log.Printf("Processing request for: %s", source)

	f := fetcher.NewSourceFetcher(sc.Adapters.Paths)

	imageDetails, downloaded, err := f.Fetch(source, namespace)
	var json map[string]string

	if err != nil {
		errorStr = fmt.Sprintf("%s", err)
		// r.JSON(w, http.StatusOK, json)
		json = map[string]string{
			"error": errorStr,
		}
		r.JSON(w, http.StatusOK, json)
		return
	}

	hash := imageDetails.Hash

	if downloaded {
		localOriginalPath := f.Paths.LocalOriginalPath(namespace, hash)
		uploader := &uploader.Uploader{sc.RemoteBasePath}
		err := uploader.CreateDirectory(sc.Adapters.Paths.RemoteImageDirectory(namespace, hash))
		if err != nil {
			log.Printf("Manta::sentToManta unable to create directory %s", sc.RemoteBasePath)
			return
		}

		destination := sc.Adapters.Paths.RemoteOriginalPath(namespace, hash)

		go sc.Adapters.Logger.OriginalDownloaded(localOriginalPath, destination)
		go func() {
			localInfoPath := sc.Adapters.Paths.LocalInfoPath(namespace, hash)
			remoteInfoPath := sc.Adapters.Paths.RemoteInfoPath(namespace, hash)

			err := info.SaveImageDetail(imageDetails, localInfoPath)
			if err != nil {
				log.Println(err)
			}

			// upload info
			err = uploader.Upload(localInfoPath, remoteInfoPath)
			if err != nil {
				log.Println(err)
			}
		}()

		// upload original image
		err = uploader.Upload(localOriginalPath, destination)
		if err != nil {
			log.Println(err)
		}
	}

	json = map[string]string{
		"error":  errorStr,
		"hash":   hash,
		"height": fmt.Sprintf("%v", imageDetails.Height),
		"width":  fmt.Sprintf("%v", imageDetails.Width),
	}

	r.JSON(w, http.StatusOK, json)
}

func ResizeHandler(w http.ResponseWriter, req *http.Request, sc *core.ServerConfiguration) {
	vars := mux.Vars(req)
	filename := vars["filename"]

	ic, err := parser.NameToConfiguration(sc, filename)
	if err != nil {
		errorHandler(err, w, req, http.StatusNotFound, sc, ic)
		return
	}

	namespace := vars["namespace"]
	id1 := vars["id1"]
	id2 := vars["id2"]
	id3 := vars["id3"]
	id4 := vars["id4"]
	hash := fmt.Sprintf("%s%s%s%s", id1, id2, id3, id4)

	ic.ID = hash
	ic.Namespace = namespace

	localPath := sc.Adapters.Paths.LocalImagePath(namespace, hash, filename)
	localOriginalPath := sc.Adapters.Paths.LocalOriginalPath(namespace, hash)

	// download original image
	remoteOriginalPath := sc.Adapters.Paths.RemoteOriginalURL(namespace, hash)
	log.Println(remoteOriginalPath)
	f := fetcher.NewUniqueFetcher(remoteOriginalPath, localOriginalPath)
	_, err = f.Fetch()
	if err != nil {
		errorHandler(err, w, req, http.StatusNotFound, sc, ic)
		return
	}

	// process image
	pchan := &processor.ProcessorChannels{
		ImageProcessed: make(chan *core.ImageConfiguration),
		Skipped:        make(chan string),
	}
	defer close(pchan.ImageProcessed)
	defer close(pchan.Skipped)

	p := processor.Processor{
		Source:             localOriginalPath,
		Destination:        localPath,
		ImageConfiguration: ic,
		Channels:           pchan,
	}

	resizedPath, err := p.CreateImage()

	if err != nil {
		errorHandler(err, w, req, http.StatusNotFound, sc, ic)
		return
	}

	select {
	case <-pchan.ImageProcessed:
		log.Println("about to upload to manta")
		uploader := &uploader.Uploader{sc.RemoteBasePath}
		remoteResizedPath := sc.Adapters.Paths.RemoteImagePath(namespace, hash, filename)
		err = uploader.Upload(localPath, remoteResizedPath)
		if err != nil {
			log.Println(err)
		}
	case path := <-pchan.Skipped:
		log.Printf("Skipped processing %s", path)
	}

	http.ServeFile(w, req, resizedPath)
}

func errorHandler(err error, w http.ResponseWriter, r *http.Request, status int, sc *core.ServerConfiguration, ic *core.ImageConfiguration) {
	w.WriteHeader(status)
	if status == http.StatusNotFound {
		fmt.Fprint(w, "404 image not available. ", err)
	}
}
<commit_msg>Upload image info synchronously<commit_after>package server

import (
	"fmt"
	"log"
	"net/http"

	"github.com/codegangsta/negroni"
	"github.com/gorilla/mux"
	"github.com/unrolled/render"
	"github.com/wanelo/image-server/core"
	"github.com/wanelo/image-server/fetcher"
	"github.com/wanelo/image-server/info"
	"github.com/wanelo/image-server/parser"
	"github.com/wanelo/image-server/processor"
	"github.com/wanelo/image-server/uploader"
)

func InitializeRouter(sc *core.ServerConfiguration, port string) {
	log.Println("starting server on http://0.0.0.0:" + port)

	router := mux.NewRouter()
	router.HandleFunc("/{namespace:[a-z0-9]+}", func(wr http.ResponseWriter, req *http.Request) {
		NewImageHandler(wr, req, sc)
	}).Methods("POST").Name("newImage")

	router.HandleFunc("/{namespace:[a-z0-9]+}/{id1:[a-f0-9]{3}}/{id2:[a-f0-9]{3}}/{id3:[a-f0-9]{3}}/{id4:[a-f0-9]{23}}/{filename}", func(wr http.ResponseWriter, req *http.Request) {
		ResizeHandler(wr, req, sc)
	}).Methods("GET").Name("resizeImage")

	// n := negroni.New()
	n := negroni.Classic()
	n.UseHandler(router)

	n.Run(":" + port)
}

func NewImageHandler(w http.ResponseWriter, req *http.Request, sc *core.ServerConfiguration) {
	r := render.New(render.Options{
		IndentJSON: true,
	})

	qs := req.URL.Query()
	vars := mux.Vars(req)
	errorStr := ""

	source := qs.Get("source")
	namespace := vars["namespace"]

	log.Printf("Processing request for: %s", source)

	f := fetcher.NewSourceFetcher(sc.Adapters.Paths)

	imageDetails, downloaded, err := f.Fetch(source, namespace)
	var json map[string]string

	if err != nil {
		errorStr = fmt.Sprintf("%s", err)
		// r.JSON(w, http.StatusOK, json)
		json = map[string]string{
			"error": errorStr,
		}
		r.JSON(w, http.StatusOK, json)
		return
	}

	hash := imageDetails.Hash

	if downloaded {
		localOriginalPath := f.Paths.LocalOriginalPath(namespace, hash)
		uploader := &uploader.Uploader{sc.RemoteBasePath}
		err := uploader.CreateDirectory(sc.Adapters.Paths.RemoteImageDirectory(namespace, hash))
		if err != nil {
			log.Printf("Manta::sentToManta unable to create directory %s", sc.RemoteBasePath)
			return
		}

		destination := sc.Adapters.Paths.RemoteOriginalPath(namespace, hash)

		go sc.Adapters.Logger.OriginalDownloaded(localOriginalPath, destination)

		localInfoPath := sc.Adapters.Paths.LocalInfoPath(namespace, hash)
		remoteInfoPath := sc.Adapters.Paths.RemoteInfoPath(namespace, hash)

		err = info.SaveImageDetail(imageDetails, localInfoPath)
		if err != nil {
			log.Println(err)
		}

		// upload info
		err = uploader.Upload(localInfoPath, remoteInfoPath)
		if err != nil {
			log.Println(err)
		}

		// upload original image
		err = uploader.Upload(localOriginalPath, destination)
		if err != nil {
			log.Println(err)
		}
	}

	json = map[string]string{
		"error":  errorStr,
		"hash":   hash,
		"height": fmt.Sprintf("%v", imageDetails.Height),
		"width":  fmt.Sprintf("%v", imageDetails.Width),
	}

	r.JSON(w, http.StatusOK, json)
}

func ResizeHandler(w http.ResponseWriter, req *http.Request, sc *core.ServerConfiguration) {
	vars := mux.Vars(req)
	filename := vars["filename"]

	ic, err := parser.NameToConfiguration(sc, filename)
	if err != nil {
		errorHandler(err, w, req, http.StatusNotFound, sc, ic)
		return
	}

	namespace := vars["namespace"]
	id1 := vars["id1"]
	id2 := vars["id2"]
	id3 := vars["id3"]
	id4 := vars["id4"]
	hash := fmt.Sprintf("%s%s%s%s", id1, id2, id3, id4)

	ic.ID = hash
	ic.Namespace = namespace

	localPath := sc.Adapters.Paths.LocalImagePath(namespace, hash, filename)
	localOriginalPath := sc.Adapters.Paths.LocalOriginalPath(namespace, hash)

	// download original image
	remoteOriginalPath := sc.Adapters.Paths.RemoteOriginalURL(namespace, hash)
	log.Println(remoteOriginalPath)
	f := fetcher.NewUniqueFetcher(remoteOriginalPath, localOriginalPath)
	_, err = f.Fetch()
	if err != nil {
		errorHandler(err, w, req, http.StatusNotFound, sc, ic)
		return
	}

	// process image
	pchan := &processor.ProcessorChannels{
		ImageProcessed: make(chan *core.ImageConfiguration),
		Skipped:        make(chan string),
	}
	defer close(pchan.ImageProcessed)
	defer close(pchan.Skipped)

	p := processor.Processor{
		Source:             localOriginalPath,
		Destination:        localPath,
		ImageConfiguration: ic,
		Channels:           pchan,
	}

	resizedPath, err := p.CreateImage()

	if err != nil {
		errorHandler(err, w, req, http.StatusNotFound, sc, ic)
		return
	}

	select {
	case <-pchan.ImageProcessed:
		log.Println("about to upload to manta")
		uploader := &uploader.Uploader{sc.RemoteBasePath}
		remoteResizedPath := sc.Adapters.Paths.RemoteImagePath(namespace, hash, filename)
		err = uploader.Upload(localPath, remoteResizedPath)
		if err != nil {
			log.Println(err)
		}
	case path := <-pchan.Skipped:
		log.Printf("Skipped processing %s", path)
	}

	http.ServeFile(w, req, resizedPath)
}

func errorHandler(err error, w http.ResponseWriter, r *http.Request, status int, sc *core.ServerConfiguration, ic *core.ImageConfiguration) {
	w.WriteHeader(status)
	if status == http.StatusNotFound {
		fmt.Fprint(w, "404 image not available. ", err)
	}
}
<|endoftext|>


<commit_before>package server

import (
	"encoding/json"

	"github.com/coreos/fleet/third_party/github.com/coreos/go-etcd/etcd"
	log "github.com/coreos/fleet/third_party/github.com/golang/glog"

	"github.com/coreos/fleet/agent"
	"github.com/coreos/fleet/config"
	"github.com/coreos/fleet/engine"
	"github.com/coreos/fleet/machine"
	"github.com/coreos/fleet/registry"
	"github.com/coreos/fleet/sign"
)

type Server struct {
	agent  *agent.Agent
	engine *engine.Engine
}

func New(cfg config.Config) (*Server, error) {
	mach := machine.New(cfg.BootID, cfg.PublicIP, cfg.Metadata())

	regClient := etcd.NewClient(cfg.EtcdServers)
	regClient.SetConsistency(etcd.STRONG_CONSISTENCY)
	reg := registry.New(regClient)

	aEventClient := etcd.NewClient(cfg.EtcdServers)
	aEventClient.SetConsistency(etcd.STRONG_CONSISTENCY)
	aEventStream := registry.NewEventStream(aEventClient, reg)

	var verifier *sign.SignatureVerifier
	if cfg.VerifyUnits {
		var err error
		verifier, err = sign.NewSignatureVerifierFromAuthorizedKeysFile(cfg.AuthorizedKeysFile)
		if err != nil {
			log.Errorln("Failed to get any key from authorized key file in verify_units mode:", err)
			verifier = sign.NewSignatureVerifier()
		}
	}

	a, err := agent.New(reg, aEventStream, mach, cfg.AgentTTL, verifier)
	if err != nil {
		log.Errorf("Error creating Agent")
		return nil, err
	}

	eEventClient := etcd.NewClient(cfg.EtcdServers)
	eEventClient.SetConsistency(etcd.STRONG_CONSISTENCY)
	eEventStream := registry.NewEventStream(eEventClient, reg)

	e := engine.New(reg, eEventStream, mach)

	return &Server{a, e}, nil
}

func (self *Server) MarshalJSON() ([]byte, error) {
	return json.Marshal(struct{ Agent *agent.Agent }{Agent: self.agent})
}

func (self *Server) Run() {
	self.agent.Run()
	self.engine.Run()
}

func (self *Server) Stop() {
	self.agent.Stop()
	self.engine.Stop()
}

func (self *Server) Purge() {
	self.agent.Purge()
}
<commit_msg>refactor(server): break apart server.New<commit_after>package server

import (
	"encoding/json"

	"github.com/coreos/fleet/third_party/github.com/coreos/go-etcd/etcd"
	log "github.com/coreos/fleet/third_party/github.com/golang/glog"

	"github.com/coreos/fleet/agent"
	"github.com/coreos/fleet/config"
	"github.com/coreos/fleet/engine"
	"github.com/coreos/fleet/machine"
	"github.com/coreos/fleet/registry"
	"github.com/coreos/fleet/sign"
)

type Server struct {
	agent  *agent.Agent
	engine *engine.Engine
}

func New(cfg config.Config) (*Server, error) {
	a, err := newAgentFromConfig(cfg)
	if err != nil {
		return nil, err
	}

	e := newEngineFromConfig(cfg)

	return &Server{a, e}, nil
}

func newAgentFromConfig(cfg config.Config) (*agent.Agent, error) {
	mach := machine.New(cfg.BootID, cfg.PublicIP, cfg.Metadata())

	regClient := etcd.NewClient(cfg.EtcdServers)
	regClient.SetConsistency(etcd.STRONG_CONSISTENCY)
	reg := registry.New(regClient)

	eClient := etcd.NewClient(cfg.EtcdServers)
	eClient.SetConsistency(etcd.STRONG_CONSISTENCY)
	eStream := registry.NewEventStream(eClient, reg)

	var verifier *sign.SignatureVerifier
	if cfg.VerifyUnits {
		var err error
		verifier, err = sign.NewSignatureVerifierFromAuthorizedKeysFile(cfg.AuthorizedKeysFile)
		if err != nil {
			log.Errorln("Failed to get any key from authorized key file in verify_units mode:", err)
			verifier = sign.NewSignatureVerifier()
		}
	}

	return agent.New(reg, eStream, mach, cfg.AgentTTL, verifier)
}

func newEngineFromConfig(cfg config.Config) *engine.Engine {
	mach := machine.New(cfg.BootID, cfg.PublicIP, cfg.Metadata())

	regClient := etcd.NewClient(cfg.EtcdServers)
	regClient.SetConsistency(etcd.STRONG_CONSISTENCY)
	reg := registry.New(regClient)

	eClient := etcd.NewClient(cfg.EtcdServers)
	eClient.SetConsistency(etcd.STRONG_CONSISTENCY)
	eStream := registry.NewEventStream(eClient, reg)

	return engine.New(reg, eStream, mach)
}

func (self *Server) Run() {
	self.agent.Run()
	self.engine.Run()
}

func (self *Server) Stop() {
	self.agent.Stop()
	self.engine.Stop()
}

func (self *Server) Purge() {
	self.agent.Purge()
}

func (self *Server) MarshalJSON() ([]byte, error) {
	return json.Marshal(struct{ Agent *agent.Agent }{Agent: self.agent})
}
<|endoftext|>


<commit_before>// chat room example
package server

import (
	"fmt"
	"log"
	"net/http"

	"github.com/fzzy/sockjs-go/sockjs"
)

func chatHandler(s sockjs.Session) {

	client := login(s)
	if err := clients.Add(client); err != nil {
		client.Send(client, []byte(err.Error()))
		chatHandler(s)
		return
	}
	defer clients.Remove(client)
	client.Send(client, []byte(fmt.Sprintf("Welcome, %s.", client.Name)))

	for {
		m := s.Receive()
		if m == nil {
			break
		}
		m = []byte(fmt.Sprintf("%s: %s", client.Name, m))
		clients.Broadcast(client, m)
	}
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "./static/index.html")
}

func Start() {
	mux := sockjs.NewServeMux(http.DefaultServeMux)
	conf := sockjs.NewConfig()
	http.Handle("/static", http.FileServer(http.Dir("./static")))
	http.HandleFunc("/", indexHandler)
	mux.Handle("/chat", chatHandler, conf)

	log.Println("The server is up an running at http://0.0.0.0:8081")
	err := http.ListenAndServe("0.0.0.0:8081", mux)
	if err != nil {
		fmt.Println(err)
	}
}
<commit_msg>Make anonymous clients and broadcast from them<commit_after>// chat room example
package server

import (
	"fmt"
	"log"
	"net/http"

	"github.com/fzzy/sockjs-go/sockjs"
)

func chatHandler(s sockjs.Session) {

	client := login(s)
	if err := clients.Add(client); err != nil {
		client.Send(new(Client), []byte(err.Error()))
		return
	}
	defer clients.Remove(client)
	client.Send(new(Client), []byte(fmt.Sprintf("Welcome, %s.", client.Name)))

	for {
		m := s.Receive()
		if m == nil {
			break
		}
		m = []byte(fmt.Sprintf("%s: %s", client.Name, m))
		clients.Broadcast(client, m)
	}
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "./static/index.html")
}

func Start() {
	mux := sockjs.NewServeMux(http.DefaultServeMux)
	conf := sockjs.NewConfig()
	http.Handle("/static", http.FileServer(http.Dir("./static")))
	http.HandleFunc("/", indexHandler)
	mux.Handle("/chat", chatHandler, conf)

	log.Println("The server is up an running at http://0.0.0.0:8081")
	err := http.ListenAndServe("0.0.0.0:8081", mux)
	if err != nil {
		fmt.Println(err)
	}
}
<|endoftext|>


<commit_before>package main

/*import "fmt"
import "strings"
import "net"  //now look at this net that I just found
import "time" //time is a tool you can put on the wall or wear it on your wrist
//the past is far behind us, the future doesn't exist
import "./players"

func getPlayerOfIp(ip string) players.NetworkPlayer {
	for _, plr := range networkPlayerList {
		plrIp := plr.GetIp()
		if plrIp == ip[:strings.Index(ip, ":")] {
			return plr
		}
	}
	//for now, add a player to the list and return it
	networkPlayerList=append(networkPlayerList,players.NewNetworkPlayer("uwe",3,ip))
	return networkPlayerList[len(networkPlayerList)-1]
}

func handleConn1(conn net.Conn) {
	fmt.Println("conn 1")
	time.Sleep(time.Second / 2)
	for {
		conn.Write([]byte{65})
	}
	fmt.Println(conn.RemoteAddr().String())
	plr := getPlayerOfIp(conn.RemoteAddr().String())
	for len(plr.GetDataToBeSent()) == 0 {
		time.Sleep(time.Second / 2)
	}
	_, err := conn.Write(plr.GetDataToBeSent()[0])
	if err == nil {
		plr.RemoveASentData()
	}
}

func handleConn2(conn net.Conn) {
	fmt.Println("conn 2")
	fmt.Println(conn.RemoteAddr().String())
	var chat []byte
	_, err := conn.Read(chat)
	fmt.Println(chat,err)
	if err != nil {
		return
	}
	chat = append([]byte("CHAT"), chat...)
	plr := getPlayerOfIp(conn.RemoteAddr().String())
	for _, plr2 := range networkPlayerList {
		if plr2 != plr {
			plr2.SendData(chat)
		}
	}
}

func checkLoop1(quitChan chan struct{}) {
	ln, _ := net.Listen("tcp", ":5252") //event listeners
	for {
		select {
		case <-quitChan:
			fmt.Println("65")
			return
		default:
			conn, _ := ln.Accept()
			go handleConn1(conn)
		}
	}
}
func checkLoop2(quitChan chan struct{}) {
	ln, _ := net.Listen("tcp", ":6565") //chat
	for {
		select {
		case <-quitChan:
			fmt.Println("65")
			return
		default:
			conn, _ := ln.Accept()
			go handleConn2(conn)
		}
	}
}

func setupServer(quitChan chan struct{}) {
	go checkLoop1(quitChan)
	go checkLoop2(quitChan)
	fmt.Println("server loaded")
}
*/

import "net/http"
import "fmt"
import "html"
import "time"
import "io/ioutil"
import "strings"
import "./players"
func getPlayerOfIp(ip string) players.NetworkPlayer {
	for _, plr := range networkPlayerList {
		plrIp := plr.GetIp()
		if plrIp == ip[:strings.Index(ip, ":")] {
			return plr
		}
	}
	//for now, add a player to the list and return it
	networkPlayerList=append(networkPlayerList,players.NewNetworkPlayer("uwe",3,ip))
	return networkPlayerList[len(networkPlayerList)-1]
}
func fileServe(w http.ResponseWriter, r *http.Request) {
	fmt.Println("conn0")
	b,e := ioutil.ReadFile("client/"+html.EscapeString(r.URL.Path)[1:])
	fmt.Println(e)
	w.Write(b)
}
func eventFunc(w http.ResponseWriter, r *http.Request) {
	fmt.Println("conn1")
	fmt.Println(r.RemoteAddr)
	plr := getPlayerOfIp(r.RemoteAddr)
	for len(plr.GetDataToBeSent()) == 0 {
		time.Sleep(time.Second / 2)
	}
	_, err := w.Write(plr.GetDataToBeSent()[0])
	if err == nil {
		plr.RemoveASentData()
	}
}
func chatFunc(w http.ResponseWriter, r *http.Request) {
	fmt.Println("conn 2")
	fmt.Println(r.RemoteAddr)
	var chat []byte
	//chat = []byte("aaa")
	r.ParseForm()
	fmt.Println(r.Form)
	for key := range r.Form {
		chat=[]byte(key)
	}
	fmt.Println(chat)
	//fmt.Println(r.Header,r.Body)
	/*if err != nil {
		return
	}*/
	chat = append([]byte("CHAT"), chat...)
	plr := getPlayerOfIp(r.RemoteAddr)
	for _, plr2 := range networkPlayerList {
		if plr2 != plr {
			plr2.SendData(chat)
		}
	}
}
func setupServer(quitChan chan struct{}) {
	http.HandleFunc("/ordos.html",fileServe)
	http.HandleFunc("/ordos.js",fileServe)
	http.HandleFunc("/ordos.css",fileServe)
	http.HandleFunc("/event",eventFunc)
	http.HandleFunc("/chat",chatFunc)
	http.ListenAndServe(":8081",nil)
}
<commit_msg>i think it works now<commit_after>package main

/*import "fmt"
import "strings"
import "net"  //now look at this net that I just found
import "time" //time is a tool you can put on the wall or wear it on your wrist
//the past is far behind us, the future doesn't exist
import "./players"

func getPlayerOfIp(ip string) players.NetworkPlayer {
	for _, plr := range networkPlayerList {
		plrIp := plr.GetIp()
		if plrIp == ip[:strings.Index(ip, ":")] {
			return plr
		}
	}
	//for now, add a player to the list and return it
	networkPlayerList=append(networkPlayerList,players.NewNetworkPlayer("uwe",3,ip))
	return networkPlayerList[len(networkPlayerList)-1]
}

func handleConn1(conn net.Conn) {
	fmt.Println("conn 1")
	time.Sleep(time.Second / 2)
	for {
		conn.Write([]byte{65})
	}
	fmt.Println(conn.RemoteAddr().String())
	plr := getPlayerOfIp(conn.RemoteAddr().String())
	for len(plr.GetDataToBeSent()) == 0 {
		time.Sleep(time.Second / 2)
	}
	_, err := conn.Write(plr.GetDataToBeSent()[0])
	if err == nil {
		plr.RemoveASentData()
	}
}

func handleConn2(conn net.Conn) {
	fmt.Println("conn 2")
	fmt.Println(conn.RemoteAddr().String())
	var chat []byte
	_, err := conn.Read(chat)
	fmt.Println(chat,err)
	if err != nil {
		return
	}
	chat = append([]byte("CHAT"), chat...)
	plr := getPlayerOfIp(conn.RemoteAddr().String())
	for _, plr2 := range networkPlayerList {
		if plr2 != plr {
			plr2.SendData(chat)
		}
	}
}

func checkLoop1(quitChan chan struct{}) {
	ln, _ := net.Listen("tcp", ":5252") //event listeners
	for {
		select {
		case <-quitChan:
			fmt.Println("65")
			return
		default:
			conn, _ := ln.Accept()
			go handleConn1(conn)
		}
	}
}
func checkLoop2(quitChan chan struct{}) {
	ln, _ := net.Listen("tcp", ":6565") //chat
	for {
		select {
		case <-quitChan:
			fmt.Println("65")
			return
		default:
			conn, _ := ln.Accept()
			go handleConn2(conn)
		}
	}
}

func setupServer(quitChan chan struct{}) {
	go checkLoop1(quitChan)
	go checkLoop2(quitChan)
	fmt.Println("server loaded")
}
*/

import "net/http"
import "fmt"
import "html"
import "time"
import "io/ioutil"
import "strings"
import "./players"
func getPlayerOfIp(ip string) players.NetworkPlayer {
	for _, plr := range networkPlayerList {
		plrIp := plr.GetIp()
		if plrIp == ip[:strings.Index(ip, ":")] {
			return plr
		}
	}
	//for now, add a player to the list and return it
	networkPlayerList=append(networkPlayerList,players.NewNetworkPlayer("uwe",3,ip))
	return networkPlayerList[len(networkPlayerList)-1]
}
func fileServe(w http.ResponseWriter, r *http.Request) {
	fmt.Println("conn0")
	b,e := ioutil.ReadFile("client/"+html.EscapeString(r.URL.Path)[1:])
	fmt.Println(e)
	w.Write(b)
}
func eventFunc(w http.ResponseWriter, r *http.Request) {
	fmt.Println("conn1")
	fmt.Println(r.RemoteAddr)
	plr := getPlayerOfIp(r.RemoteAddr)
	for len(plr.GetDataToBeSent()) == 0 {
		time.Sleep(time.Second / 2)
	}
	_, err := w.Write(plr.GetDataToBeSent()[0])
	if err == nil {
		plr.RemoveASentData()
	}
}
func chatFunc(w http.ResponseWriter, r *http.Request) {
	fmt.Println("conn 2")
	fmt.Println(r.RemoteAddr)
	chat := ""
	r.ParseForm()
	for key := range r.Form {
		chat=key
		break
	}
	plr := getPlayerOfIp(r.RemoteAddr)
	data := []byte("CHAT"+plr.name+": "+chat)
	for _, plr2 := range networkPlayerList {
		plr2.SendData(data)
	}
}
func setupServer(quitChan chan struct{}) {
	http.HandleFunc("/ordos.html",fileServe)
	http.HandleFunc("/ordos.js",fileServe)
	http.HandleFunc("/ordos.css",fileServe)
	http.HandleFunc("/event",eventFunc)
	http.HandleFunc("/chat",chatFunc)
	http.ListenAndServe(":8081",nil)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net/http"
	"time"

	"github.com/NYTimes/gziphandler"
	"github.com/justinas/alice"
	chttpd "github.com/nochso/colourl/http"
	log "github.com/sirupsen/logrus"
)

var (
	port    int
	verbose bool
)

var (
	Version   string
	BuildDate string
)

func main() {
	flag.IntVar(&port, "p", 9191, "HTTP listening port")
	flag.BoolVar(&verbose, "v", false, "Enable verbose / debug output")
	flag.Parse()
	log.SetFormatter(&log.TextFormatter{FullTimestamp: true})
	if verbose {
		log.SetLevel(log.DebugLevel)
	}
	log.WithFields(log.Fields{
		"version":    Version,
		"build_date": BuildDate,
	}).Info("colourl-http")
	log.WithFields(log.Fields{
		"port":    port,
		"verbose": verbose,
	}).Info("Starting HTTP server")

	mux := http.NewServeMux()
	mux.Handle("/", chttpd.IndexMux())
	mux.HandleFunc("/svg", chttpd.SVGHandler)
	h := alice.New(
		logHandler,
		gziphandler.GzipHandler,
	).Then(mux)

	panic(http.ListenAndServe(fmt.Sprintf(":%d", port), h))
}

func logHandler(fn http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		start := time.Now()
		fn.ServeHTTP(w, r)
		log.WithFields(log.Fields{
			"duration": time.Now().Sub(start),
			"url":      r.URL,
			"method":   r.Method,
			"remote":   r.RemoteAddr,
		}).Debug("HTTP request")
	})
}
<commit_msg>Extract newServer and newHandler<commit_after>package main

import (
	"flag"
	"fmt"
	"net/http"
	"time"

	"github.com/NYTimes/gziphandler"
	"github.com/justinas/alice"
	chttpd "github.com/nochso/colourl/http"
	log "github.com/sirupsen/logrus"
)

var (
	port    int
	verbose bool
)

var (
	Version   string
	BuildDate string
)

func main() {
	flag.IntVar(&port, "p", 9191, "HTTP listening port")
	flag.BoolVar(&verbose, "v", false, "Enable verbose / debug output")
	flag.Parse()
	log.SetFormatter(&log.TextFormatter{FullTimestamp: true})
	if verbose {
		log.SetLevel(log.DebugLevel)
	}
	log.WithFields(log.Fields{
		"version":    Version,
		"build_date": BuildDate,
	}).Info("colourl-http")

	srv := newServer()
	log.WithFields(log.Fields{
		"port":    port,
		"verbose": verbose,
	}).Info("Starting HTTP server")
	log.Fatal(srv.ListenAndServe())
}

func newServer() *http.Server {
	return &http.Server{
		Handler:        newHandler(),
		Addr:           fmt.Sprintf(":%d", port),
		ReadTimeout:    time.Second * 5,
		WriteTimeout:   time.Second * 10,
		IdleTimeout:    time.Second * 60,
		MaxHeaderBytes: 1 << 17, // 128kB
	}
}

func newHandler() http.Handler {
	mux := http.NewServeMux()
	mux.Handle("/", chttpd.IndexMux())
	mux.HandleFunc("/svg", chttpd.SVGHandler)
	return alice.New(
		logHandler,
		gziphandler.GzipHandler,
	).Then(mux)
}

func logHandler(fn http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		start := time.Now()
		fn.ServeHTTP(w, r)
		log.WithFields(log.Fields{
			"duration": time.Now().Sub(start),
			"url":      r.URL,
			"method":   r.Method,
			"remote":   r.RemoteAddr,
		}).Debug("HTTP request")
	})
}
<|endoftext|>