<commit_before>package cmds

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"testing"

	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	"github.com/spf13/cobra"
)

const badJSON1 = `
{
"356weryt

}
`

const badJSON2 = `
{
    "a": 1,
    "b": [23,4,4,64,56,36,7456,7],
    "c": {a,f,g,h,j,j},
    "d": 3452.36456,
}
`

func testJSONSyntaxErrorsReported(inputFile string, inputFileValue string, inputCommand []string) {
	address := "0.0.0.0:30650"

	rootCmd := &cobra.Command{
		Use: os.Args[0],
		Long: `Access the Pachyderm API.

Envronment variables:
  ADDRESS=0.0.0.0:30650, the server to connect to.
`,
	}

	cmds, _ := Cmds(address)
	for _, cmd := range cmds {
		rootCmd.AddCommand(cmd)
	}

	ioutil.WriteFile(inputFile, []byte(inputFileValue), 0644)
	os.Args = inputCommand
	rootCmd.Execute()
}

func testBadJSON(t *testing.T, testName string, inputFile string, inputFileValue string, inputCommand []string, expectedOutput string) {

	if os.Getenv("BE_CRASHER") == "1" {
		testJSONSyntaxErrorsReported(inputFile, inputFileValue, inputCommand)
		return
	}

	cmd := exec.Command(os.Args[0], fmt.Sprintf("-test.run=%v", testName))
	cmd.Env = append(os.Environ(), "BE_CRASHER=1")
	out, err := cmd.CombinedOutput()

	// Do our cleanup here, since we have an exit 1 in the actual run:
	wd, _ := os.Getwd()
	fileName := filepath.Join(wd, inputFile)
	os.Remove(fileName)

	require.YesError(t, err)
	if e, ok := err.(*exec.ExitError); ok && !e.Success() {
		require.Equal(t, expectedOutput, string(out))
		return
	}
	t.Fatalf("process ran with err %v, want exit status 1", err)

}

func TestJSONSyntaxErrorsReportedCreateJob(t *testing.T) {
	descriptiveOutput := `error parsing job spec: Syntax Error on line 3:

"356weryt

         ^
invalid character '\n' in string literal
`
	cmd := []string{"pachctl", "create-job", "-f", "bad1.json"}
	testBadJSON(t, "TestJSONSyntaxErrorsReportedCreateJob", "bad1.json", badJSON1, cmd, descriptiveOutput)
}

func TestJSONSyntaxErrorsReportedCreateJob2(t *testing.T) {
	descriptiveOutput := `error parsing job spec: Syntax Error on line 5:

    "c": {a
          ^
invalid character 'a' looking for beginning of object key string
`
	cmd := []string{"pachctl", "create-job", "-f", "bad2.json"}
	testBadJSON(t, "TestJSONSyntaxErrorsReportedCreateJob2", "bad2.json", badJSON2, cmd, descriptiveOutput)
}

func TestJSONSyntaxErrorsReportedCreatePipeline(t *testing.T) {
	descriptiveOutput := `error parsing pipeline spec: Syntax Error on line 5:

    "c": {a
          ^
invalid character 'a' looking for beginning of object key string
`
	cmd := []string{"pachctl", "create-pipeline", "-f", "bad2.json"}
	testBadJSON(t, "TestJSONSyntaxErrorsReportedCreatePipeline", "bad2.json", badJSON2, cmd, descriptiveOutput)
}

func TestJSONSyntaxErrorsReportedRunPipeline(t *testing.T) {
	descriptiveOutput := `error parsing pipeline spec: Syntax Error on line 5:

    "c": {a
          ^
invalid character 'a' looking for beginning of object key string
`
	cmd := []string{"pachctl", "run-pipeline", "-f", "bad2.json", "somePipelineName"}
	testBadJSON(t, "TestJSONSyntaxErrorsReportedRunPipeline", "bad2.json", badJSON2, cmd, descriptiveOutput)
}

func TestJSONSyntaxErrorsReportedCreatePipelineFromStdin(t *testing.T) {
	descriptiveOutput := `Reading from stdin.
error parsing pipeline spec: Syntax Error on line 5:

    "c": {a
          ^
invalid character 'a' looking for beginning of object key string
`
	rawCmd := []string{"pachctl", "create-pipeline"}
	testName := "TestJSONSyntaxErrorsReportedCreatePipelineFromStdin"

	if os.Getenv("BE_CRASHER") == "1" {
		address := "0.0.0.0:30650"
		rootCmd := &cobra.Command{
			Use: os.Args[0],
			Long: `Access the Pachyderm API.

Envronment variables:
  ADDRESS=0.0.0.0:30650, the server to connect to.
`,
		}

		cmds, _ := Cmds(address)
		for _, cmd := range cmds {
			rootCmd.AddCommand(cmd)
		}

		os.Args = rawCmd
		ioutil.WriteFile("bad2.json", []byte(badJSON2), 0644)
		os.Stdin, _ = os.Open("bad2.json")
		rootCmd.Execute()
		return
	}

	cmd := exec.Command(os.Args[0], fmt.Sprintf("-test.run=%v", testName))
	cmd.Env = append(os.Environ(), "BE_CRASHER=1")
	out, err := cmd.CombinedOutput()

	// Put our cleanup here, since we have an exit 1 in the actual run:

	wd, _ := os.Getwd()
	os.Remove(filepath.Join(wd, "bad2.json"))

	require.YesError(t, err)
	if e, ok := err.(*exec.ExitError); ok && !e.Success() {
		require.Equal(t, descriptiveOutput, string(out))
		return
	}
	t.Fatalf("process ran with err %v, want exit status 1", err)

}
<commit_msg>Fix json error tests.<commit_after>package cmds

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"testing"

	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	"github.com/spf13/cobra"
)

const badJSON1 = `
{
"356weryt

}
`

const badJSON2 = `
{
    "a": 1,
    "b": [23,4,4,64,56,36,7456,7],
    "c": {a,f,g,h,j,j},
    "d": 3452.36456,
}
`

func testJSONSyntaxErrorsReported(inputFile string, inputFileValue string, inputCommand []string) {
	address := "0.0.0.0:30650"

	rootCmd := &cobra.Command{
		Use: os.Args[0],
		Long: `Access the Pachyderm API.

Envronment variables:
  ADDRESS=0.0.0.0:30650, the server to connect to.
`,
	}

	cmds, _ := Cmds(address)
	for _, cmd := range cmds {
		rootCmd.AddCommand(cmd)
	}

	ioutil.WriteFile(inputFile, []byte(inputFileValue), 0644)
	os.Args = inputCommand
	rootCmd.Execute()
}

func testBadJSON(t *testing.T, testName string, inputFile string, inputFileValue string, inputCommand []string, expectedOutput string) {

	if os.Getenv("BE_CRASHER") == "1" {
		testJSONSyntaxErrorsReported(inputFile, inputFileValue, inputCommand)
		return
	}

	cmd := exec.Command(os.Args[0], fmt.Sprintf("-test.run=%v", testName))
	cmd.Env = append(os.Environ(), "BE_CRASHER=1")
	out, err := cmd.CombinedOutput()

	// Do our cleanup here, since we have an exit 1 in the actual run:
	wd, _ := os.Getwd()
	fileName := filepath.Join(wd, inputFile)
	os.Remove(fileName)

	require.YesError(t, err)
	if e, ok := err.(*exec.ExitError); ok && !e.Success() {
		fmt.Printf("expectedOutput: %s\nout: %s\n", expectedOutput, string(out))
		require.Equal(t, expectedOutput, string(out))
		return
	}
	t.Fatalf("process ran with err %v, want exit status 1", err)

}

func TestJSONSyntaxErrorsReportedCreateJob(t *testing.T) {
	descriptiveOutput := `Syntax Error on line 3:

"356weryt

         ^
invalid character '\n' in string literal
`
	cmd := []string{"pachctl", "create-job", "-f", "bad1.json"}
	testBadJSON(t, "TestJSONSyntaxErrorsReportedCreateJob", "bad1.json", badJSON1, cmd, descriptiveOutput)
}

func TestJSONSyntaxErrorsReportedCreateJob2(t *testing.T) {
	descriptiveOutput := `Syntax Error on line 5:

    "c": {a
          ^
invalid character 'a' looking for beginning of object key string
`
	cmd := []string{"pachctl", "create-job", "-f", "bad2.json"}
	testBadJSON(t, "TestJSONSyntaxErrorsReportedCreateJob2", "bad2.json", badJSON2, cmd, descriptiveOutput)
}

func TestJSONSyntaxErrorsReportedCreatePipeline(t *testing.T) {
	descriptiveOutput := `Syntax Error on line 5:

    "c": {a
          ^
invalid character 'a' looking for beginning of object key string
`
	cmd := []string{"pachctl", "create-pipeline", "-f", "bad2.json"}
	testBadJSON(t, "TestJSONSyntaxErrorsReportedCreatePipeline", "bad2.json", badJSON2, cmd, descriptiveOutput)
}

func TestJSONSyntaxErrorsReportedRunPipeline(t *testing.T) {
	descriptiveOutput := `Syntax Error on line 5:

    "c": {a
          ^
invalid character 'a' looking for beginning of object key string
`
	cmd := []string{"pachctl", "run-pipeline", "-f", "bad2.json", "somePipelineName"}
	testBadJSON(t, "TestJSONSyntaxErrorsReportedRunPipeline", "bad2.json", badJSON2, cmd, descriptiveOutput)
}

func TestJSONSyntaxErrorsReportedCreatePipelineFromStdin(t *testing.T) {
	descriptiveOutput := `Reading from stdin.
Syntax Error on line 5:

    "c": {a
          ^
invalid character 'a' looking for beginning of object key string
`
	rawCmd := []string{"pachctl", "create-pipeline"}
	testName := "TestJSONSyntaxErrorsReportedCreatePipelineFromStdin"

	if os.Getenv("BE_CRASHER") == "1" {
		address := "0.0.0.0:30650"
		rootCmd := &cobra.Command{
			Use: os.Args[0],
			Long: `Access the Pachyderm API.

Envronment variables:
  ADDRESS=0.0.0.0:30650, the server to connect to.
`,
		}

		cmds, _ := Cmds(address)
		for _, cmd := range cmds {
			rootCmd.AddCommand(cmd)
		}

		os.Args = rawCmd
		ioutil.WriteFile("bad2.json", []byte(badJSON2), 0644)
		os.Stdin, _ = os.Open("bad2.json")
		rootCmd.Execute()
		return
	}

	cmd := exec.Command(os.Args[0], fmt.Sprintf("-test.run=%v", testName))
	cmd.Env = append(os.Environ(), "BE_CRASHER=1")
	out, err := cmd.CombinedOutput()

	// Put our cleanup here, since we have an exit 1 in the actual run:

	wd, _ := os.Getwd()
	os.Remove(filepath.Join(wd, "bad2.json"))

	require.YesError(t, err)
	if e, ok := err.(*exec.ExitError); ok && !e.Success() {
		require.Equal(t, descriptiveOutput, string(out))
		return
	}
	t.Fatalf("process ran with err %v, want exit status 1", err)

}
<|endoftext|>


<commit_before>// Package request abstracts a client's request so that all middleware
// will handle them in an unified way.
package request

import (
	"net"
	"strings"

	"github.com/coredns/coredns/middleware/pkg/edns"

	"github.com/miekg/dns"
)

// Request contains some connection state and is useful in middleware.
type Request struct {
	Req *dns.Msg
	W   dns.ResponseWriter

	// Cache size after first call to Size or Do.
	size int
	do   int // 0: not, 1: true: 2: false
	// TODO(miek): opt record itself as well?

	// Cache lowercase qname.
	name string
}

// NewWithQuestion returns a new request based on the old, but with a new question
// section in the request.
func (r *Request) NewWithQuestion(name string, typ uint16) Request {
	req1 := Request{W: r.W, Req: r.Req.Copy()}
	req1.Req.Question[0] = dns.Question{Name: dns.Fqdn(name), Qclass: dns.ClassINET, Qtype: typ}
	return req1
}

// IP gets the (remote) IP address of the client making the request.
func (r *Request) IP() string {
	ip, _, err := net.SplitHostPort(r.W.RemoteAddr().String())
	if err != nil {
		return r.W.RemoteAddr().String()
	}
	return ip
}

// Port gets the (remote) Port of the client making the request.
func (r *Request) Port() string {
	_, port, err := net.SplitHostPort(r.W.RemoteAddr().String())
	if err != nil {
		return "0"
	}
	return port
}

// RemoteAddr returns the net.Addr of the client that sent the current request.
func (r *Request) RemoteAddr() string {
	return r.W.RemoteAddr().String()
}

// Proto gets the protocol used as the transport. This will be udp or tcp.
func (r *Request) Proto() string { return Proto(r.W) }

// Proto gets the protocol used as the transport. This will be udp or tcp.
func Proto(w dns.ResponseWriter) string {
	// FIXME(miek): why not a method on Request
	if _, ok := w.RemoteAddr().(*net.UDPAddr); ok {
		return "udp"
	}
	if _, ok := w.RemoteAddr().(*net.TCPAddr); ok {
		return "tcp"
	}
	return "udp"
}

// Family returns the family of the transport, 1 for IPv4 and 2 for IPv6.
func (r *Request) Family() int {
	var a net.IP
	ip := r.W.RemoteAddr()
	if i, ok := ip.(*net.UDPAddr); ok {
		a = i.IP
	}
	if i, ok := ip.(*net.TCPAddr); ok {
		a = i.IP
	}

	if a.To4() != nil {
		return 1
	}
	return 2
}

// Do returns if the request has the DO (DNSSEC OK) bit set.
func (r *Request) Do() bool {
	if r.do != 0 {
		return r.do == doTrue
	}

	if o := r.Req.IsEdns0(); o != nil {
		if o.Do() {
			r.do = doTrue
		} else {
			r.do = doFalse
		}
		return o.Do()
	}
	r.do = doFalse
	return false
}

// Len returns the length in bytes in the request.
func (r *Request) Len() int { return r.Req.Len() }

// Size returns if buffer size *advertised* in the requests OPT record.
// Or when the request was over TCP, we return the maximum allowed size of 64K.
func (r *Request) Size() int {
	if r.size != 0 {
		return r.size
	}

	size := 0
	if o := r.Req.IsEdns0(); o != nil {
		if o.Do() {
			r.do = doTrue
		} else {
			r.do = doFalse
		}
		size = int(o.UDPSize())
	}
	// TODO(miek) move edns.Size to dnsutil?
	size = edns.Size(r.Proto(), size)
	r.size = size
	return size
}

// SizeAndDo adds an OPT record that the reflects the intent from request.
// The returned bool indicated if an record was found and normalised.
func (r *Request) SizeAndDo(m *dns.Msg) bool {
	o := r.Req.IsEdns0() // TODO(miek): speed this up
	if o == nil {
		return false
	}

	odo := o.Do()

	if mo := m.IsEdns0(); mo != nil {
		mo.Hdr.Name = "."
		mo.Hdr.Rrtype = dns.TypeOPT
		mo.SetVersion(0)
		mo.SetUDPSize(o.UDPSize())
		mo.Hdr.Ttl &= 0xff00 // clear flags

		if odo {
			mo.SetDo()
		}
		return true
	}

	o.Hdr.Name = "."
	o.Hdr.Rrtype = dns.TypeOPT
	o.SetVersion(0)
	o.Hdr.Ttl &= 0xff00 // clear flags

	if odo {
		o.SetDo()
	}

	m.Extra = append(m.Extra, o)
	return true
}

// Result is the result of Scrub.
type Result int

const (
	// ScrubIgnored is returned when Scrub did nothing to the message.
	ScrubIgnored Result = iota
	// ScrubDone is returned when the reply has been scrubbed.
	ScrubDone
)

// Scrub scrubs the reply message so that it will fit the client's buffer. If even after dropping
// the additional section, it still does not fit the TC bit will be set on the message. Note,
// the TC bit will be set regardless of protocol, even TCP message will get the bit, the client
// should then retry with pigeons.
// TODO(referral).
func (r *Request) Scrub(reply *dns.Msg) (*dns.Msg, Result) {
	size := r.Size()
	l := reply.Len()
	if size >= l {
		return reply, ScrubIgnored
	}
	// TODO(miek): check for delegation

	// If not delegation, drop additional section.
	reply.Extra = nil
	r.SizeAndDo(reply)
	l = reply.Len()
	if size >= l {
		return reply, ScrubDone
	}
	// Still?!! does not fit.
	reply.Truncated = true
	return reply, ScrubDone
}

// Type returns the type of the question as a string.
func (r *Request) Type() string { return dns.Type(r.Req.Question[0].Qtype).String() }

// QType returns the type of the question as an uint16.
func (r *Request) QType() uint16 { return r.Req.Question[0].Qtype }

// Name returns the name of the question in the request. Note
// this name will always have a closing dot and will be lower cased. After a call Name
// the value will be cached. To clear this caching call Clear.
func (r *Request) Name() string {
	if r.name != "" {
		return r.name
	}
	r.name = strings.ToLower(dns.Name(r.Req.Question[0].Name).String())
	return r.name
}

// QName returns the name of the question in the request.
func (r *Request) QName() string { return dns.Name(r.Req.Question[0].Name).String() }

// Class returns the class of the question in the request.
func (r *Request) Class() string { return dns.Class(r.Req.Question[0].Qclass).String() }

// QClass returns the class of the question in the request.
func (r *Request) QClass() uint16 { return r.Req.Question[0].Qclass }

// ErrorMessage returns an error message suitable for sending
// back to the client.
func (r *Request) ErrorMessage(rcode int) *dns.Msg {
	m := new(dns.Msg)
	m.SetRcode(r.Req, rcode)
	return m
}

// Clear clears all caching from Request s.
func (r *Request) Clear() {
	r.name = ""
}

const (
	// TODO(miek): make this less awkward.
	doTrue  = 1
	doFalse = 2
)
<commit_msg>core: add Zone in request.Request (#857)<commit_after>// Package request abstracts a client's request so that all middleware
// will handle them in an unified way.
package request

import (
	"net"
	"strings"

	"github.com/coredns/coredns/middleware/pkg/edns"

	"github.com/miekg/dns"
)

// Request contains some connection state and is useful in middleware.
type Request struct {
	Req *dns.Msg
	W   dns.ResponseWriter

	// Optional lowercased zone of this query.
	Zone string

	// Cache size after first call to Size or Do.
	size int
	do   int // 0: not, 1: true: 2: false
	// TODO(miek): opt record itself as well?

	// Cache lowercase qname.
	name string
}

// NewWithQuestion returns a new request based on the old, but with a new question
// section in the request.
func (r *Request) NewWithQuestion(name string, typ uint16) Request {
	req1 := Request{W: r.W, Req: r.Req.Copy()}
	req1.Req.Question[0] = dns.Question{Name: dns.Fqdn(name), Qclass: dns.ClassINET, Qtype: typ}
	return req1
}

// IP gets the (remote) IP address of the client making the request.
func (r *Request) IP() string {
	ip, _, err := net.SplitHostPort(r.W.RemoteAddr().String())
	if err != nil {
		return r.W.RemoteAddr().String()
	}
	return ip
}

// Port gets the (remote) Port of the client making the request.
func (r *Request) Port() string {
	_, port, err := net.SplitHostPort(r.W.RemoteAddr().String())
	if err != nil {
		return "0"
	}
	return port
}

// RemoteAddr returns the net.Addr of the client that sent the current request.
func (r *Request) RemoteAddr() string {
	return r.W.RemoteAddr().String()
}

// Proto gets the protocol used as the transport. This will be udp or tcp.
func (r *Request) Proto() string { return Proto(r.W) }

// Proto gets the protocol used as the transport. This will be udp or tcp.
func Proto(w dns.ResponseWriter) string {
	// FIXME(miek): why not a method on Request
	if _, ok := w.RemoteAddr().(*net.UDPAddr); ok {
		return "udp"
	}
	if _, ok := w.RemoteAddr().(*net.TCPAddr); ok {
		return "tcp"
	}
	return "udp"
}

// Family returns the family of the transport, 1 for IPv4 and 2 for IPv6.
func (r *Request) Family() int {
	var a net.IP
	ip := r.W.RemoteAddr()
	if i, ok := ip.(*net.UDPAddr); ok {
		a = i.IP
	}
	if i, ok := ip.(*net.TCPAddr); ok {
		a = i.IP
	}

	if a.To4() != nil {
		return 1
	}
	return 2
}

// Do returns if the request has the DO (DNSSEC OK) bit set.
func (r *Request) Do() bool {
	if r.do != 0 {
		return r.do == doTrue
	}

	if o := r.Req.IsEdns0(); o != nil {
		if o.Do() {
			r.do = doTrue
		} else {
			r.do = doFalse
		}
		return o.Do()
	}
	r.do = doFalse
	return false
}

// Len returns the length in bytes in the request.
func (r *Request) Len() int { return r.Req.Len() }

// Size returns if buffer size *advertised* in the requests OPT record.
// Or when the request was over TCP, we return the maximum allowed size of 64K.
func (r *Request) Size() int {
	if r.size != 0 {
		return r.size
	}

	size := 0
	if o := r.Req.IsEdns0(); o != nil {
		if o.Do() {
			r.do = doTrue
		} else {
			r.do = doFalse
		}
		size = int(o.UDPSize())
	}
	// TODO(miek) move edns.Size to dnsutil?
	size = edns.Size(r.Proto(), size)
	r.size = size
	return size
}

// SizeAndDo adds an OPT record that the reflects the intent from request.
// The returned bool indicated if an record was found and normalised.
func (r *Request) SizeAndDo(m *dns.Msg) bool {
	o := r.Req.IsEdns0() // TODO(miek): speed this up
	if o == nil {
		return false
	}

	odo := o.Do()

	if mo := m.IsEdns0(); mo != nil {
		mo.Hdr.Name = "."
		mo.Hdr.Rrtype = dns.TypeOPT
		mo.SetVersion(0)
		mo.SetUDPSize(o.UDPSize())
		mo.Hdr.Ttl &= 0xff00 // clear flags

		if odo {
			mo.SetDo()
		}
		return true
	}

	o.Hdr.Name = "."
	o.Hdr.Rrtype = dns.TypeOPT
	o.SetVersion(0)
	o.Hdr.Ttl &= 0xff00 // clear flags

	if odo {
		o.SetDo()
	}

	m.Extra = append(m.Extra, o)
	return true
}

// Result is the result of Scrub.
type Result int

const (
	// ScrubIgnored is returned when Scrub did nothing to the message.
	ScrubIgnored Result = iota
	// ScrubDone is returned when the reply has been scrubbed.
	ScrubDone
)

// Scrub scrubs the reply message so that it will fit the client's buffer. If even after dropping
// the additional section, it still does not fit the TC bit will be set on the message. Note,
// the TC bit will be set regardless of protocol, even TCP message will get the bit, the client
// should then retry with pigeons.
// TODO(referral).
func (r *Request) Scrub(reply *dns.Msg) (*dns.Msg, Result) {
	size := r.Size()
	l := reply.Len()
	if size >= l {
		return reply, ScrubIgnored
	}
	// TODO(miek): check for delegation

	// If not delegation, drop additional section.
	reply.Extra = nil
	r.SizeAndDo(reply)
	l = reply.Len()
	if size >= l {
		return reply, ScrubDone
	}
	// Still?!! does not fit.
	reply.Truncated = true
	return reply, ScrubDone
}

// Type returns the type of the question as a string.
func (r *Request) Type() string { return dns.Type(r.Req.Question[0].Qtype).String() }

// QType returns the type of the question as an uint16.
func (r *Request) QType() uint16 { return r.Req.Question[0].Qtype }

// Name returns the name of the question in the request. Note
// this name will always have a closing dot and will be lower cased. After a call Name
// the value will be cached. To clear this caching call Clear.
func (r *Request) Name() string {
	if r.name != "" {
		return r.name
	}
	r.name = strings.ToLower(dns.Name(r.Req.Question[0].Name).String())
	return r.name
}

// QName returns the name of the question in the request.
func (r *Request) QName() string { return dns.Name(r.Req.Question[0].Name).String() }

// Class returns the class of the question in the request.
func (r *Request) Class() string { return dns.Class(r.Req.Question[0].Qclass).String() }

// QClass returns the class of the question in the request.
func (r *Request) QClass() uint16 { return r.Req.Question[0].Qclass }

// ErrorMessage returns an error message suitable for sending
// back to the client.
func (r *Request) ErrorMessage(rcode int) *dns.Msg {
	m := new(dns.Msg)
	m.SetRcode(r.Req, rcode)
	return m
}

// Clear clears all caching from Request s.
func (r *Request) Clear() {
	r.name = ""
}

const (
	// TODO(miek): make this less awkward.
	doTrue  = 1
	doFalse = 2
)
<|endoftext|>


<commit_before>package consts

import "time"

// Optimizations
const (
	OPTIMIZE_LOCAL_ENTITIES = true // should be true for performance, set to false for testing
)

// Tunable Options
const (
	// For Underlying Networking
	// BUFFERED_READ_BUFFSIZE is the read buffer size for BufferedReadConnection
	BUFFERED_READ_BUFFSIZE = 16384
	// BUFFERED_WRITE_BUFFSIZE is the write buffer size for BufferedWriteConnection
	BUFFERED_WRITE_BUFFSIZE = 16384

	// For Packets Send & Recv
	// PACKET_PAYLOAD_LEN_COMPRESS_THRESHOLD is the minimal packet payload length that should be compressed
	PACKET_PAYLOAD_LEN_COMPRESS_THRESHOLD = 512

	// For Dispatcher
	// DISPATCHER_CLIENT_PROXY_WRITE_FLUSH_INTERVAL is the flush interval for client proxy. Smaller interval costs more CPU but dispatches patckets sooner
	DISPATCHER_CLIENT_PROXY_WRITE_FLUSH_INTERVAL = 5 * time.Millisecond
	// DISPATCHER_GC_PERCENT is the GC percent for dispatcher
	DISPATCHER_GC_PERCENT = 1000
	// DISPATCHER_CLIENT_PROXY_WRITE_BUFFER_SIZE is dispatcher client proxies' write buffer size
	DISPATCHER_CLIENT_PROXY_WRITE_BUFFER_SIZE = 1024 * 1024
	// DISPATCHER_CLIENT_PROXY_READ_BUFFER_SIZE is dispatcher client proxies' read buffer size
	DISPATCHER_CLIENT_PROXY_READ_BUFFER_SIZE = 1024 * 1024
	// GAME_PENDING_PACKET_QUEUE_MAX_LEN is the maxium number of packets in pending queue when game is blocked
	GAME_PENDING_PACKET_QUEUE_MAX_LEN = 1000000
	// ENTITY_PENDING_PACKET_QUEUE_MAX_LEN is the maxium number of packets in pending queue when entity is blocked
	ENTITY_PENDING_PACKET_QUEUE_MAX_LEN = 1000
	// MAX_ENTITY_SYNC_INFOS_CACHE_SIZE_PER_GAME is maxium number of bytes of entity sync info cached for each game
	MAX_ENTITY_SYNC_INFOS_CACHE_SIZE_PER_GAME = 1024 * 1024

	DISPATCHER_SERVICE_PACKET_QUEUE_SIZE = 10000

	// For Game Service
	// GAME_SERVICE_PACKET_QUEUE_SIZE is the max packet queue length for game service
	GAME_SERVICE_PACKET_QUEUE_SIZE = 10000 // packet queue size
	// GAME_SERVICE_TICK_INTERVAL is the tick interval to tick timers in game service
	GAME_SERVICE_TICK_INTERVAL = time.Millisecond * 10 // server tick interval => affect timer resolution

	// DISPATCHER_CLIENT_WRITE_BUFFER_SIZE is the writer buffer size for gates/games' connections to dispatcher
	DISPATCHER_CLIENT_WRITE_BUFFER_SIZE = 1024 * 1024
	// DISPATCHER_CLIENT_READ_BUFFER_SIZE is the read buffer size for gates/games' connections to dispatcher
	DISPATCHER_CLIENT_READ_BUFFER_SIZE = 1024 * 1024

	// For Gate Service
	// CLIENT_PROXY_WRITE_BUFFER_SIZE is the write buffer size for gates' client proxies
	CLIENT_PROXY_WRITE_BUFFER_SIZE = 1024 * 1024
	// CLIENT_PROXY_READ_BUFFER_SIZE is the read buffer size for gates' client proxies
	CLIENT_PROXY_READ_BUFFER_SIZE = 1024 * 1024
	// COMPRESS_WRITER_POOL_SIZE is number of write compressors in the pool for gate
	//COMPRESS_WRITER_POOL_SIZE = 100
	// CLIENT_PROXY_SET_TCP_NO_DELAY = true sets client proxies to TcpNoDelay
	CLIENT_PROXY_SET_TCP_NO_DELAY = true

	//SAVE_INTERVAL      = time.Minute * 5 // Save interval of entities

	// ENTER_SPACE_REQUEST_TIMEOUT is the timeout for enter space request
	ENTER_SPACE_REQUEST_TIMEOUT = DISPATCHER_MIGRATE_TIMEOUT + time.Minute // enter space should finish in limited seconds
	// DISPATCHER_MIGRATE_TIMEOUT is timeout for entity migration
	DISPATCHER_MIGRATE_TIMEOUT = time.Minute * 5
	// DISPATCHER_LOAD_TIMEOUT is timeout for loading entity
	DISPATCHER_LOAD_TIMEOUT = time.Minute
	// DISPATCHER_FREEZE_GAME_TIMEOUT is timeout for freezing & restoring game
	DISPATCHER_FREEZE_GAME_TIMEOUT = time.Minute * 5
	// For Storage
	// For Operation Monitor
	// OPMON_DUMP_INTERVAL is the interval to print opmon infos to output
	OPMON_DUMP_INTERVAL = 0

	// For Snappy Compress
	// MIN_DATA_SIZE_TO_COMPRESS is the minimal data size to compress
	MIN_DATA_SIZE_TO_COMPRESS = 1024

	// For UDP Connections between Gates and Clients
	// UDP_MAX_PACKET_PAYLOAD_SIZE is the max packet payload size of UDP packets. Since UDP are only used for sync, this value can be very small
	UDP_MAX_PACKET_PAYLOAD_SIZE = 128 // try to make sure that this value is smaller or equal to _MIN_PAYLOAD_CAP, so that no buffer needs to be allocated
)

// Debug Options
const (
	// DEBUG_PACKETS prints packet send/recv debug logs
	DEBUG_PACKETS = false
	// DEBUG_SPACES prints space operation debug logs
	DEBUG_SPACES = false
	// DEBUG_SAVE_LOAD prints save & load debug logs
	DEBUG_SAVE_LOAD = false
	// DEBUG_CLIENTS prints clients operation debug logs
	DEBUG_CLIENTS = false
	// DEBUG_MIGRATE prints migration debug logs
	DEBUG_MIGRATE = false
	// DEBUG_PACKET_ALLOC prints  packet allocation debug logs
	DEBUG_PACKET_ALLOC = false
	// DEBUG_FILTER_PROP prints filter props debug logs
	DEBUG_FILTER_PROP = false
)

//  System level configurations
const (
	// DEBUG_MODE = true turns on debug mode
	DEBUG_MODE = false
)

// Async & MongoDB configurations
const (
	ASYNC_JOB_QUEUE_MAXLEN  = 10000
	MONGODB_OP_QUEUE_MAXLEN = 10000
)

// KCP Options
const (
	KCP_NO_DELAY                       = 1  // Whether nodelay mode is enabled, 0 is not enabled; 1 enabled
	KCP_INTERNAL_UPDATE_TIMER_INTERVAL = 10 // Protocol internal work interval, in milliseconds, such as 10 ms or 20 ms.
	KCP_ENABLE_FAST_RESEND             = 2  // Fast retransmission mode, 0 represents off by default, 2 can be set (2 ACK spans will result in direct retransmission)
	KCP_DISABLE_CONGESTION_CONTROL     = 1  // Whether to turn off flow control, 0 represents “Do not turn off” by default, 1 represents “Turn off”.

	KCP_SET_STREAM_MODE  = true
	KCP_SET_WRITE_DELAY  = true
	KCP_SET_ACK_NO_DELAY = true
)

const (
	DISPATCHER_STARTED_TAG = "<!--XSUPERVISOR:BEGIN--> DISPATCHER STARTED <!--XSUPERVISOR:END-->"
	GAME_STARTED_TAG       = "<!--XSUPERVISOR:BEGIN--> GAME STARTED <!--XSUPERVISOR:END-->"
	GATE_STARTED_TAG       = "<!--XSUPERVISOR:BEGIN--> GATE STARTED <!--XSUPERVISOR:END-->"
)
<commit_msg>adjust params<commit_after>package consts

import "time"

// Optimizations
const (
	OPTIMIZE_LOCAL_ENTITIES = true // should be true for performance, set to false for testing
)

// Tunable Options
const (
	// For Underlying Networking
	// BUFFERED_READ_BUFFSIZE is the read buffer size for BufferedReadConnection
	BUFFERED_READ_BUFFSIZE = 16384
	// BUFFERED_WRITE_BUFFSIZE is the write buffer size for BufferedWriteConnection
	BUFFERED_WRITE_BUFFSIZE = 16384

	// For Packets Send & Recv
	// PACKET_PAYLOAD_LEN_COMPRESS_THRESHOLD is the minimal packet payload length that should be compressed
	PACKET_PAYLOAD_LEN_COMPRESS_THRESHOLD = 512

	// For Dispatcher
	// DISPATCHER_CLIENT_PROXY_WRITE_FLUSH_INTERVAL is the flush interval for client proxy. Smaller interval costs more CPU but dispatches patckets sooner
	DISPATCHER_CLIENT_PROXY_WRITE_FLUSH_INTERVAL = 5 * time.Millisecond
	// DISPATCHER_GC_PERCENT is the GC percent for dispatcher
	DISPATCHER_GC_PERCENT = 1000
	// DISPATCHER_CLIENT_PROXY_WRITE_BUFFER_SIZE is dispatcher client proxies' write buffer size
	DISPATCHER_CLIENT_PROXY_WRITE_BUFFER_SIZE = 1024 * 1024
	// DISPATCHER_CLIENT_PROXY_READ_BUFFER_SIZE is dispatcher client proxies' read buffer size
	DISPATCHER_CLIENT_PROXY_READ_BUFFER_SIZE = 1024 * 1024
	// GAME_PENDING_PACKET_QUEUE_MAX_LEN is the maxium number of packets in pending queue when game is blocked
	GAME_PENDING_PACKET_QUEUE_MAX_LEN = 1000000
	// ENTITY_PENDING_PACKET_QUEUE_MAX_LEN is the maxium number of packets in pending queue when entity is blocked
	ENTITY_PENDING_PACKET_QUEUE_MAX_LEN = 1000
	// MAX_ENTITY_SYNC_INFOS_CACHE_SIZE_PER_GAME is maxium number of bytes of entity sync info cached for each game
	MAX_ENTITY_SYNC_INFOS_CACHE_SIZE_PER_GAME = 1024 * 1024

	DISPATCHER_SERVICE_PACKET_QUEUE_SIZE = 10000

	// For Game Service
	// GAME_SERVICE_PACKET_QUEUE_SIZE is the max packet queue length for game service
	GAME_SERVICE_PACKET_QUEUE_SIZE = 10000 // packet queue size
	// GAME_SERVICE_TICK_INTERVAL is the tick interval to tick timers in game service
	GAME_SERVICE_TICK_INTERVAL = time.Millisecond * 10 // server tick interval => affect timer resolution

	// DISPATCHER_CLIENT_WRITE_BUFFER_SIZE is the writer buffer size for gates/games' connections to dispatcher
	DISPATCHER_CLIENT_WRITE_BUFFER_SIZE = 1024 * 1024
	// DISPATCHER_CLIENT_READ_BUFFER_SIZE is the read buffer size for gates/games' connections to dispatcher
	DISPATCHER_CLIENT_READ_BUFFER_SIZE = 1024 * 1024

	// For Gate Service
	// CLIENT_PROXY_WRITE_BUFFER_SIZE is the write buffer size for gates' client proxies
	CLIENT_PROXY_WRITE_BUFFER_SIZE = 1024 * 1024
	// CLIENT_PROXY_READ_BUFFER_SIZE is the read buffer size for gates' client proxies
	CLIENT_PROXY_READ_BUFFER_SIZE = 1024 * 1024
	// COMPRESS_WRITER_POOL_SIZE is number of write compressors in the pool for gate
	//COMPRESS_WRITER_POOL_SIZE = 100
	// CLIENT_PROXY_SET_TCP_NO_DELAY = true sets client proxies to TcpNoDelay
	CLIENT_PROXY_SET_TCP_NO_DELAY = true

	//SAVE_INTERVAL      = time.Minute * 5 // Save interval of entities

	// ENTER_SPACE_REQUEST_TIMEOUT is the timeout for enter space request
	ENTER_SPACE_REQUEST_TIMEOUT = DISPATCHER_MIGRATE_TIMEOUT + time.Minute // enter space should finish in limited seconds
	// DISPATCHER_MIGRATE_TIMEOUT is timeout for entity migration
	DISPATCHER_MIGRATE_TIMEOUT = time.Minute * 5
	// DISPATCHER_LOAD_TIMEOUT is timeout for loading entity
	DISPATCHER_LOAD_TIMEOUT = time.Minute
	// DISPATCHER_FREEZE_GAME_TIMEOUT is timeout for freezing & restoring game
	DISPATCHER_FREEZE_GAME_TIMEOUT = time.Minute * 5
	// For Storage
	// For Operation Monitor
	// OPMON_DUMP_INTERVAL is the interval to print opmon infos to output
	OPMON_DUMP_INTERVAL = 0

	// For Snappy Compress
	// MIN_DATA_SIZE_TO_COMPRESS is the minimal data size to compress
	MIN_DATA_SIZE_TO_COMPRESS = 512

	// For UDP Connections between Gates and Clients
	// UDP_MAX_PACKET_PAYLOAD_SIZE is the max packet payload size of UDP packets. Since UDP are only used for sync, this value can be very small
	UDP_MAX_PACKET_PAYLOAD_SIZE = 128 // try to make sure that this value is smaller or equal to _MIN_PAYLOAD_CAP, so that no buffer needs to be allocated
)

// Debug Options
const (
	// DEBUG_PACKETS prints packet send/recv debug logs
	DEBUG_PACKETS = false
	// DEBUG_SPACES prints space operation debug logs
	DEBUG_SPACES = false
	// DEBUG_SAVE_LOAD prints save & load debug logs
	DEBUG_SAVE_LOAD = false
	// DEBUG_CLIENTS prints clients operation debug logs
	DEBUG_CLIENTS = false
	// DEBUG_MIGRATE prints migration debug logs
	DEBUG_MIGRATE = false
	// DEBUG_PACKET_ALLOC prints  packet allocation debug logs
	DEBUG_PACKET_ALLOC = false
	// DEBUG_FILTER_PROP prints filter props debug logs
	DEBUG_FILTER_PROP = false
)

//  System level configurations
const (
	// DEBUG_MODE = true turns on debug mode
	DEBUG_MODE = false
)

// Async configurations
const (
	ASYNC_JOB_QUEUE_MAXLEN = 10000
)

// KCP Options
const (
	KCP_NO_DELAY                       = 1  // Whether nodelay mode is enabled, 0 is not enabled; 1 enabled
	KCP_INTERNAL_UPDATE_TIMER_INTERVAL = 10 // Protocol internal work interval, in milliseconds, such as 10 ms or 20 ms.
	KCP_ENABLE_FAST_RESEND             = 2  // Fast retransmission mode, 0 represents off by default, 2 can be set (2 ACK spans will result in direct retransmission)
	KCP_DISABLE_CONGESTION_CONTROL     = 1  // Whether to turn off flow control, 0 represents “Do not turn off” by default, 1 represents “Turn off”.

	KCP_SET_STREAM_MODE  = true
	KCP_SET_WRITE_DELAY  = true
	KCP_SET_ACK_NO_DELAY = true
)

const (
	DISPATCHER_STARTED_TAG = "<!--XSUPERVISOR:BEGIN--> DISPATCHER STARTED <!--XSUPERVISOR:END-->"
	GAME_STARTED_TAG       = "<!--XSUPERVISOR:BEGIN--> GAME STARTED <!--XSUPERVISOR:END-->"
	GATE_STARTED_TAG       = "<!--XSUPERVISOR:BEGIN--> GATE STARTED <!--XSUPERVISOR:END-->"
)
<|endoftext|>


<commit_before>/* Gorilla Sessions backend for MySQL.

Copyright (c) 2013 Contributors. See the list of contributors in the CONTRIBUTORS file for details.

This software is licensed under a MIT style license available in the LICENSE file.
*/
package sqlitestore

import (
	"database/sql"
	"encoding/gob"
	"errors"
	"fmt"
	"log"
	"net/http"
	"strings"
	"time"

	"github.com/gorilla/securecookie"
	"github.com/gorilla/sessions"
	_ "github.com/mattn/go-sqlite3"
)

type SqliteStore struct {
	db         DB
	stmtInsert *sql.Stmt
	stmtDelete *sql.Stmt
	stmtUpdate *sql.Stmt
	stmtSelect *sql.Stmt

	Codecs  []securecookie.Codec
	Options *sessions.Options
	table   string
}

type sessionRow struct {
	id         string
	data       string
	createdOn  time.Time
	modifiedOn time.Time
	expiresOn  time.Time
}

type DB interface {
	Exec(query string, args ...interface{}) (sql.Result, error)
	Prepare(query string) (*sql.Stmt, error)
	Close() error
}

func init() {
	gob.Register(time.Time{})
}

func NewSqliteStore(endpoint string, tableName string, path string, maxAge int, keyPairs ...[]byte) (*SqliteStore, error) {
	db, err := sql.Open("sqlite3", endpoint)
	if err != nil {
		return nil, err
	}

	return NewSqliteStoreFromConnection(db, tableName, path, maxAge, keyPairs...)
}

func NewSqliteStoreFromConnection(db DB, tableName string, path string, maxAge int, keyPairs ...[]byte) (*SqliteStore, error) {
	// Make sure table name is enclosed.
	tableName = "`" + strings.Trim(tableName, "`") + "`"

	cTableQ := "CREATE TABLE IF NOT EXISTS " +
		tableName + " (id INTEGER PRIMARY KEY, " +
		"session_data LONGBLOB, " +
		"created_on TIMESTAMP DEFAULT 0, " +
		"modified_on TIMESTAMP DEFAULT CURRENT_TIMESTAMP, " +
		"expires_on TIMESTAMP DEFAULT 0);"
	if _, err := db.Exec(cTableQ); err != nil {
		return nil, err
	}

	insQ := "INSERT INTO " + tableName +
		"(id, session_data, created_on, modified_on, expires_on) VALUES (NULL, ?, ?, ?, ?)"
	stmtInsert, stmtErr := db.Prepare(insQ)
	if stmtErr != nil {
		return nil, stmtErr
	}

	delQ := "DELETE FROM " + tableName + " WHERE id = ?"
	stmtDelete, stmtErr := db.Prepare(delQ)
	if stmtErr != nil {
		return nil, stmtErr
	}

	updQ := "UPDATE " + tableName + " SET session_data = ?, created_on = ?, expires_on = ? " +
		"WHERE id = ?"
	stmtUpdate, stmtErr := db.Prepare(updQ)
	if stmtErr != nil {
		return nil, stmtErr
	}

	selQ := "SELECT id, session_data, created_on, modified_on, expires_on from " +
		tableName + " WHERE id = ?"
	stmtSelect, stmtErr := db.Prepare(selQ)
	if stmtErr != nil {
		return nil, stmtErr
	}

	return &SqliteStore{
		db:         db,
		stmtInsert: stmtInsert,
		stmtDelete: stmtDelete,
		stmtUpdate: stmtUpdate,
		stmtSelect: stmtSelect,
		Codecs:     securecookie.CodecsFromPairs(keyPairs...),
		Options: &sessions.Options{
			Path:   path,
			MaxAge: maxAge,
		},
		table: tableName,
	}, nil
}

func (m *SqliteStore) Close() {
	m.stmtSelect.Close()
	m.stmtUpdate.Close()
	m.stmtDelete.Close()
	m.stmtInsert.Close()
	m.db.Close()
}

func (m *SqliteStore) Get(r *http.Request, name string) (*sessions.Session, error) {
	return sessions.GetRegistry(r).Get(m, name)
}

func (m *SqliteStore) New(r *http.Request, name string) (*sessions.Session, error) {
	session := sessions.NewSession(m, name)
	session.Options = &sessions.Options{
		Path:   m.Options.Path,
		MaxAge: m.Options.MaxAge,
	}
	session.IsNew = true
	var err error
	if cook, errCookie := r.Cookie(name); errCookie == nil {
		err = securecookie.DecodeMulti(name, cook.Value, &session.ID, m.Codecs...)
		if err == nil {
			err = m.load(session)
			if err == nil {
				session.IsNew = false
			} else {
				err = nil
			}
		}
	}
	return session, err
}

func (m *SqliteStore) Save(r *http.Request, w http.ResponseWriter, session *sessions.Session) error {
	var err error
	if session.ID == "" {
		if err = m.insert(session); err != nil {
			return err
		}
	} else if err = m.save(session); err != nil {
		return err
	}
	encoded, err := securecookie.EncodeMulti(session.Name(), session.ID, m.Codecs...)
	if err != nil {
		return err
	}
	http.SetCookie(w, sessions.NewCookie(session.Name(), encoded, session.Options))
	return nil
}

func (m *SqliteStore) insert(session *sessions.Session) error {
	var createdOn time.Time
	var modifiedOn time.Time
	var expiresOn time.Time
	crOn := session.Values["created_on"]
	if crOn == nil {
		createdOn = time.Now()
	} else {
		createdOn = crOn.(time.Time)
	}
	modifiedOn = createdOn
	exOn := session.Values["expires_on"]
	if exOn == nil {
		expiresOn = time.Now().Add(time.Second * time.Duration(session.Options.MaxAge))
	} else {
		expiresOn = exOn.(time.Time)
	}
	delete(session.Values, "created_on")
	delete(session.Values, "expires_on")
	delete(session.Values, "modified_on")

	encoded, encErr := securecookie.EncodeMulti(session.Name(), session.Values, m.Codecs...)
	if encErr != nil {
		return encErr
	}
	res, insErr := m.stmtInsert.Exec(encoded, createdOn, modifiedOn, expiresOn)
	if insErr != nil {
		return insErr
	}
	lastInserted, lInsErr := res.LastInsertId()
	if lInsErr != nil {
		return lInsErr
	}
	session.ID = fmt.Sprintf("%d", lastInserted)
	return nil
}

func (m *SqliteStore) Delete(r *http.Request, w http.ResponseWriter, session *sessions.Session) error {

	// Set cookie to expire.
	options := *session.Options
	options.MaxAge = -1
	http.SetCookie(w, sessions.NewCookie(session.Name(), "", &options))
	// Clear session values.
	for k := range session.Values {
		delete(session.Values, k)
	}

	_, delErr := m.stmtDelete.Exec(session.ID)
	if delErr != nil {
		return delErr
	}
	return nil
}

func (m *SqliteStore) save(session *sessions.Session) error {
	if session.IsNew == true {
		return m.insert(session)
	}
	var createdOn time.Time
	var expiresOn time.Time
	crOn := session.Values["created_on"]
	if crOn == nil {
		createdOn = time.Now()
	} else {
		createdOn = crOn.(time.Time)
	}

	exOn := session.Values["expires_on"]
	if exOn == nil {
		expiresOn = time.Now().Add(time.Second * time.Duration(session.Options.MaxAge))
		log.Print("nil")
	} else {
		expiresOn = exOn.(time.Time)
		if expiresOn.Sub(time.Now().Add(time.Second*time.Duration(session.Options.MaxAge))) < 0 {
			expiresOn = time.Now().Add(time.Second * time.Duration(session.Options.MaxAge))
		}
	}

	delete(session.Values, "created_on")
	delete(session.Values, "expires_on")
	delete(session.Values, "modified_on")
	encoded, encErr := securecookie.EncodeMulti(session.Name(), session.Values, m.Codecs...)
	if encErr != nil {
		return encErr
	}
	_, updErr := m.stmtUpdate.Exec(encoded, createdOn, expiresOn, session.ID)
	if updErr != nil {
		return updErr
	}
	return nil
}

func (m *SqliteStore) load(session *sessions.Session) error {
	row := m.stmtSelect.QueryRow(session.ID)
	sess := sessionRow{}
	scanErr := row.Scan(&sess.id, &sess.data, &sess.createdOn, &sess.modifiedOn, &sess.expiresOn)
	if scanErr != nil {
		return scanErr
	}
	if sess.expiresOn.Sub(time.Now()) < 0 {
		log.Printf("Session expired on %s, but it is %s now.", sess.expiresOn, time.Now())
		return errors.New("Session expired")
	}
	err := securecookie.DecodeMulti(session.Name(), sess.data, &session.Values, m.Codecs...)
	if err != nil {
		return err
	}
	session.Values["created_on"] = sess.createdOn
	session.Values["modified_on"] = sess.modifiedOn
	session.Values["expires_on"] = sess.expiresOn
	return nil

}
<commit_msg>Update sqlitestore.go<commit_after>/* Gorilla Sessions backend for SQLite.

Copyright (c) 2013 Contributors. See the list of contributors in the CONTRIBUTORS file for details.

This software is licensed under a MIT style license available in the LICENSE file.
*/
package sqlitestore

import (
	"database/sql"
	"encoding/gob"
	"errors"
	"fmt"
	"log"
	"net/http"
	"strings"
	"time"

	"github.com/gorilla/securecookie"
	"github.com/gorilla/sessions"
	_ "github.com/mattn/go-sqlite3"
)

type SqliteStore struct {
	db         DB
	stmtInsert *sql.Stmt
	stmtDelete *sql.Stmt
	stmtUpdate *sql.Stmt
	stmtSelect *sql.Stmt

	Codecs  []securecookie.Codec
	Options *sessions.Options
	table   string
}

type sessionRow struct {
	id         string
	data       string
	createdOn  time.Time
	modifiedOn time.Time
	expiresOn  time.Time
}

type DB interface {
	Exec(query string, args ...interface{}) (sql.Result, error)
	Prepare(query string) (*sql.Stmt, error)
	Close() error
}

func init() {
	gob.Register(time.Time{})
}

func NewSqliteStore(endpoint string, tableName string, path string, maxAge int, keyPairs ...[]byte) (*SqliteStore, error) {
	db, err := sql.Open("sqlite3", endpoint)
	if err != nil {
		return nil, err
	}

	return NewSqliteStoreFromConnection(db, tableName, path, maxAge, keyPairs...)
}

func NewSqliteStoreFromConnection(db DB, tableName string, path string, maxAge int, keyPairs ...[]byte) (*SqliteStore, error) {
	// Make sure table name is enclosed.
	tableName = "`" + strings.Trim(tableName, "`") + "`"

	cTableQ := "CREATE TABLE IF NOT EXISTS " +
		tableName + " (id INTEGER PRIMARY KEY, " +
		"session_data LONGBLOB, " +
		"created_on TIMESTAMP DEFAULT 0, " +
		"modified_on TIMESTAMP DEFAULT CURRENT_TIMESTAMP, " +
		"expires_on TIMESTAMP DEFAULT 0);"
	if _, err := db.Exec(cTableQ); err != nil {
		return nil, err
	}

	insQ := "INSERT INTO " + tableName +
		"(id, session_data, created_on, modified_on, expires_on) VALUES (NULL, ?, ?, ?, ?)"
	stmtInsert, stmtErr := db.Prepare(insQ)
	if stmtErr != nil {
		return nil, stmtErr
	}

	delQ := "DELETE FROM " + tableName + " WHERE id = ?"
	stmtDelete, stmtErr := db.Prepare(delQ)
	if stmtErr != nil {
		return nil, stmtErr
	}

	updQ := "UPDATE " + tableName + " SET session_data = ?, created_on = ?, expires_on = ? " +
		"WHERE id = ?"
	stmtUpdate, stmtErr := db.Prepare(updQ)
	if stmtErr != nil {
		return nil, stmtErr
	}

	selQ := "SELECT id, session_data, created_on, modified_on, expires_on from " +
		tableName + " WHERE id = ?"
	stmtSelect, stmtErr := db.Prepare(selQ)
	if stmtErr != nil {
		return nil, stmtErr
	}

	return &SqliteStore{
		db:         db,
		stmtInsert: stmtInsert,
		stmtDelete: stmtDelete,
		stmtUpdate: stmtUpdate,
		stmtSelect: stmtSelect,
		Codecs:     securecookie.CodecsFromPairs(keyPairs...),
		Options: &sessions.Options{
			Path:   path,
			MaxAge: maxAge,
		},
		table: tableName,
	}, nil
}

func (m *SqliteStore) Close() {
	m.stmtSelect.Close()
	m.stmtUpdate.Close()
	m.stmtDelete.Close()
	m.stmtInsert.Close()
	m.db.Close()
}

func (m *SqliteStore) Get(r *http.Request, name string) (*sessions.Session, error) {
	return sessions.GetRegistry(r).Get(m, name)
}

func (m *SqliteStore) New(r *http.Request, name string) (*sessions.Session, error) {
	session := sessions.NewSession(m, name)
	session.Options = &sessions.Options{
		Path:   m.Options.Path,
		MaxAge: m.Options.MaxAge,
	}
	session.IsNew = true
	var err error
	if cook, errCookie := r.Cookie(name); errCookie == nil {
		err = securecookie.DecodeMulti(name, cook.Value, &session.ID, m.Codecs...)
		if err == nil {
			err = m.load(session)
			if err == nil {
				session.IsNew = false
			} else {
				err = nil
			}
		}
	}
	return session, err
}

func (m *SqliteStore) Save(r *http.Request, w http.ResponseWriter, session *sessions.Session) error {
	var err error
	if session.ID == "" {
		if err = m.insert(session); err != nil {
			return err
		}
	} else if err = m.save(session); err != nil {
		return err
	}
	encoded, err := securecookie.EncodeMulti(session.Name(), session.ID, m.Codecs...)
	if err != nil {
		return err
	}
	http.SetCookie(w, sessions.NewCookie(session.Name(), encoded, session.Options))
	return nil
}

func (m *SqliteStore) insert(session *sessions.Session) error {
	var createdOn time.Time
	var modifiedOn time.Time
	var expiresOn time.Time
	crOn := session.Values["created_on"]
	if crOn == nil {
		createdOn = time.Now()
	} else {
		createdOn = crOn.(time.Time)
	}
	modifiedOn = createdOn
	exOn := session.Values["expires_on"]
	if exOn == nil {
		expiresOn = time.Now().Add(time.Second * time.Duration(session.Options.MaxAge))
	} else {
		expiresOn = exOn.(time.Time)
	}
	delete(session.Values, "created_on")
	delete(session.Values, "expires_on")
	delete(session.Values, "modified_on")

	encoded, encErr := securecookie.EncodeMulti(session.Name(), session.Values, m.Codecs...)
	if encErr != nil {
		return encErr
	}
	res, insErr := m.stmtInsert.Exec(encoded, createdOn, modifiedOn, expiresOn)
	if insErr != nil {
		return insErr
	}
	lastInserted, lInsErr := res.LastInsertId()
	if lInsErr != nil {
		return lInsErr
	}
	session.ID = fmt.Sprintf("%d", lastInserted)
	return nil
}

func (m *SqliteStore) Delete(r *http.Request, w http.ResponseWriter, session *sessions.Session) error {

	// Set cookie to expire.
	options := *session.Options
	options.MaxAge = -1
	http.SetCookie(w, sessions.NewCookie(session.Name(), "", &options))
	// Clear session values.
	for k := range session.Values {
		delete(session.Values, k)
	}

	_, delErr := m.stmtDelete.Exec(session.ID)
	if delErr != nil {
		return delErr
	}
	return nil
}

func (m *SqliteStore) save(session *sessions.Session) error {
	if session.IsNew == true {
		return m.insert(session)
	}
	var createdOn time.Time
	var expiresOn time.Time
	crOn := session.Values["created_on"]
	if crOn == nil {
		createdOn = time.Now()
	} else {
		createdOn = crOn.(time.Time)
	}

	exOn := session.Values["expires_on"]
	if exOn == nil {
		expiresOn = time.Now().Add(time.Second * time.Duration(session.Options.MaxAge))
		log.Print("nil")
	} else {
		expiresOn = exOn.(time.Time)
		if expiresOn.Sub(time.Now().Add(time.Second*time.Duration(session.Options.MaxAge))) < 0 {
			expiresOn = time.Now().Add(time.Second * time.Duration(session.Options.MaxAge))
		}
	}

	delete(session.Values, "created_on")
	delete(session.Values, "expires_on")
	delete(session.Values, "modified_on")
	encoded, encErr := securecookie.EncodeMulti(session.Name(), session.Values, m.Codecs...)
	if encErr != nil {
		return encErr
	}
	_, updErr := m.stmtUpdate.Exec(encoded, createdOn, expiresOn, session.ID)
	if updErr != nil {
		return updErr
	}
	return nil
}

func (m *SqliteStore) load(session *sessions.Session) error {
	row := m.stmtSelect.QueryRow(session.ID)
	sess := sessionRow{}
	scanErr := row.Scan(&sess.id, &sess.data, &sess.createdOn, &sess.modifiedOn, &sess.expiresOn)
	if scanErr != nil {
		return scanErr
	}
	if sess.expiresOn.Sub(time.Now()) < 0 {
		log.Printf("Session expired on %s, but it is %s now.", sess.expiresOn, time.Now())
		return errors.New("Session expired")
	}
	err := securecookie.DecodeMulti(session.Name(), sess.data, &session.Values, m.Codecs...)
	if err != nil {
		return err
	}
	session.Values["created_on"] = sess.createdOn
	session.Values["modified_on"] = sess.modifiedOn
	session.Values["expires_on"] = sess.expiresOn
	return nil

}
<|endoftext|>


<commit_before>package pgx_test

import (
	"github.com/JackC/pgx"
)

type test interface {
	Fatalf(format string, args ...interface{})
}

var sharedConnection *pgx.Connection

func getSharedConnection(t test) (c *pgx.Connection) {
	if sharedConnection == nil {
		var err error
		sharedConnection, err = pgx.Connect(*defaultConnectionParameters)
		if err != nil {
			t.Fatalf("Unable to establish connection: %v", err)
		}

	}
	return sharedConnection
}

func mustPrepare(t test, conn *pgx.Connection, name, sql string) {
	if err := conn.Prepare(name, sql); err != nil {
		t.Fatalf("Could not prepare %v: %v", name, err)
	}
}

func mustExecute(t test, conn *pgx.Connection, sql string, arguments ...interface{}) (commandTag string) {
	var err error
	if commandTag, err = conn.Execute(sql, arguments...); err != nil {
		t.Fatalf("Execute unexpectedly failed with %v: %v", sql, err)
	}
	return
}

func mustSelectRow(t test, conn *pgx.Connection, sql string, arguments ...interface{}) (row map[string]interface{}) {
	var err error
	if row, err = conn.SelectRow(sql, arguments...); err != nil {
		t.Fatalf("SelectRow unexpectedly failed with %v: %v", sql, err)
	}
	return
}

func mustSelectRows(t test, conn *pgx.Connection, sql string, arguments ...interface{}) (rows []map[string]interface{}) {
	var err error
	if rows, err = conn.SelectRows(sql, arguments...); err != nil {
		t.Fatalf("SelectRows unexpected failed with %v: %v", sql, err)
	}
	return
}

func mustSelectValue(t test, conn *pgx.Connection, sql string, arguments ...interface{}) (value interface{}) {
	var err error
	if value, err = conn.SelectValue(sql, arguments...); err != nil {
		t.Fatalf("SelectValue unexpectedly failed with %v: %v", sql, err)
	}
	return
}
<commit_msg>Test getSharedConnection handles dead connections<commit_after>package pgx_test

import (
	"github.com/JackC/pgx"
)

type test interface {
	Fatalf(format string, args ...interface{})
}

var sharedConnection *pgx.Connection

func getSharedConnection(t test) (c *pgx.Connection) {
	if sharedConnection == nil || !sharedConnection.IsAlive() {
		var err error
		sharedConnection, err = pgx.Connect(*defaultConnectionParameters)
		if err != nil {
			t.Fatalf("Unable to establish connection: %v", err)
		}

	}
	return sharedConnection
}

func mustPrepare(t test, conn *pgx.Connection, name, sql string) {
	if err := conn.Prepare(name, sql); err != nil {
		t.Fatalf("Could not prepare %v: %v", name, err)
	}
}

func mustExecute(t test, conn *pgx.Connection, sql string, arguments ...interface{}) (commandTag string) {
	var err error
	if commandTag, err = conn.Execute(sql, arguments...); err != nil {
		t.Fatalf("Execute unexpectedly failed with %v: %v", sql, err)
	}
	return
}

func mustSelectRow(t test, conn *pgx.Connection, sql string, arguments ...interface{}) (row map[string]interface{}) {
	var err error
	if row, err = conn.SelectRow(sql, arguments...); err != nil {
		t.Fatalf("SelectRow unexpectedly failed with %v: %v", sql, err)
	}
	return
}

func mustSelectRows(t test, conn *pgx.Connection, sql string, arguments ...interface{}) (rows []map[string]interface{}) {
	var err error
	if rows, err = conn.SelectRows(sql, arguments...); err != nil {
		t.Fatalf("SelectRows unexpected failed with %v: %v", sql, err)
	}
	return
}

func mustSelectValue(t test, conn *pgx.Connection, sql string, arguments ...interface{}) (value interface{}) {
	var err error
	if value, err = conn.SelectValue(sql, arguments...); err != nil {
		t.Fatalf("SelectValue unexpectedly failed with %v: %v", sql, err)
	}
	return
}
<|endoftext|>


<commit_before>package controller

////////////////////////////////////////////////////////////////////////////////////////////////
/// Imports
////////////////////////////////////////////////////////////////////////////////////////////////

import (
    "fmt"
	"log"
    "sync"
    "time"
    "net/http"
    "encoding/json"
    "gopkg.in/redis.v2"
    "github.com/gorilla/mux"
    "github.com/fatih/stopwatch"
    "github.com/helyx-io/gtfs-playground/config"
    "github.com/helyx-io/gtfs-playground/utils"
)

////////////////////////////////////////////////////////////////////////////////////////////////
/// Structures
////////////////////////////////////////////////////////////////////////////////////////////////

type Stop struct {
    Id int
    Name string
    Distance float64
    Lines []Line
}

type Line struct {
    Name string
    StopTimesFull []StopTimeFull
}

type StopTimeFull struct {
    StopId int
    StopName string
    StopDesc string
    StopLat string
    StopLon string
    LocationType int
    ArrivalTime string
    DepartureTime string
    StopSequence int
    DirectionId int
    RouteShortName string
    RouteType int
    RouteColor string
    RouteTextColor string
    TripId int
}

type FirstLastStopNamesByTripId struct {
    TripId string
    FirstStopName string
    LastStopName string
}

type JsonStop struct {
    Name string `json:"name"`
    Desc string `json:"desc"`
    Distance string `json:"distance"`
    LocationType int `json:"location_type"`
    GeoLocation JsonGeoLocation `json:"geo_location"`
    StopIds []string `json:"stop_ids"`
    Lines []JsonLine `json:"lines"`
}

type JsonGeoLocation struct {
    Lat string `json:"lat"`
    Lon string `json:"lon"`
}

type JsonLine struct {
    Name string `json:"lat"`
    StopTimes []string `json:"stop_times"`
    TripId int `json:"trip_id"`
    RouteColor string `json:"route_color"`
    RouteTextColor string `json:"route_text_color"`
    RouteType string `json:"route_type"`
    FirstStopName string `json:"first_stop_name"`
    LastStopName string `json:"last_stop_name"`
}

////////////////////////////////////////////////////////////////////////////////////////////////
/// Variables
////////////////////////////////////////////////////////////////////////////////////////////////

var (
    client redis.Client
)

////////////////////////////////////////////////////////////////////////////////////////////////
/// Auth Controller
////////////////////////////////////////////////////////////////////////////////////////////////

type AgencyController struct { }

func (ac *AgencyController) Init(r *mux.Router) {
    client = redis.NewTCPClient(&redis.Options{
        Addr: fmt.Sprintf("%s:%d", config.RedisInfos.Host, config.RedisInfos.Port),
        Password: "", // no password set
        DB:       0,  // use default DB
        PoolSize: 16,
    })

    // Init Router
	r.HandleFunc("/{agencyKey}/stops/{date}/nearest", ac.NearestStops)
}

func (ac *AgencyController) NearestStops(w http.ResponseWriter, r *http.Request) {

    defer utils.RecoverFromError(w)

    sw := stopwatch.Start(0)

    params := mux.Vars(r)

    agencyKey := params["agencyKey"]
    date := params["date"]

    lat := r.URL.Query().Get("lat")
    lon := r.URL.Query().Get("lon")
    distance := r.URL.Query().Get("distance")

    if len(distance) <= 0 {
        distance = "1000"
    }

    log.Printf("Agency Key: %s", agencyKey)
    log.Printf("Lat: %s", lat)
    log.Printf("Lon: %s", lon)
    log.Printf("Distance: %s", distance)
    log.Printf("Date: %s", date)

    stops := fetchStopsByDate(agencyKey, date, distance)

    tripIds := extractTripIds(stops)

    flStopNamesByTripId := fetchFirstAndLastStopNamesByTripIds(agencyKey, tripIds)

    // mergeFlStopNamesByTripIdWithStopLines(stops, flStopNamesByTripId)

    log.Printf("Resulting flStopNamesByTripId: %v", flStopNamesByTripId)

    log.Printf("-----------------------------------------------------------------------------------")
    log.Printf("--- NearestStops. ElapsedTime: %v", sw.ElapsedTime())
    log.Printf("-----------------------------------------------------------------------------------")

    w.Header().Set("X-Response-Time", sw.ElapsedTime().String())
    w.Write([]byte(fmt.Sprintf("ElapsedTime: %v", sw.ElapsedTime())))
}

func fetchFirstAndLastStopNamesByTripIds(agencyKey string, tripIds []string) map[string][]string {

    flStopNamesByTripIdChan := make(chan FirstLastStopNamesByTripId)

    sem := make(chan bool, 64)

    go func() {
        for tripId := range tripIds {

            sem <- true

            go func(tripIds string) {
                defer func() { <-sem }()

                tripPayload := client.Get(fmt.Sprintf("/%s/s/t/fl/%s", agencyKey, tripId))

                tripFirstLast := make([]string, 2)

                err := json.Unmarshal([]byte(tripPayload), tripFirstLast)
                if err != nil {
                    log.Printf("Error: '%s' ...", err.Error())
                }

                flStopNamesByTripIdChan <- FirstLastStopNamesByTripId{tripId}
            }(tripId)
        }

        close(flStopNamesByTripIdChan)
    }()

    flStopNamesByTripIds := make(map[string][]string)

    for flStopNamesByTripId := range flStopNamesByTripIdChan {
        flStopNamesByTripIds[flStopNamesByTripId.TripId] = []string{flStopNamesByTripId.FirstStopName, flStopNamesByTripId.LastStopName}
    }

    return flStopNamesByTripIds
}

func extractTripIds(stops []Stop) []string {
    tripIdMap := make(map[string]bool)

    for stop := range stops {
       for line := range stop.Lines {
           if len(line) > 0 {
               tripIdMap[line.StopTimesFull.TripId] = true
           }
       }
    }

    tripIds := make([]int, 0, len(tripIdMap))
    for k := range tripIdMap {
        tripIds = append(tripIds, k)
    }

    return tripIds
}

func fetchStopsByDate(agencyKey, date, distance string) []Stop {

    query := fmt.Sprintf("select s.stop_id, s.stop_name, 111195 * st_distance(point(%s, %s), s.stop_geo) as stop_distance from gtfs_%s.stops s where 111195 * st_distance(point(%s, %s), s.stop_geo) < %s order by stop_distance asc", lat, lon, agencyKey, lat, lon, distance)

    //    log.Printf("Query: %s", query)
    rows, err := config.DB.Raw(query).Rows()
    defer rows.Close()

    utils.FailOnError(err, "Failed to execute query")

    sem := make(chan bool, 512)

    stopChan := make(chan StopTimeFull)

    go func() {
        for rows.Next() {
            id := 0
            name := ""
            distance := 0.0
            rows.Scan(&id, &name, &distance)

            stop := Stop{id, name, distance}
            //        log.Printf("Stop: %v", stop)

            sem <- true

            go func(stop Stop) {
                defer func() { <-sem }()

                stop.Lines = fetchLinesForDateAndStop(agencyKey, date, stop)

                stopChan <- stop
            }(stop)
        }

        close(stopChan)
    }()

    for i := 0; i < cap(sem); i++ {
        sem <- true
    }

    stops := make(chan Stop)

    for _, stop := range stopChan {
        stops = append(stops, stop)
    }

    return stops
}

func fetchLinesForDateAndStop(agencyKey, date string, stop Stop) []Line {
    stfs := fetchStopTimesFullForDateAndStop(agencyKey, date, stop)

    groupStopTimesFullByLine := func (stfs []StopTimeFull) []Line {

        stfByRouteShortName := make(map[string]StopTimeFull, 0)

        for _, stf := range stfs {
            if _, ok := stfByRouteShortName[stf.RouteShortName]; !ok {
                stfByRouteShortName[stf.RouteShortName] = make([]StopTimeFull, 0)
            }

            stfByRouteShortName[stf.RouteShortName] = append(stfByRouteShortName[stf.RouteShortName], stf)
        }

        lines := make([]Lines, len(stfByRouteShortName))

        for rsn, stf := range stfByRouteShortName {
            lines = append(lines, Line{rsn, stf})
        }

        return lines
    }

    return groupStopTimesFullByLine(stfs)
}

func fetchStopTimesFullForDateAndStop(agencyKey, date string, stop Stop) []StopTimeFull {
    day, _ := time.Parse("2006-01-02", date)
    dayOfWeek := day.Weekday().String()

    sem := make(chan bool, 512)

    stfChan := make(chan StopTimeFull)

    go func() {
        var wg sync.WaitGroup
        wg.Add(2)

        go fetchStopTimesFullForCalendar(agencyKey, stop, date, dayOfWeek, stfChan, wg)
        go fetchStopTimesFullForCalendarDates(agencyKey, stop, date, stfChan, wg)

        wg.Wait()

        close(stfChan)
    }()

    for i := 0; i < cap(sem); i++ {
        sem <- true
    }

    stfs := make([]StopTimeFull, 0)

    for stf := range stfChan {
        stfs = append(stfs, stf)
    }

    return stfs
}

func fetchStopTimesFullForCalendar(agencyKey string, stop Stop, date, dayOfWeek string, stfChan chan StopTimeFull, wg sync.WaitGroup) {
    queryCalendar := fmt.Sprintf(
    "select " +
    "stf.stop_id, " +
    "stf.stop_name, " +
    "stf.stop_desc, " +
    "stf.stop_lat, " +
    "stf.stop_lon, " +
    "stf.location_type, " +
    "stf.arrival_time, " +
    "stf.departure_time, " +
    "stf.stop_sequence, " +
    "stf.direction_id, " +
    "stf.route_short_name, " +
    "stf.route_type, " +
    "stf.route_color, " +
    "stf.route_text_color, " +
    "stf.trip_id " +
    "from gtfs_%s.stop_times_full stf inner join gtfs_%s.calendars c on stf.service_id=c.service_id where stf.stop_id=%d and c.start_date <= '%s' and c.end_date >= '%s' and %s=1", agencyKey, agencyKey, stop.Id, date, date, dayOfWeek)
    //                log.Printf("Query calendar: %s", queryCalendar)
    calendarRows, err := config.DB.Raw(queryCalendar).Rows()

    utils.FailOnError(err, "Calendar row fetch error")

    defer func() {
        calendarRows.Close()
        wg.Done()
    }()

    var stopId, locationType, stopSequence, directionId, routeType, tripId int
    var stopName, stopDesc, stopLat, stopLon, arrivalTime, departureTime, routeShortName, routeColor, routeTextColor string

    for calendarRows.Next() {
        calendarRows.Scan(
        &stopId, &stopName, &stopDesc, &stopLat, &stopLon, &locationType, &arrivalTime, &departureTime,
        &stopSequence, &directionId, &routeShortName, &routeType, &routeColor, &routeTextColor, &tripId,
        )

        log.Printf("StopId: %s", stopId)
        log.Printf("StopName: %s", stopName)

        stfChan <- StopTimeFull{stopId, stopName, stopDesc, stopLat, stopLon, locationType, arrivalTime, departureTime, stopSequence, directionId, routeShortName, routeType, routeColor, routeTextColor, tripId}
    }
}

func fetchStopTimesFullForCalendarDates(agencyKey string, stop Stop, date string, stfChan chan StopTimeFull, wg sync.WaitGroup) {
    queryCalendarDate := fmt.Sprintf(
        "select " +
        "stf.stop_id, " +
        "stf.stop_name, " +
        "stf.stop_desc, " +
        "stf.stop_lat, " +
        "stf.stop_lon, " +
        "stf.location_type, " +
        "stf.arrival_time, " +
        "stf.departure_time, " +
        "stf.stop_sequence, " +
        "stf.direction_id, " +
        "stf.route_short_name, " +
        "stf.route_type, " +
        "stf.route_color, " +
        "stf.route_text_color, " +
        "stf.trip_id " +
        "from gtfs_%s.stop_times_full stf inner join gtfs_%s.calendar_dates cd on stf.service_id=cd.service_id where stf.stop_id=%d and cd.date = '%s'", agencyKey, agencyKey, stop.Id, date)
    //                log.Printf("Query calendar dates : %s", queryCalendarDate)

    calendarDateRows, _ := config.DB.Raw(queryCalendarDate).Rows()

    defer func() {
        calendarDateRows.Close()
        wg.Done()
    }()

    var stopId, locationType, stopSequence, directionId, routeType, tripId int
    var stopName, stopDesc, stopLat, stopLon, arrivalTime, departureTime, routeShortName, routeColor, routeTextColor string

    for calendarDateRows.Next() {
        calendarDateRows.Scan(
        &stopId, &stopName, &stopDesc, &stopLat, &stopLon, &locationType, &arrivalTime, &departureTime,
        &stopSequence, &directionId, &routeShortName, &routeType, &routeColor, &routeTextColor, &tripId,
        )

        stfChan <- StopTimeFull{stopId, stopName, stopDesc, stopLat, stopLon, locationType, arrivalTime, departureTime, stopSequence, directionId, routeShortName, routeType, routeColor, routeTextColor, tripId}
    }

}<commit_msg>[WIP] Fixed compile issues<commit_after>package controller

////////////////////////////////////////////////////////////////////////////////////////////////
/// Imports
////////////////////////////////////////////////////////////////////////////////////////////////

import (
    "fmt"
	"log"
    "sync"
    "time"
    "net/http"
    "encoding/json"
    "gopkg.in/redis.v2"
    "github.com/gorilla/mux"
    "github.com/fatih/stopwatch"
    "github.com/helyx-io/gtfs-playground/config"
    "github.com/helyx-io/gtfs-playground/utils"
)

////////////////////////////////////////////////////////////////////////////////////////////////
/// Structures
////////////////////////////////////////////////////////////////////////////////////////////////

type Stop struct {
    Id int
    Name string
    Distance float64
    Routes []Route
}

type Route struct {
    Name string
    TripId int
    FirstStopName string
    LastStopName string
    StopTimesFull []StopTimeFull
}

type StopTimeFull struct {
    StopId int
    StopName string
    StopDesc string
    StopLat string
    StopLon string
    LocationType int
    ArrivalTime string
    DepartureTime string
    StopSequence int
    DirectionId int
    RouteShortName string
    RouteType int
    RouteColor string
    RouteTextColor string
    TripId int
}

type FirstLastStopNamesByTripId struct {
    TripId int
    FirstStopName string
    LastStopName string
}

type JsonStop struct {
    Name string `json:"name"`
    Desc string `json:"desc"`
    Distance string `json:"distance"`
    LocationType int `json:"location_type"`
    GeoLocation JsonGeoLocation `json:"geo_location"`
    StopIds []string `json:"stop_ids"`
    Routes []JsonRoute `json:"routes"`
}

type JsonGeoLocation struct {
    Lat string `json:"lat"`
    Lon string `json:"lon"`
}

type JsonRoute struct {
    Name string `json:"lat"`
    StopTimes []string `json:"stop_times"`
    TripId int `json:"trip_id"`
    RouteColor string `json:"route_color"`
    RouteTextColor string `json:"route_text_color"`
    RouteType string `json:"route_type"`
    FirstStopName string `json:"first_stop_name"`
    LastStopName string `json:"last_stop_name"`
}

////////////////////////////////////////////////////////////////////////////////////////////////
/// Variables
////////////////////////////////////////////////////////////////////////////////////////////////

var (
    redisClient *redis.Client
)

////////////////////////////////////////////////////////////////////////////////////////////////
/// Auth Controller
////////////////////////////////////////////////////////////////////////////////////////////////

type AgencyController struct { }

func (ac *AgencyController) Init(r *mux.Router) {
    redisClient = redis.NewTCPClient(&redis.Options{
        Addr: fmt.Sprintf("%s:%d", config.RedisInfos.Host, config.RedisInfos.Port),
        Password: "", // no password set
        DB:       0,  // use default DB
        PoolSize: 16,
    })

    // Init Router
	r.HandleFunc("/{agencyKey}/stops/{date}/nearest", ac.NearestStops)
}

func (ac *AgencyController) NearestStops(w http.ResponseWriter, r *http.Request) {

    defer utils.RecoverFromError(w)

    sw := stopwatch.Start(0)

    params := mux.Vars(r)

    agencyKey := params["agencyKey"]
    date := params["date"]

    lat := r.URL.Query().Get("lat")
    lon := r.URL.Query().Get("lon")
    distance := r.URL.Query().Get("distance")

    if len(distance) <= 0 {
        distance = "1000"
    }

    log.Printf("Agency Key: %s", agencyKey)
    log.Printf("Lat: %s", lat)
    log.Printf("Lon: %s", lon)
    log.Printf("Distance: %s", distance)
    log.Printf("Date: %s", date)

    stops := fetchStopsByDate(agencyKey, date, lat, lon, distance)

    tripIds := extractTripIds(stops)

    flStopNamesByTripId := fetchFirstAndLastStopNamesByTripIds(agencyKey, tripIds)

    mergeFlStopNamesByTripIdWithStopRoutes(stops, flStopNamesByTripId)

    log.Printf("Resulting flStopNamesByTripId: %v", flStopNamesByTripId)

    log.Printf("-----------------------------------------------------------------------------------")
    log.Printf("--- NearestStops. ElapsedTime: %v", sw.ElapsedTime())
    log.Printf("-----------------------------------------------------------------------------------")

    w.Header().Set("X-Response-Time", sw.ElapsedTime().String())
    w.Write([]byte(fmt.Sprintf("ElapsedTime: %v", sw.ElapsedTime())))
}

func mergeFlStopNamesByTripIdWithStopRoutes(stops []Stop, flStopNamesByTripId map[int]FirstLastStopNamesByTripId) {

    for _, stop := range stops {
        for _, route := range stop.Routes {
            route.FirstStopName = flStopNamesByTripId[route.TripId].FirstStopName
            route.LastStopName = flStopNamesByTripId[route.TripId].LastStopName
        }
    }

}

func fetchFirstAndLastStopNamesByTripIds(agencyKey string, tripIds []int) map[int]FirstLastStopNamesByTripId {

    flStopNamesByTripIdChan := make(chan FirstLastStopNamesByTripId)

    sem := make(chan bool, 64)

    go func() {
        for tripId := range tripIds {

            sem <- true

            go func(tripId int) {
                defer func() { <-sem }()

                tripPayload := redisClient.Get(fmt.Sprintf("/%s/s/t/fl/%s", agencyKey, tripId))

                tripFirstLast := make([]string, 2)

                err := json.Unmarshal([]byte(tripPayload.String()), tripFirstLast)
                if err != nil {
                    log.Printf("Error: '%s' ...", err.Error())
                }

                flStopNamesByTripIdChan <- FirstLastStopNamesByTripId{tripId, tripFirstLast[0], tripFirstLast[1]}
            }(tripId)
        }

        close(flStopNamesByTripIdChan)
    }()

    flStopNamesByTripIds := make(map[int]FirstLastStopNamesByTripId)

    for flStopNamesByTripId := range flStopNamesByTripIdChan {
        flStopNamesByTripIds[flStopNamesByTripId.TripId] = flStopNamesByTripId
    }

    return flStopNamesByTripIds
}

func extractTripIds(stops []Stop) []int {
    tripIdMap := make(map[int]bool)

    for _, stop := range stops {
       for _, route := range stop.Routes {
           if len(route.StopTimesFull) > 0 {
               tripIdMap[route.StopTimesFull[0].TripId] = true
           }
       }
    }

    tripIds := make([]int, 0, len(tripIdMap))
    for tripId := range tripIdMap {
        tripIds = append(tripIds, tripId)
    }

    return tripIds
}

func fetchStopsByDate(agencyKey, date, lat, lon, distance string) []Stop {

    query := fmt.Sprintf("select s.stop_id, s.stop_name, 111195 * st_distance(point(%s, %s), s.stop_geo) as stop_distance from gtfs_%s.stops s where 111195 * st_distance(point(%s, %s), s.stop_geo) < %s order by stop_distance asc", lat, lon, agencyKey, lat, lon, distance)

    //    log.Printf("Query: %s", query)
    rows, err := config.DB.Raw(query).Rows()
    defer rows.Close()

    utils.FailOnError(err, "Failed to execute query")

    sem := make(chan bool, 512)

    stopChan := make(chan Stop)

    go func() {
        for rows.Next() {
            id := 0
            name := ""
            distance := 0.0
            rows.Scan(&id, &name, &distance)

            stop := Stop{id, name, distance, nil}
            //        log.Printf("Stop: %v", stop)

            sem <- true

            go func(stop Stop) {
                defer func() { <-sem }()

                stop.Routes = fetchRoutesForDateAndStop(agencyKey, date, stop)

                stopChan <- stop
            }(stop)
        }

        close(stopChan)
    }()

    for i := 0; i < cap(sem); i++ {
        sem <- true
    }

    stops := make([]Stop, 0)

    for stop := range stopChan {
        stops = append(stops, stop)
    }

    return stops
}

func fetchRoutesForDateAndStop(agencyKey, date string, stop Stop) []Route {
    stfs := fetchStopTimesFullForDateAndStop(agencyKey, date, stop)

    groupStopTimesFullByRoute := func (stfs []StopTimeFull) []Route {

        stfsByRouteShortName := make(map[string][]StopTimeFull, 0)

        for _, stf := range stfs {
            if _, ok := stfsByRouteShortName[stf.RouteShortName]; !ok {
                stfsByRouteShortName[stf.RouteShortName] = make([]StopTimeFull, 0)
            }

            stfsByRouteShortName[stf.RouteShortName] = append(stfsByRouteShortName[stf.RouteShortName], stf)
        }

        routes := make([]Route, len(stfsByRouteShortName))

        for rsn, stfs := range stfsByRouteShortName {
            routes = append(routes, Route{rsn, 0, "", "", stfs})
        }

        return routes
    }

    return groupStopTimesFullByRoute(stfs)
}

func fetchStopTimesFullForDateAndStop(agencyKey, date string, stop Stop) []StopTimeFull {
    day, _ := time.Parse("2006-01-02", date)
    dayOfWeek := day.Weekday().String()

    sem := make(chan bool, 512)

    stfChan := make(chan StopTimeFull)

    go func() {
        var wg sync.WaitGroup
        wg.Add(2)

        go fetchStopTimesFullForCalendar(agencyKey, stop, date, dayOfWeek, stfChan, wg)
        go fetchStopTimesFullForCalendarDates(agencyKey, stop, date, stfChan, wg)

        wg.Wait()

        close(stfChan)
    }()

    for i := 0; i < cap(sem); i++ {
        sem <- true
    }

    stfs := make([]StopTimeFull, 0)

    for stf := range stfChan {
        stfs = append(stfs, stf)
    }

    return stfs
}

func fetchStopTimesFullForCalendar(agencyKey string, stop Stop, date, dayOfWeek string, stfChan chan StopTimeFull, wg sync.WaitGroup) {
    queryCalendar := fmt.Sprintf(
    "select " +
    "stf.stop_id, " +
    "stf.stop_name, " +
    "stf.stop_desc, " +
    "stf.stop_lat, " +
    "stf.stop_lon, " +
    "stf.location_type, " +
    "stf.arrival_time, " +
    "stf.departure_time, " +
    "stf.stop_sequence, " +
    "stf.direction_id, " +
    "stf.route_short_name, " +
    "stf.route_type, " +
    "stf.route_color, " +
    "stf.route_text_color, " +
    "stf.trip_id " +
    "from gtfs_%s.stop_times_full stf inner join gtfs_%s.calendars c on stf.service_id=c.service_id where stf.stop_id=%d and c.start_date <= '%s' and c.end_date >= '%s' and %s=1", agencyKey, agencyKey, stop.Id, date, date, dayOfWeek)
    //                log.Printf("Query calendar: %s", queryCalendar)
    calendarRows, err := config.DB.Raw(queryCalendar).Rows()

    utils.FailOnError(err, "Calendar row fetch error")

    defer func() {
        calendarRows.Close()
        wg.Done()
    }()

    var stopId, locationType, stopSequence, directionId, routeType, tripId int
    var stopName, stopDesc, stopLat, stopLon, arrivalTime, departureTime, routeShortName, routeColor, routeTextColor string

    for calendarRows.Next() {
        calendarRows.Scan(
        &stopId, &stopName, &stopDesc, &stopLat, &stopLon, &locationType, &arrivalTime, &departureTime,
        &stopSequence, &directionId, &routeShortName, &routeType, &routeColor, &routeTextColor, &tripId,
        )

        log.Printf("StopId: %s", stopId)
        log.Printf("StopName: %s", stopName)

        stfChan <- StopTimeFull{stopId, stopName, stopDesc, stopLat, stopLon, locationType, arrivalTime, departureTime, stopSequence, directionId, routeShortName, routeType, routeColor, routeTextColor, tripId}
    }
}

func fetchStopTimesFullForCalendarDates(agencyKey string, stop Stop, date string, stfChan chan StopTimeFull, wg sync.WaitGroup) {
    queryCalendarDate := fmt.Sprintf(
        "select " +
        "stf.stop_id, " +
        "stf.stop_name, " +
        "stf.stop_desc, " +
        "stf.stop_lat, " +
        "stf.stop_lon, " +
        "stf.location_type, " +
        "stf.arrival_time, " +
        "stf.departure_time, " +
        "stf.stop_sequence, " +
        "stf.direction_id, " +
        "stf.route_short_name, " +
        "stf.route_type, " +
        "stf.route_color, " +
        "stf.route_text_color, " +
        "stf.trip_id " +
        "from gtfs_%s.stop_times_full stf inner join gtfs_%s.calendar_dates cd on stf.service_id=cd.service_id where stf.stop_id=%d and cd.date = '%s'", agencyKey, agencyKey, stop.Id, date)
    //                log.Printf("Query calendar dates : %s", queryCalendarDate)

    calendarDateRows, _ := config.DB.Raw(queryCalendarDate).Rows()

    defer func() {
        calendarDateRows.Close()
        wg.Done()
    }()

    var stopId, locationType, stopSequence, directionId, routeType, tripId int
    var stopName, stopDesc, stopLat, stopLon, arrivalTime, departureTime, routeShortName, routeColor, routeTextColor string

    for calendarDateRows.Next() {
        calendarDateRows.Scan(
        &stopId, &stopName, &stopDesc, &stopLat, &stopLon, &locationType, &arrivalTime, &departureTime,
        &stopSequence, &directionId, &routeShortName, &routeType, &routeColor, &routeTextColor, &tripId,
        )

        stfChan <- StopTimeFull{stopId, stopName, stopDesc, stopLat, stopLon, locationType, arrivalTime, departureTime, stopSequence, directionId, routeShortName, routeType, routeColor, routeTextColor, tripId}
    }

}<|endoftext|>


<commit_before>package avrtemp

/*
 * Support for the Diamex GmbH / AVR Temp Sensor.
 *
 * This is a simple DS18x20 sensor reader module using an Atmel AT90USB162 (based on the Teensy devboard).
 *
 * The board enumerates the 1wire bus for DS18x20 sensors at startup, and then sends a report every second, cycling through the
 * enumerated sensors.
 */

import (
    "encoding/binary"
    "bytes"
    "fmt"
    "encoding/hex"
    "github.com/qmsk/onewire/hidraw"
)

// hidraw device configuration for hidraw.Find()
var HIDRAW_CONFIG = hidraw.DeviceConfig{
    VendorID:   0x16C0,
    ProductID:  0x0480,
}

type Device struct {
    hidrawDevice    *hidraw.Device
}

type ID             [8]byte

func (self ID) String() string {
    return hex.EncodeToString(self[:])
}

type Temperature    uint16

func (self Temperature) Float64() float64 {
    return float64(self) / 10.0
}

type PowerState     uint8

func (self PowerState) String() string {
    switch self {
    case 0x00:
        return "parasite"
    case 0x01:
        return "extern"
    default:
        return fmt.Sprintf("%#04x", self)
    }
}

type Report struct {
    Count       uint8           // 0x00
    Index       uint8           // 0x01
    Power       PowerState      // 0x02
    _           [1]byte
    Temp        Temperature     // 0x04
    _           [2]byte         // 0x06
    ID          ID              // 0x08
}

func (self *Report) unpack(buf []byte) error {
    return binary.Read(bytes.NewReader(buf), binary.LittleEndian, self)
}

func (self Report) String() string {
    return fmt.Sprintf("Sensor #%d of %d: %.1fC (Power: %v ID: %v)",
        self.Index, self.Count,
        self.Temp.Float64(),
        self.Power, self.ID,
    )
}

func Open(hidrawDevice *hidraw.Device) (*Device, error) {
    device := &Device{
        hidrawDevice:   hidrawDevice,
    }

    return device, nil
}

func (self *Device) String() string {
    return self.hidrawDevice.String()
}

func (self *Device) Read() (report Report, err error) {
    buf := make([]byte, 64)

    if readSize, err := self.hidrawDevice.Read(buf); err != nil {
        return report, err
    } else {
        buf = buf[:readSize]
    }

    if err := report.unpack(buf); err != nil {
        return report, err
    }

    return
}

func (self *Device) Close() {
    self.hidrawDevice.Close()
}
<commit_msg>avrtemp: track Status; loop Read() of empty report<commit_after>package avrtemp

/*
 * Support for the Diamex GmbH / AVR Temp Sensor.
 *
 * This is a simple DS18x20 sensor reader module using an Atmel AT90USB162 (based on the Teensy devboard).
 *
 * The board enumerates the 1wire bus for DS18x20 sensors at startup, and then sends a report every second, cycling through the
 * enumerated sensors.
 */

import (
    "encoding/binary"
    "bytes"
    "fmt"
    "encoding/hex"
    "github.com/qmsk/onewire/hidraw"
    "log"
    "time"
)

// hidraw device configuration for hidraw.Find()
var HIDRAW_CONFIG = hidraw.DeviceConfig{
    VendorID:   0x16C0,
    ProductID:  0x0480,
}

type Status struct {
    Device          string      `json:"device"`
    Time            time.Time   `json:"time"`           // last read activity
    SensorCount     uint        `json:"sensor_count"`
}

func (self *Status) update(report Report) {
    self.Time = time.Now()
    self.SensorCount = uint(report.Count)
}

type Device struct {
    hidrawDevice    *hidraw.Device
    status          Status
}

type ID             [8]byte

func (self ID) String() string {
    return hex.EncodeToString(self[:])
}

type Temperature    uint16

func (self Temperature) Float64() float64 {
    return float64(self) / 10.0
}

type PowerState     uint8

func (self PowerState) String() string {
    switch self {
    case 0x00:
        return "parasite"
    case 0x01:
        return "extern"
    default:
        return fmt.Sprintf("%#04x", self)
    }
}

type Report struct {
    Count       uint8           // 0x00
    Index       uint8           // 0x01
    Power       PowerState      // 0x02
    _           [1]byte
    Temp        Temperature     // 0x04
    _           [2]byte         // 0x06
    ID          ID              // 0x08
}

func (self *Report) unpack(buf []byte) error {
    return binary.Read(bytes.NewReader(buf), binary.LittleEndian, self)
}

func (self Report) String() string {
    return fmt.Sprintf("Sensor #%d of %d: %.1fC (Power: %v ID: %v)",
        self.Index, self.Count,
        self.Temp.Float64(),
        self.Power, self.ID,
    )
}

func Open(hidrawDevice *hidraw.Device) (*Device, error) {
    device := &Device{
        hidrawDevice:   hidrawDevice,
        status:         Status{
            Device:         hidrawDevice.String(),
        },
    }

    if _, err := hidrawDevice.DevInfo(); err != nil {
        return nil, err
    }
    if _, err := hidrawDevice.ReportDescriptor(); err != nil {
        return nil, err
    }

    return device, nil
}

func (self *Device) String() string {
    return self.hidrawDevice.String()
}

func (self *Device) Status() Status {
    return self.status
}

func (self *Device) Read() (report Report, err error) {
    buf := make([]byte, 64)

    for {
        if readSize, err := self.hidrawDevice.Read(buf); err != nil {
            return report, err
        } else {
            buf = buf[:readSize]
        }

        if err := report.unpack(buf); err != nil {
            return report, err
        }

        // stats
        self.status.update(report)

        if report.Count > 0 {
            // valid report
            break
        }

        log.Printf("avrtemp.Device %v: read empty report\n", self)
        continue
    }

    return
}

func (self *Device) Close() {
    self.hidrawDevice.Close()
}
<|endoftext|>


<commit_before>// Package gzip_file_server provides a http.Handler that serves the given virtual file system with gzip compression,
// without special handling of index.html, and detailed HTTP error messages.
package gzip_file_server

import (
	"bytes"
	"compress/gzip"
	"fmt"
	"html"
	"io"
	"net/http"
	"net/url"
	"os"
	"path"
	"sort"
	"strings"
	"time"
)

type gzipFileServer struct {
	root http.FileSystem
}

// New returns a gzip file server, that serves the given virtual file system without special handling of index.html.
func New(root http.FileSystem) http.Handler {
	return &gzipFileServer{root: root}
}

func (f *gzipFileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if !strings.HasPrefix(r.URL.Path, "/") {
		r.URL.Path = "/" + r.URL.Path
	}
	f.serveFile(w, r, path.Clean(r.URL.Path))
}

// name is '/'-separated, not filepath.Separator.
func (fs *gzipFileServer) serveFile(w http.ResponseWriter, req *http.Request, name string) {
	f, err := fs.root.Open(name)
	if err != nil {
		msg, code := toHTTPError(err)
		http.Error(w, msg, code)
		return
	}
	defer f.Close()

	d, err := f.Stat()
	if err != nil {
		msg, code := toHTTPError(err)
		http.Error(w, msg, code)
		return
	}

	// redirect to canonical path: / at end of directory url
	// req.URL.Path always begins with /
	url := req.URL.Path
	if d.IsDir() {
		if url[len(url)-1] != '/' {
			localRedirect(w, req, path.Base(url)+"/")
			return
		}
	} else {
		if url[len(url)-1] == '/' {
			localRedirect(w, req, "../"+path.Base(url))
			return
		}
	}

	// A directory?
	if d.IsDir() {
		if checkLastModified(w, req, d.ModTime()) {
			return
		}
		dirList(w, f, name)
		return
	}

	/*if _, plain := req.URL.Query()["plain"]; plain {
		w.Header().Set("Content-Type", "text/plain")
	}*/

	// If client doesn't accept gzip encoding, serve without compression.
	if !isGzipEncodingAccepted(req) {
		http.ServeContent(w, req, d.Name(), d.ModTime(), f)
		return
	}

	// If the file is not worth gzip compressing, serve it as is.
	if _, ok := f.(notWorthGzipCompressing); ok {
		http.ServeContent(w, req, d.Name(), d.ModTime(), f)
		return
	}

	// If there are gzip encoded bytes available, use them directly.
	if gzipFile, ok := f.(gzipByter); ok {
		w.Header().Set("Content-Encoding", "gzip")
		http.ServeContent(w, req, d.Name(), d.ModTime(), bytes.NewReader(gzipFile.GzipBytes()))
		return
	}

	// Perform compression and serve gzip compressed bytes.
	rs, err := gzipCompress(f)
	if err != nil {
		msg, code := toHTTPError(err)
		http.Error(w, msg, code)
		return
	}
	w.Header().Set("Content-Encoding", "gzip")
	http.ServeContent(w, req, d.Name(), d.ModTime(), rs)
}

func gzipCompress(r io.Reader) (io.ReadSeeker, error) {
	var buf bytes.Buffer
	gw := gzip.NewWriter(&buf)
	_, err := io.Copy(gw, r)
	if err != nil {
		return nil, err
	}
	err = gw.Close()
	if err != nil {
		return nil, err
	}
	return bytes.NewReader(buf.Bytes()), nil
}

type gzipByter interface {
	GzipBytes() []byte
}

type notWorthGzipCompressing interface {
	NotWorthGzipCompressing()
}

func dirList(w http.ResponseWriter, f http.File, name string) {
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	fmt.Fprintf(w, "
\n")
	switch name {
	case "/":
		fmt.Fprintf(w, "%s\n", "/", ".")
	default:
		fmt.Fprintf(w, "%s\n", path.Clean(name+"/.."), "..")
	}
	for {
		dirs, err := f.Readdir(100)
		if err != nil || len(dirs) == 0 {
			break
		}
		sort.Sort(byName(dirs))
		for _, d := range dirs {
			name := d.Name()
			if d.IsDir() {
				name += "/"
			}
			// name may contain '?' or '#', which must be escaped to remain
			// part of the URL path, and not indicate the start of a query
			// string or fragment.
			url := url.URL{Path: name}
			fmt.Fprintf(w, "%s\n", url.String(), html.EscapeString(name))
		}
	}
	fmt.Fprintf(w, "
\n")
}

// localRedirect gives a Moved Permanently response.
// It does not convert relative paths to absolute paths like Redirect does.
func localRedirect(w http.ResponseWriter, r *http.Request, newPath string) {
	if q := r.URL.RawQuery; q != "" {
		newPath += "?" + q
	}
	w.Header().Set("Location", newPath)
	w.WriteHeader(http.StatusMovedPermanently)
}

// modtime is the modification time of the resource to be served, or IsZero().
// return value is whether this request is now complete.
func checkLastModified(w http.ResponseWriter, r *http.Request, modtime time.Time) bool {
	if modtime.IsZero() {
		return false
	}

	// The Date-Modified header truncates sub-second precision, so
	// use mtime < t+1s instead of mtime <= t to check for unmodified.
	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && modtime.Before(t.Add(1*time.Second)) {
		h := w.Header()
		delete(h, "Content-Type")
		delete(h, "Content-Length")
		w.WriteHeader(http.StatusNotModified)
		return true
	}
	w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat))
	return false
}

// byName implements sort.Interface.
type byName []os.FileInfo

func (f byName) Len() int           { return len(f) }
func (f byName) Less(i, j int) bool { return f[i].Name() < f[j].Name() }
func (f byName) Swap(i, j int)      { f[i], f[j] = f[j], f[i] }

// toHTTPError returns a detailed HTTP error message and status code
// for a given non-nil error value.
func toHTTPError(err error) (msg string, httpStatus int) {
	switch {
	case os.IsNotExist(err):
		return fmt.Sprintf("404 Page Not Found\n\n%v", err), http.StatusNotFound
	case os.IsPermission(err):
		return fmt.Sprintf("403 Forbidden\n\n%v", err), http.StatusForbidden
	default:
		return fmt.Sprintf("500 Internal Server Error\n\n%v", err), http.StatusInternalServerError
	}
}

// isGzipEncodingAccepted returns true if the request includes "gzip" under Accept-Encoding header.
func isGzipEncodingAccepted(req *http.Request) bool {
	for _, v := range strings.Split(req.Header.Get("Accept-Encoding"), ",") {
		if strings.TrimSpace(v) == "gzip" {
			return true
		}
	}
	return false
}
<commit_msg>Move vfsgen interfaces closer to usage.<commit_after>// Package gzip_file_server provides a http.Handler that serves the given virtual file system with gzip compression,
// without special handling of index.html, and detailed HTTP error messages.
package gzip_file_server

import (
	"bytes"
	"compress/gzip"
	"fmt"
	"html"
	"io"
	"net/http"
	"net/url"
	"os"
	"path"
	"sort"
	"strings"
	"time"
)

type gzipFileServer struct {
	root http.FileSystem
}

// New returns a gzip file server, that serves the given virtual file system without special handling of index.html.
func New(root http.FileSystem) http.Handler {
	return &gzipFileServer{root: root}
}

func (f *gzipFileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if !strings.HasPrefix(r.URL.Path, "/") {
		r.URL.Path = "/" + r.URL.Path
	}
	f.serveFile(w, r, path.Clean(r.URL.Path))
}

// name is '/'-separated, not filepath.Separator.
func (fs *gzipFileServer) serveFile(w http.ResponseWriter, req *http.Request, name string) {
	f, err := fs.root.Open(name)
	if err != nil {
		msg, code := toHTTPError(err)
		http.Error(w, msg, code)
		return
	}
	defer f.Close()

	d, err := f.Stat()
	if err != nil {
		msg, code := toHTTPError(err)
		http.Error(w, msg, code)
		return
	}

	// redirect to canonical path: / at end of directory url
	// req.URL.Path always begins with /
	url := req.URL.Path
	if d.IsDir() {
		if url[len(url)-1] != '/' {
			localRedirect(w, req, path.Base(url)+"/")
			return
		}
	} else {
		if url[len(url)-1] == '/' {
			localRedirect(w, req, "../"+path.Base(url))
			return
		}
	}

	// A directory?
	if d.IsDir() {
		if checkLastModified(w, req, d.ModTime()) {
			return
		}
		dirList(w, f, name)
		return
	}

	/*if _, plain := req.URL.Query()["plain"]; plain {
		w.Header().Set("Content-Type", "text/plain")
	}*/

	// If client doesn't accept gzip encoding, serve without compression.
	if !isGzipEncodingAccepted(req) {
		http.ServeContent(w, req, d.Name(), d.ModTime(), f)
		return
	}

	// If the file is not worth gzip compressing, serve it as is.
	type notWorthGzipCompressing interface {
		NotWorthGzipCompressing()
	}
	if _, ok := f.(notWorthGzipCompressing); ok {
		http.ServeContent(w, req, d.Name(), d.ModTime(), f)
		return
	}

	// If there are gzip encoded bytes available, use them directly.
	type gzipByter interface {
		GzipBytes() []byte
	}
	if gzipFile, ok := f.(gzipByter); ok {
		w.Header().Set("Content-Encoding", "gzip")
		http.ServeContent(w, req, d.Name(), d.ModTime(), bytes.NewReader(gzipFile.GzipBytes()))
		return
	}

	// Perform compression and serve gzip compressed bytes.
	rs, err := gzipCompress(f)
	if err != nil {
		msg, code := toHTTPError(err)
		http.Error(w, msg, code)
		return
	}
	w.Header().Set("Content-Encoding", "gzip")
	http.ServeContent(w, req, d.Name(), d.ModTime(), rs)
}

func gzipCompress(r io.Reader) (io.ReadSeeker, error) {
	var buf bytes.Buffer
	gw := gzip.NewWriter(&buf)
	_, err := io.Copy(gw, r)
	if err != nil {
		return nil, err
	}
	err = gw.Close()
	if err != nil {
		return nil, err
	}
	return bytes.NewReader(buf.Bytes()), nil
}

func dirList(w http.ResponseWriter, f http.File, name string) {
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	fmt.Fprintf(w, "
\n")
	switch name {
	case "/":
		fmt.Fprintf(w, "%s\n", "/", ".")
	default:
		fmt.Fprintf(w, "%s\n", path.Clean(name+"/.."), "..")
	}
	for {
		dirs, err := f.Readdir(100)
		if err != nil || len(dirs) == 0 {
			break
		}
		sort.Sort(byName(dirs))
		for _, d := range dirs {
			name := d.Name()
			if d.IsDir() {
				name += "/"
			}
			// name may contain '?' or '#', which must be escaped to remain
			// part of the URL path, and not indicate the start of a query
			// string or fragment.
			url := url.URL{Path: name}
			fmt.Fprintf(w, "%s\n", url.String(), html.EscapeString(name))
		}
	}
	fmt.Fprintf(w, "
\n")
}

// localRedirect gives a Moved Permanently response.
// It does not convert relative paths to absolute paths like Redirect does.
func localRedirect(w http.ResponseWriter, r *http.Request, newPath string) {
	if q := r.URL.RawQuery; q != "" {
		newPath += "?" + q
	}
	w.Header().Set("Location", newPath)
	w.WriteHeader(http.StatusMovedPermanently)
}

// modtime is the modification time of the resource to be served, or IsZero().
// return value is whether this request is now complete.
func checkLastModified(w http.ResponseWriter, r *http.Request, modtime time.Time) bool {
	if modtime.IsZero() {
		return false
	}

	// The Date-Modified header truncates sub-second precision, so
	// use mtime < t+1s instead of mtime <= t to check for unmodified.
	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && modtime.Before(t.Add(1*time.Second)) {
		h := w.Header()
		delete(h, "Content-Type")
		delete(h, "Content-Length")
		w.WriteHeader(http.StatusNotModified)
		return true
	}
	w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat))
	return false
}

// byName implements sort.Interface.
type byName []os.FileInfo

func (f byName) Len() int           { return len(f) }
func (f byName) Less(i, j int) bool { return f[i].Name() < f[j].Name() }
func (f byName) Swap(i, j int)      { f[i], f[j] = f[j], f[i] }

// toHTTPError returns a detailed HTTP error message and status code
// for a given non-nil error value.
func toHTTPError(err error) (msg string, httpStatus int) {
	switch {
	case os.IsNotExist(err):
		return fmt.Sprintf("404 Page Not Found\n\n%v", err), http.StatusNotFound
	case os.IsPermission(err):
		return fmt.Sprintf("403 Forbidden\n\n%v", err), http.StatusForbidden
	default:
		return fmt.Sprintf("500 Internal Server Error\n\n%v", err), http.StatusInternalServerError
	}
}

// isGzipEncodingAccepted returns true if the request includes "gzip" under Accept-Encoding header.
func isGzipEncodingAccepted(req *http.Request) bool {
	for _, v := range strings.Split(req.Header.Get("Accept-Encoding"), ",") {
		if strings.TrimSpace(v) == "gzip" {
			return true
		}
	}
	return false
}
<|endoftext|>