<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tls

import (
	"crypto"
	"crypto/rsa"
	"crypto/subtle"
	"crypto/x509"
	"io"
	"os"
)

func (c *Conn) clientHandshake() os.Error {
	finishedHash := newFinishedHash()

	if c.config == nil {
		c.config = defaultConfig()
	}

	hello := &clientHelloMsg{
		vers:               maxVersion,
		cipherSuites:       c.config.cipherSuites(),
		compressionMethods: []uint8{compressionNone},
		random:             make([]byte, 32),
		ocspStapling:       true,
		serverName:         c.config.ServerName,
		supportedCurves:    []uint16{curveP256, curveP384, curveP521},
		supportedPoints:    []uint8{pointFormatUncompressed},
	}

	t := uint32(c.config.time())
	hello.random[0] = byte(t >> 24)
	hello.random[1] = byte(t >> 16)
	hello.random[2] = byte(t >> 8)
	hello.random[3] = byte(t)
	_, err := io.ReadFull(c.config.rand(), hello.random[4:])
	if err != nil {
		c.sendAlert(alertInternalError)
		return os.ErrorString("short read from Rand")
	}

	finishedHash.Write(hello.marshal())
	c.writeRecord(recordTypeHandshake, hello.marshal())

	msg, err := c.readHandshake()
	if err != nil {
		return err
	}
	serverHello, ok := msg.(*serverHelloMsg)
	if !ok {
		return c.sendAlert(alertUnexpectedMessage)
	}
	finishedHash.Write(serverHello.marshal())

	vers, ok := mutualVersion(serverHello.vers)
	if !ok {
		c.sendAlert(alertProtocolVersion)
	}
	c.vers = vers
	c.haveVers = true

	if serverHello.compressionMethod != compressionNone {
		return c.sendAlert(alertUnexpectedMessage)
	}

	suite, suiteId := mutualCipherSuite(c.config.cipherSuites(), serverHello.cipherSuite)
	if suite == nil {
		return c.sendAlert(alertHandshakeFailure)
	}

	msg, err = c.readHandshake()
	if err != nil {
		return err
	}
	certMsg, ok := msg.(*certificateMsg)
	if !ok || len(certMsg.certificates) == 0 {
		return c.sendAlert(alertUnexpectedMessage)
	}
	finishedHash.Write(certMsg.marshal())

	certs := make([]*x509.Certificate, len(certMsg.certificates))
	chain := NewCASet()
	for i, asn1Data := range certMsg.certificates {
		cert, err := x509.ParseCertificate(asn1Data)
		if err != nil {
			c.sendAlert(alertBadCertificate)
			return os.ErrorString("failed to parse certificate from server: " + err.String())
		}
		certs[i] = cert
		chain.AddCert(cert)
	}

	// If we don't have a root CA set configured then anything is accepted.
	// TODO(rsc): Find certificates for OS X 10.6.
	for cur := certs[0]; c.config.RootCAs != nil; {
		parent := c.config.RootCAs.FindVerifiedParent(cur)
		if parent != nil {
			break
		}

		parent = chain.FindVerifiedParent(cur)
		if parent == nil {
			c.sendAlert(alertBadCertificate)
			return os.ErrorString("could not find root certificate for chain")
		}

		if !parent.BasicConstraintsValid || !parent.IsCA {
			c.sendAlert(alertBadCertificate)
			return os.ErrorString("intermediate certificate does not have CA bit set")
		}
		// KeyUsage status flags are ignored. From Engineering
		// Security, Peter Gutmann: A European government CA marked its
		// signing certificates as being valid for encryption only, but
		// no-one noticed. Another European CA marked its signature
		// keys as not being valid for signatures. A different CA
		// marked its own trusted root certificate as being invalid for
		// certificate signing.  Another national CA distributed a
		// certificate to be used to encrypt data for the country’s tax
		// authority that was marked as only being usable for digital
		// signatures but not for encryption. Yet another CA reversed
		// the order of the bit flags in the keyUsage due to confusion
		// over encoding endianness, essentially setting a random
		// keyUsage in certificates that it issued. Another CA created
		// a self-invalidating certificate by adding a certificate
		// policy statement stipulating that the certificate had to be
		// used strictly as specified in the keyUsage, and a keyUsage
		// containing a flag indicating that the RSA encryption key
		// could only be used for Diffie-Hellman key agreement.

		cur = parent
	}

	if _, ok := certs[0].PublicKey.(*rsa.PublicKey); !ok {
		return c.sendAlert(alertUnsupportedCertificate)
	}

	c.peerCertificates = certs

	if serverHello.certStatus {
		msg, err = c.readHandshake()
		if err != nil {
			return err
		}
		cs, ok := msg.(*certificateStatusMsg)
		if !ok {
			return c.sendAlert(alertUnexpectedMessage)
		}
		finishedHash.Write(cs.marshal())

		if cs.statusType == statusTypeOCSP {
			c.ocspResponse = cs.response
		}
	}

	msg, err = c.readHandshake()
	if err != nil {
		return err
	}

	keyAgreement := suite.ka()

	skx, ok := msg.(*serverKeyExchangeMsg)
	if ok {
		finishedHash.Write(skx.marshal())
		err = keyAgreement.processServerKeyExchange(c.config, hello, serverHello, certs[0], skx)
		if err != nil {
			c.sendAlert(alertUnexpectedMessage)
			return err
		}

		msg, err = c.readHandshake()
		if err != nil {
			return err
		}
	}

	transmitCert := false
	certReq, ok := msg.(*certificateRequestMsg)
	if ok {
		// We only accept certificates with RSA keys.
		rsaAvail := false
		for _, certType := range certReq.certificateTypes {
			if certType == certTypeRSASign {
				rsaAvail = true
				break
			}
		}

		// For now, only send a certificate back if the server gives us an
		// empty list of certificateAuthorities.
		//
		// RFC 4346 on the certificateAuthorities field:
		// A list of the distinguished names of acceptable certificate
		// authorities.  These distinguished names may specify a desired
		// distinguished name for a root CA or for a subordinate CA; thus,
		// this message can be used to describe both known roots and a
		// desired authorization space.  If the certificate_authorities
		// list is empty then the client MAY send any certificate of the
		// appropriate ClientCertificateType, unless there is some
		// external arrangement to the contrary.
		if rsaAvail && len(certReq.certificateAuthorities) == 0 {
			transmitCert = true
		}

		finishedHash.Write(certReq.marshal())

		msg, err = c.readHandshake()
		if err != nil {
			return err
		}
	}

	shd, ok := msg.(*serverHelloDoneMsg)
	if !ok {
		return c.sendAlert(alertUnexpectedMessage)
	}
	finishedHash.Write(shd.marshal())

	var cert *x509.Certificate
	if transmitCert {
		certMsg = new(certificateMsg)
		if len(c.config.Certificates) > 0 {
			cert, err = x509.ParseCertificate(c.config.Certificates[0].Certificate[0])
			if err == nil && cert.PublicKeyAlgorithm == x509.RSA {
				certMsg.certificates = c.config.Certificates[0].Certificate
			} else {
				cert = nil
			}
		}
		finishedHash.Write(certMsg.marshal())
		c.writeRecord(recordTypeHandshake, certMsg.marshal())
	}

	preMasterSecret, ckx, err := keyAgreement.generateClientKeyExchange(c.config, hello, certs[0])
	if err != nil {
		c.sendAlert(alertInternalError)
		return err
	}
	if ckx != nil {
		finishedHash.Write(ckx.marshal())
		c.writeRecord(recordTypeHandshake, ckx.marshal())
	}

	if cert != nil {
		certVerify := new(certificateVerifyMsg)
		var digest [36]byte
		copy(digest[0:16], finishedHash.serverMD5.Sum())
		copy(digest[16:36], finishedHash.serverSHA1.Sum())
		signed, err := rsa.SignPKCS1v15(c.config.rand(), c.config.Certificates[0].PrivateKey, crypto.MD5SHA1, digest[0:])
		if err != nil {
			return c.sendAlert(alertInternalError)
		}
		certVerify.signature = signed

		finishedHash.Write(certVerify.marshal())
		c.writeRecord(recordTypeHandshake, certVerify.marshal())
	}

	masterSecret, clientMAC, serverMAC, clientKey, serverKey, clientIV, serverIV :=
		keysFromPreMasterSecret10(preMasterSecret, hello.random, serverHello.random, suite.macLen, suite.keyLen, suite.ivLen)

	clientCipher := suite.cipher(clientKey, clientIV, false /* not for reading */ )
	clientHash := suite.mac(clientMAC)
	c.out.prepareCipherSpec(clientCipher, clientHash)
	c.writeRecord(recordTypeChangeCipherSpec, []byte{1})

	finished := new(finishedMsg)
	finished.verifyData = finishedHash.clientSum(masterSecret)
	finishedHash.Write(finished.marshal())
	c.writeRecord(recordTypeHandshake, finished.marshal())

	serverCipher := suite.cipher(serverKey, serverIV, true /* for reading */ )
	serverHash := suite.mac(serverMAC)
	c.in.prepareCipherSpec(serverCipher, serverHash)
	c.readRecord(recordTypeChangeCipherSpec)
	if c.err != nil {
		return c.err
	}

	msg, err = c.readHandshake()
	if err != nil {
		return err
	}
	serverFinished, ok := msg.(*finishedMsg)
	if !ok {
		return c.sendAlert(alertUnexpectedMessage)
	}

	verify := finishedHash.serverSum(masterSecret)
	if len(verify) != len(serverFinished.verifyData) ||
		subtle.ConstantTimeCompare(verify, serverFinished.verifyData) != 1 {
		return c.sendAlert(alertHandshakeFailure)
	}

	c.handshakeComplete = true
	c.cipherSuite = suiteId
	return nil
}
<commit_msg>crypto/tls: make protocol negotiation failure fatal<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tls

import (
	"crypto"
	"crypto/rsa"
	"crypto/subtle"
	"crypto/x509"
	"io"
	"os"
)

func (c *Conn) clientHandshake() os.Error {
	finishedHash := newFinishedHash()

	if c.config == nil {
		c.config = defaultConfig()
	}

	hello := &clientHelloMsg{
		vers:               maxVersion,
		cipherSuites:       c.config.cipherSuites(),
		compressionMethods: []uint8{compressionNone},
		random:             make([]byte, 32),
		ocspStapling:       true,
		serverName:         c.config.ServerName,
		supportedCurves:    []uint16{curveP256, curveP384, curveP521},
		supportedPoints:    []uint8{pointFormatUncompressed},
	}

	t := uint32(c.config.time())
	hello.random[0] = byte(t >> 24)
	hello.random[1] = byte(t >> 16)
	hello.random[2] = byte(t >> 8)
	hello.random[3] = byte(t)
	_, err := io.ReadFull(c.config.rand(), hello.random[4:])
	if err != nil {
		c.sendAlert(alertInternalError)
		return os.ErrorString("short read from Rand")
	}

	finishedHash.Write(hello.marshal())
	c.writeRecord(recordTypeHandshake, hello.marshal())

	msg, err := c.readHandshake()
	if err != nil {
		return err
	}
	serverHello, ok := msg.(*serverHelloMsg)
	if !ok {
		return c.sendAlert(alertUnexpectedMessage)
	}
	finishedHash.Write(serverHello.marshal())

	vers, ok := mutualVersion(serverHello.vers)
	if !ok {
		return c.sendAlert(alertProtocolVersion)
	}
	c.vers = vers
	c.haveVers = true

	if serverHello.compressionMethod != compressionNone {
		return c.sendAlert(alertUnexpectedMessage)
	}

	suite, suiteId := mutualCipherSuite(c.config.cipherSuites(), serverHello.cipherSuite)
	if suite == nil {
		return c.sendAlert(alertHandshakeFailure)
	}

	msg, err = c.readHandshake()
	if err != nil {
		return err
	}
	certMsg, ok := msg.(*certificateMsg)
	if !ok || len(certMsg.certificates) == 0 {
		return c.sendAlert(alertUnexpectedMessage)
	}
	finishedHash.Write(certMsg.marshal())

	certs := make([]*x509.Certificate, len(certMsg.certificates))
	chain := NewCASet()
	for i, asn1Data := range certMsg.certificates {
		cert, err := x509.ParseCertificate(asn1Data)
		if err != nil {
			c.sendAlert(alertBadCertificate)
			return os.ErrorString("failed to parse certificate from server: " + err.String())
		}
		certs[i] = cert
		chain.AddCert(cert)
	}

	// If we don't have a root CA set configured then anything is accepted.
	// TODO(rsc): Find certificates for OS X 10.6.
	for cur := certs[0]; c.config.RootCAs != nil; {
		parent := c.config.RootCAs.FindVerifiedParent(cur)
		if parent != nil {
			break
		}

		parent = chain.FindVerifiedParent(cur)
		if parent == nil {
			c.sendAlert(alertBadCertificate)
			return os.ErrorString("could not find root certificate for chain")
		}

		if !parent.BasicConstraintsValid || !parent.IsCA {
			c.sendAlert(alertBadCertificate)
			return os.ErrorString("intermediate certificate does not have CA bit set")
		}
		// KeyUsage status flags are ignored. From Engineering
		// Security, Peter Gutmann: A European government CA marked its
		// signing certificates as being valid for encryption only, but
		// no-one noticed. Another European CA marked its signature
		// keys as not being valid for signatures. A different CA
		// marked its own trusted root certificate as being invalid for
		// certificate signing.  Another national CA distributed a
		// certificate to be used to encrypt data for the country’s tax
		// authority that was marked as only being usable for digital
		// signatures but not for encryption. Yet another CA reversed
		// the order of the bit flags in the keyUsage due to confusion
		// over encoding endianness, essentially setting a random
		// keyUsage in certificates that it issued. Another CA created
		// a self-invalidating certificate by adding a certificate
		// policy statement stipulating that the certificate had to be
		// used strictly as specified in the keyUsage, and a keyUsage
		// containing a flag indicating that the RSA encryption key
		// could only be used for Diffie-Hellman key agreement.

		cur = parent
	}

	if _, ok := certs[0].PublicKey.(*rsa.PublicKey); !ok {
		return c.sendAlert(alertUnsupportedCertificate)
	}

	c.peerCertificates = certs

	if serverHello.certStatus {
		msg, err = c.readHandshake()
		if err != nil {
			return err
		}
		cs, ok := msg.(*certificateStatusMsg)
		if !ok {
			return c.sendAlert(alertUnexpectedMessage)
		}
		finishedHash.Write(cs.marshal())

		if cs.statusType == statusTypeOCSP {
			c.ocspResponse = cs.response
		}
	}

	msg, err = c.readHandshake()
	if err != nil {
		return err
	}

	keyAgreement := suite.ka()

	skx, ok := msg.(*serverKeyExchangeMsg)
	if ok {
		finishedHash.Write(skx.marshal())
		err = keyAgreement.processServerKeyExchange(c.config, hello, serverHello, certs[0], skx)
		if err != nil {
			c.sendAlert(alertUnexpectedMessage)
			return err
		}

		msg, err = c.readHandshake()
		if err != nil {
			return err
		}
	}

	transmitCert := false
	certReq, ok := msg.(*certificateRequestMsg)
	if ok {
		// We only accept certificates with RSA keys.
		rsaAvail := false
		for _, certType := range certReq.certificateTypes {
			if certType == certTypeRSASign {
				rsaAvail = true
				break
			}
		}

		// For now, only send a certificate back if the server gives us an
		// empty list of certificateAuthorities.
		//
		// RFC 4346 on the certificateAuthorities field:
		// A list of the distinguished names of acceptable certificate
		// authorities.  These distinguished names may specify a desired
		// distinguished name for a root CA or for a subordinate CA; thus,
		// this message can be used to describe both known roots and a
		// desired authorization space.  If the certificate_authorities
		// list is empty then the client MAY send any certificate of the
		// appropriate ClientCertificateType, unless there is some
		// external arrangement to the contrary.
		if rsaAvail && len(certReq.certificateAuthorities) == 0 {
			transmitCert = true
		}

		finishedHash.Write(certReq.marshal())

		msg, err = c.readHandshake()
		if err != nil {
			return err
		}
	}

	shd, ok := msg.(*serverHelloDoneMsg)
	if !ok {
		return c.sendAlert(alertUnexpectedMessage)
	}
	finishedHash.Write(shd.marshal())

	var cert *x509.Certificate
	if transmitCert {
		certMsg = new(certificateMsg)
		if len(c.config.Certificates) > 0 {
			cert, err = x509.ParseCertificate(c.config.Certificates[0].Certificate[0])
			if err == nil && cert.PublicKeyAlgorithm == x509.RSA {
				certMsg.certificates = c.config.Certificates[0].Certificate
			} else {
				cert = nil
			}
		}
		finishedHash.Write(certMsg.marshal())
		c.writeRecord(recordTypeHandshake, certMsg.marshal())
	}

	preMasterSecret, ckx, err := keyAgreement.generateClientKeyExchange(c.config, hello, certs[0])
	if err != nil {
		c.sendAlert(alertInternalError)
		return err
	}
	if ckx != nil {
		finishedHash.Write(ckx.marshal())
		c.writeRecord(recordTypeHandshake, ckx.marshal())
	}

	if cert != nil {
		certVerify := new(certificateVerifyMsg)
		var digest [36]byte
		copy(digest[0:16], finishedHash.serverMD5.Sum())
		copy(digest[16:36], finishedHash.serverSHA1.Sum())
		signed, err := rsa.SignPKCS1v15(c.config.rand(), c.config.Certificates[0].PrivateKey, crypto.MD5SHA1, digest[0:])
		if err != nil {
			return c.sendAlert(alertInternalError)
		}
		certVerify.signature = signed

		finishedHash.Write(certVerify.marshal())
		c.writeRecord(recordTypeHandshake, certVerify.marshal())
	}

	masterSecret, clientMAC, serverMAC, clientKey, serverKey, clientIV, serverIV :=
		keysFromPreMasterSecret10(preMasterSecret, hello.random, serverHello.random, suite.macLen, suite.keyLen, suite.ivLen)

	clientCipher := suite.cipher(clientKey, clientIV, false /* not for reading */ )
	clientHash := suite.mac(clientMAC)
	c.out.prepareCipherSpec(clientCipher, clientHash)
	c.writeRecord(recordTypeChangeCipherSpec, []byte{1})

	finished := new(finishedMsg)
	finished.verifyData = finishedHash.clientSum(masterSecret)
	finishedHash.Write(finished.marshal())
	c.writeRecord(recordTypeHandshake, finished.marshal())

	serverCipher := suite.cipher(serverKey, serverIV, true /* for reading */ )
	serverHash := suite.mac(serverMAC)
	c.in.prepareCipherSpec(serverCipher, serverHash)
	c.readRecord(recordTypeChangeCipherSpec)
	if c.err != nil {
		return c.err
	}

	msg, err = c.readHandshake()
	if err != nil {
		return err
	}
	serverFinished, ok := msg.(*finishedMsg)
	if !ok {
		return c.sendAlert(alertUnexpectedMessage)
	}

	verify := finishedHash.serverSum(masterSecret)
	if len(verify) != len(serverFinished.verifyData) ||
		subtle.ConstantTimeCompare(verify, serverFinished.verifyData) != 1 {
		return c.sendAlert(alertHandshakeFailure)
	}

	c.handshakeComplete = true
	c.cipherSuite = suiteId
	return nil
}
<|endoftext|>


<commit_before>package alp

import (
	"fmt"
	"io"
	"os"

	"github.com/tkuchiki/alp/errors"

	"github.com/tkuchiki/alp/stats"

	"github.com/tkuchiki/alp/flags"
	"github.com/tkuchiki/alp/options"
	"github.com/tkuchiki/alp/parsers"
	"gopkg.in/alecthomas/kingpin.v2"
)

const version = "0.4.0"

type Profiler struct {
	outWriter    io.Writer
	errWriter    io.Writer
	inReader     *os.File
	optionParser *kingpin.Application
	subcmdLTSV   *kingpin.CmdClause
	subcmdRegexp *kingpin.CmdClause
	subcmdJSON   *kingpin.CmdClause
	globalFlags  *flags.GlobalFlags
	ltsvFlags    *flags.LTSVFlags
	regexpFlags  *flags.RegexpFlags
	jsonFlags    *flags.JSONFlags
}

func NewProfiler(outw, errw io.Writer) *Profiler {
	app := kingpin.New("alp", "Access Log Profiler for LTSV (read from file or stdin).")
	p := &Profiler{
		outWriter:    outw,
		errWriter:    errw,
		inReader:     os.Stdin,
		optionParser: app,
	}

	p.subcmdLTSV = app.Command("ltsv", "ltsv")
	p.subcmdRegexp = app.Command("regexp", "regexp")
	p.subcmdJSON = app.Command("json", "json")

	p.globalFlags = flags.NewGlobalFlags()
	p.globalFlags.InitGlobalFlags(p.optionParser)

	p.ltsvFlags = flags.NewLTSVFlags()
	p.ltsvFlags.InitFlags(p.subcmdLTSV)

	p.regexpFlags = flags.NewRegexpFlags()
	p.regexpFlags.InitFlags(p.subcmdRegexp)

	p.jsonFlags = flags.NewJSONFlags()
	p.jsonFlags.InitFlags(p.subcmdJSON)

	return p
}

func (p *Profiler) SetFlags(flags *flags.GlobalFlags) {
	p.globalFlags = flags
}

func (p *Profiler) SetInReader(f *os.File) {
	p.inReader = f
}

func (p *Profiler) Open(filename string) (*os.File, error) {
	var f *os.File
	var err error

	if filename != "" {
		f, err = os.Open(filename)
	} else {
		f = p.inReader
	}

	return f, err
}

func (p *Profiler) Run() error {
	var command string
	p.optionParser.Version(version)
	switch kingpin.MustParse(p.optionParser.Parse(os.Args[1:])) {
	case p.subcmdLTSV.FullCommand():
		command = p.subcmdLTSV.FullCommand()
	case p.subcmdRegexp.FullCommand():
		command = p.subcmdRegexp.FullCommand()
	case p.subcmdJSON.FullCommand():
		command = p.subcmdJSON.FullCommand()
	}

	sort := flags.SortOptions[p.globalFlags.Sort]

	var err error
	var opts *options.Options
	if p.globalFlags.Config != "" {
		cf, err := os.Open(p.globalFlags.Config)
		if err != nil {
			return err
		}
		defer cf.Close()

		opts, err = options.LoadOptionsFromReader(cf)
		if err != nil {
			return err
		}
	} else {
		opts = options.NewOptions()
	}

	opts = options.SetOptions(opts,
		options.File(p.globalFlags.File),
		options.Sort(sort),
		options.Reverse(p.globalFlags.Reverse),
		options.QueryString(p.globalFlags.QueryString),
		options.Tsv(p.globalFlags.Tsv),
		options.Limit(p.globalFlags.Limit),
		options.Location(p.globalFlags.Location),
		options.Output(p.globalFlags.Output),
		options.NoHeaders(p.globalFlags.NoHeaders),
		options.CSVGroups(p.globalFlags.MatchingGroups),
		options.Filters(p.globalFlags.Filters),
		//ltsv
		options.ApptimeLabel(p.ltsvFlags.ApptimeLabel),
		options.StatusLabel(p.ltsvFlags.StatusLabel),
		options.SizeLabel(p.ltsvFlags.SizeLabel),
		options.MethodLabel(p.ltsvFlags.MethodLabel),
		options.UriLabel(p.ltsvFlags.UriLabel),
		options.TimeLabel(p.ltsvFlags.TimeLabel),
	)

	sts := stats.NewHTTPStats(true, false, false)

	err = sts.InitFilter(opts)
	if err != nil {
		return err
	}

	sts.SetOptions(opts)

	printFormat := "table"
	if opts.Tsv {
		printFormat = "tsv"
	}
	printer := stats.NewPrinter(p.outWriter, opts.Output, printFormat)
	if err = printer.Validate(); err != nil {
		return err
	}

	if p.globalFlags.Load != "" {
		lf, err := os.Open(p.globalFlags.Load)
		if err != nil {
			return err
		}
		err = sts.LoadStats(lf)
		if err != nil {
			return err
		}
		defer lf.Close()

		sts.SortWithOptions()
		printer.Print(sts)
		return nil
	}

	f, err := p.Open(opts.File)
	if err != nil {
		return err
	}
	defer f.Close()

	if len(opts.MatchingGroups) > 0 {
		err = sts.SetURIMatchingGroups(opts.MatchingGroups)
		if err != nil {
			return err
		}
	}

	var parser parsers.Parser
	switch command {
	case "ltsv":
		label := parsers.NewLTSVLabel(opts.LTSV.UriLabel, opts.LTSV.MethodLabel, opts.LTSV.TimeLabel,
			opts.LTSV.ApptimeLabel, opts.LTSV.SizeLabel, opts.LTSV.StatusLabel,
		)
		parser = parsers.NewLTSVParser(f, label, opts.QueryString)
	case "regexp":
		expr := `^(\S+)\s` + // remote host
			`\S+\s+` +
			`(\S+\s+)+` + // user
			`\[(?P<time>[^]]+)\]\s` + // time
			`"(?P<method>\S*)\s?` + // method
			`(?P<uri>(?:[^"]*(?:\\")?)*)\s` + // URL
			`([^"]*)"\s` + // protocol
			`(?P<status>\S+)\s` + // status code
			`(?P<body_bytes>\S+)\s` + // bytes
			`"((?:[^"]*(?:\\")?)*)"\s` + // referer
			`"(.*)"` + // user agent
			`\s(?P<response_time>.*)$`
		names := parsers.NewSubexpNames("", "", "", "", "", "")
		parser, err = parsers.NewRegexpParser(f, expr, names, opts.QueryString)
	case "json":
		keys := parsers.NewJSONKeys("", "", "", "", "", "")
		parser = parsers.NewJSONParser(f, keys, opts.QueryString)
	}

	if err != nil {
		return err
	}

Loop:
	for {
		s, err := parser.Parse()
		if err != nil {
			if err == io.EOF {
				break
			} else if err == errors.SkipReadLineErr {
				continue Loop
			}

			return err
		}

		var b bool
		b, err = sts.DoFilter(s)
		if err != nil {
			return err
		}

		if !b {
			continue Loop
		}

		sts.Set(s.Uri, s.Method, s.Status, s.ResponseTime, s.BodyBytes, 0)

		if sts.CountUris() > opts.Limit {
			return fmt.Errorf("Too many URI's (%d or less)", opts.Limit)
		}
	}

	if p.globalFlags.Dump != "" {
		df, err := os.OpenFile(p.globalFlags.Dump, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
		err = sts.DumpStats(df)
		if err != nil {
			return err
		}
		defer df.Close()
	}

	sts.SortWithOptions()
	printer.Print(sts)

	return nil
}
<commit_msg>Fix sub command's help message / refactor<commit_after>package alp

import (
	"fmt"
	"io"
	"os"

	"github.com/tkuchiki/alp/errors"

	"github.com/tkuchiki/alp/stats"

	"github.com/tkuchiki/alp/flags"
	"github.com/tkuchiki/alp/options"
	"github.com/tkuchiki/alp/parsers"
	"gopkg.in/alecthomas/kingpin.v2"
)

const version = "0.4.0"

type Profiler struct {
	outWriter    io.Writer
	errWriter    io.Writer
	inReader     *os.File
	optionParser *kingpin.Application
	subcmdLTSV   *kingpin.CmdClause
	subcmdRegexp *kingpin.CmdClause
	subcmdJSON   *kingpin.CmdClause
	globalFlags  *flags.GlobalFlags
	ltsvFlags    *flags.LTSVFlags
	regexpFlags  *flags.RegexpFlags
	jsonFlags    *flags.JSONFlags
}

func NewProfiler(outw, errw io.Writer) *Profiler {
	app := kingpin.New("alp", "alp is the access log profiler for LTSV, JSON, and others.")
	p := &Profiler{
		outWriter:    outw,
		errWriter:    errw,
		inReader:     os.Stdin,
		optionParser: app,
	}

	p.globalFlags = flags.NewGlobalFlags()
	p.globalFlags.InitGlobalFlags(p.optionParser)

	p.subcmdLTSV = app.Command("ltsv", "Profile the logs for LTSV")
	p.ltsvFlags = flags.NewLTSVFlags()
	p.ltsvFlags.InitFlags(p.subcmdLTSV)

	p.subcmdJSON = app.Command("json", "Profile the logs for JSON")
	p.jsonFlags = flags.NewJSONFlags()
	p.jsonFlags.InitFlags(p.subcmdJSON)

	p.subcmdRegexp = app.Command("regexp", "Profile the logs that match a regular expression")
	p.regexpFlags = flags.NewRegexpFlags()
	p.regexpFlags.InitFlags(p.subcmdRegexp)

	return p
}

func (p *Profiler) SetFlags(flags *flags.GlobalFlags) {
	p.globalFlags = flags
}

func (p *Profiler) SetInReader(f *os.File) {
	p.inReader = f
}

func (p *Profiler) Open(filename string) (*os.File, error) {
	var f *os.File
	var err error

	if filename != "" {
		f, err = os.Open(filename)
	} else {
		f = p.inReader
	}

	return f, err
}

func (p *Profiler) Run() error {
	var command string
	p.optionParser.Version(version)
	switch kingpin.MustParse(p.optionParser.Parse(os.Args[1:])) {
	case p.subcmdLTSV.FullCommand():
		command = p.subcmdLTSV.FullCommand()
	case p.subcmdRegexp.FullCommand():
		command = p.subcmdRegexp.FullCommand()
	case p.subcmdJSON.FullCommand():
		command = p.subcmdJSON.FullCommand()
	}

	sort := flags.SortOptions[p.globalFlags.Sort]

	var err error
	var opts *options.Options
	if p.globalFlags.Config != "" {
		cf, err := os.Open(p.globalFlags.Config)
		if err != nil {
			return err
		}
		defer cf.Close()

		opts, err = options.LoadOptionsFromReader(cf)
		if err != nil {
			return err
		}
	} else {
		opts = options.NewOptions()
	}

	opts = options.SetOptions(opts,
		options.File(p.globalFlags.File),
		options.Sort(sort),
		options.Reverse(p.globalFlags.Reverse),
		options.QueryString(p.globalFlags.QueryString),
		options.Format(p.globalFlags.Format),
		options.Limit(p.globalFlags.Limit),
		options.Location(p.globalFlags.Location),
		options.Output(p.globalFlags.Output),
		options.NoHeaders(p.globalFlags.NoHeaders),
		options.CSVGroups(p.globalFlags.MatchingGroups),
		options.Filters(p.globalFlags.Filters),
		// ltsv
		options.UriLabel(p.ltsvFlags.UriLabel),
		options.MethodLabel(p.ltsvFlags.MethodLabel),
		options.TimeLabel(p.ltsvFlags.TimeLabel),
		options.ApptimeLabel(p.ltsvFlags.ApptimeLabel),
		options.SizeLabel(p.ltsvFlags.SizeLabel),
		options.StatusLabel(p.ltsvFlags.StatusLabel),
		// json
		options.UriKey(p.jsonFlags.UriKey),
		options.MethodKey(p.jsonFlags.MethodKey),
		options.TimeKey(p.jsonFlags.TimeKey),
		options.ResponseTimeKey(p.jsonFlags.ResponseTimeKey),
		options.BodyBytesKey(p.jsonFlags.BodyBytesKey),
		options.StatusKey(p.jsonFlags.StatusKey),
		// regexp
		options.UriSubexp(p.regexpFlags.UriSubexp),
		options.MethodSubexp(p.regexpFlags.MethodSubexp),
		options.TimeSubexp(p.regexpFlags.TimeSubexp),
		options.ResponseTimeSubexp(p.regexpFlags.ResponseTimeSubexp),
		options.BodyBytesSubexp(p.regexpFlags.BodyBytesSubexp),
		options.StatusSubexp(p.regexpFlags.StatusSubexp),
	)

	sts := stats.NewHTTPStats(true, false, false)

	err = sts.InitFilter(opts)
	if err != nil {
		return err
	}

	sts.SetOptions(opts)

	printer := stats.NewPrinter(p.outWriter, opts.Output, opts.Format)
	if err = printer.Validate(); err != nil {
		return err
	}

	if p.globalFlags.Load != "" {
		lf, err := os.Open(p.globalFlags.Load)
		if err != nil {
			return err
		}
		err = sts.LoadStats(lf)
		if err != nil {
			return err
		}
		defer lf.Close()

		sts.SortWithOptions()
		printer.Print(sts)
		return nil
	}

	f, err := p.Open(opts.File)
	if err != nil {
		return err
	}
	defer f.Close()

	if len(opts.MatchingGroups) > 0 {
		err = sts.SetURIMatchingGroups(opts.MatchingGroups)
		if err != nil {
			return err
		}
	}

	var parser parsers.Parser
	switch command {
	case "ltsv":
		label := parsers.NewLTSVLabel(opts.LTSV.UriLabel, opts.LTSV.MethodLabel, opts.LTSV.TimeLabel,
			opts.LTSV.ApptimeLabel, opts.LTSV.SizeLabel, opts.LTSV.StatusLabel,
		)
		parser = parsers.NewLTSVParser(f, label, opts.QueryString)
	case "json":
		keys := parsers.NewJSONKeys(opts.JSON.UriKey, opts.JSON.MethodKey, opts.JSON.TimeKey,
			opts.JSON.ResponseTimeKey, opts.JSON.BodyBytesKey, opts.JSON.StatusKey)
		parser = parsers.NewJSONParser(f, keys, opts.QueryString)
	case "regexp":
		names := parsers.NewSubexpNames(opts.Regexp.UriSubexp, opts.Regexp.MethodSubexp, opts.Regexp.TimeSubexp,
			opts.Regexp.ResponseTimeSubexp, opts.Regexp.BodyBytesSubexp, opts.Regexp.StatusSubexp)
		parser, err = parsers.NewRegexpParser(f, options.DefaultPatternOption, names, opts.QueryString)
	}

	if err != nil {
		return err
	}

Loop:
	for {
		s, err := parser.Parse()
		if err != nil {
			if err == io.EOF {
				break
			} else if err == errors.SkipReadLineErr {
				continue Loop
			}

			return err
		}

		var b bool
		b, err = sts.DoFilter(s)
		if err != nil {
			return err
		}

		if !b {
			continue Loop
		}

		sts.Set(s.Uri, s.Method, s.Status, s.ResponseTime, s.BodyBytes, 0)

		if sts.CountUris() > opts.Limit {
			return fmt.Errorf("Too many URI's (%d or less)", opts.Limit)
		}
	}

	if p.globalFlags.Dump != "" {
		df, err := os.OpenFile(p.globalFlags.Dump, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
		err = sts.DumpStats(df)
		if err != nil {
			return err
		}
		defer df.Close()
	}

	sts.SortWithOptions()
	printer.Print(sts)

	return nil
}
<|endoftext|>


<commit_before>package app

import (
	"encoding/base64"
	"net"
	"net/http"
	"net/http/httputil"
	"strconv"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/prometheus/client_golang/prometheus"
)

func respond(log *logrus.Entry, w http.ResponseWriter, req *http.Request, reply string, status int) {
	host, _, err := net.SplitHostPort(req.RemoteAddr)
	if err != nil {
		host = req.RemoteAddr
	}

	if status >= 400 {
		log.Warnf("Failed to serve URL: %s", reply)
	}
	HTTPResponseCtr.With(prometheus.Labels{"code": string(strconv.Itoa(status))}).Inc()
	log.Printf("%s %s %s %d \"%s\"", host, req.Method, req.RequestURI, status, reply)
	http.Error(w, reply, status)
}

func serveBasicAuth(backend Backend, w http.ResponseWriter, req *http.Request) bool {
	if authInfo := backend.GetInfo().BasicAuth; authInfo != nil {
		authError := func() bool {
			w.Header().Set("WWW-Authenticate", "Basic realm=\"Restricted Access\"")
			respond(backend.GetLogger(), w, req, "authorization failed", http.StatusUnauthorized)
			return true
		}

		auth := strings.SplitN(req.Header.Get("Authorization"), " ", 2)
		if len(auth) != 2 || auth[0] != "Basic" {
			return authError()
		}

		payload, err := base64.StdEncoding.DecodeString(auth[1])
		if err != nil {
			return authError()
		}

		pair := strings.SplitN(string(payload), ":", 2)
		if len(pair) != 2 || !(pair[0] == authInfo.Username && pair[1] == authInfo.Password) {
			return authError()
		}
	}
	return false
}

func forward(w http.ResponseWriter, req *http.Request) {
	backend := LookupBackend(req.Host, req.URL.Path)

	if backend == nil {
		log := logrus.New().WithField("type", "access_log")
		log.Logger = logrus.StandardLogger()
		respond(log, w, req, "Path not mapped", http.StatusNotFound)
		return
	}
	log := backend.GetLogger().WithField("type", "access_log")
	log.Logger = logrus.StandardLogger()
	log.Infof("%s %s%s", req.Method, req.Host, req.URL.Path)

	if serveBasicAuth(backend, w, req) {
		return
	}

	// handles /__ug_auth?code=$code
	if serveValidateServerAuth(backend, w, req) {
		return
	}

	// if server auth is enabled, verify that the user is authenticated.
	if serveServerAuth(backend, w, req) {
		return
	}

	if serveProgress(backend, w, req) {
		return
	}

	if serveStatic(backend, w, req) {
		return
	}

	conn := backend.Connect()
	if conn == nil {
		respond(log, w, req, "Couldn't connect to backend server", http.StatusServiceUnavailable)
		return
	}

	director := func(req *http.Request) {
		req.URL.Path = backend.GetInfo().Backend.BasePath + strings.TrimPrefix(req.URL.Path, backend.GetInfo().Prefix)
		req.URL.Scheme = "http"
		req.URL.Host = backend.GetInfo().Backend.Address
	}

	var revProxy http.Handler
	if isWebsocket(req) {
		revProxy = &websocketReverseProxy{
			Backend:  backend,
			Director: director,
			Dial: func(network, addr string) (net.Conn, error) {
				return conn, nil
			},
		}

	} else {
		revProxy = &httputil.ReverseProxy{
			Director: director,
			Transport: &http.Transport{
				Dial: func(network, addr string) (net.Conn, error) {
					return conn, nil
				},
			},
		}
	}
	revProxy.ServeHTTP(w, req)
}
<commit_msg>Explicitly close SSH->backend connection when done<commit_after>package app

import (
	"encoding/base64"
	"net"
	"net/http"
	"net/http/httputil"
	"strconv"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/prometheus/client_golang/prometheus"
)

func respond(log *logrus.Entry, w http.ResponseWriter, req *http.Request, reply string, status int) {
	host, _, err := net.SplitHostPort(req.RemoteAddr)
	if err != nil {
		host = req.RemoteAddr
	}

	if status >= 400 {
		log.Warnf("Failed to serve URL: %s", reply)
	}
	HTTPResponseCtr.With(prometheus.Labels{"code": string(strconv.Itoa(status))}).Inc()
	log.Printf("%s %s %s %d \"%s\"", host, req.Method, req.RequestURI, status, reply)
	http.Error(w, reply, status)
}

func serveBasicAuth(backend Backend, w http.ResponseWriter, req *http.Request) bool {
	if authInfo := backend.GetInfo().BasicAuth; authInfo != nil {
		authError := func() bool {
			w.Header().Set("WWW-Authenticate", "Basic realm=\"Restricted Access\"")
			respond(backend.GetLogger(), w, req, "authorization failed", http.StatusUnauthorized)
			return true
		}

		auth := strings.SplitN(req.Header.Get("Authorization"), " ", 2)
		if len(auth) != 2 || auth[0] != "Basic" {
			return authError()
		}

		payload, err := base64.StdEncoding.DecodeString(auth[1])
		if err != nil {
			return authError()
		}

		pair := strings.SplitN(string(payload), ":", 2)
		if len(pair) != 2 || !(pair[0] == authInfo.Username && pair[1] == authInfo.Password) {
			return authError()
		}
	}
	return false
}

func forward(w http.ResponseWriter, req *http.Request) {
	backend := LookupBackend(req.Host, req.URL.Path)

	if backend == nil {
		log := logrus.New().WithField("type", "access_log")
		log.Logger = logrus.StandardLogger()
		respond(log, w, req, "Path not mapped", http.StatusNotFound)
		return
	}
	log := backend.GetLogger().WithField("type", "access_log")
	log.Logger = logrus.StandardLogger()
	log.Infof("%s %s%s", req.Method, req.Host, req.URL.Path)

	if serveBasicAuth(backend, w, req) {
		return
	}

	// handles /__ug_auth?code=$code
	if serveValidateServerAuth(backend, w, req) {
		return
	}

	// if server auth is enabled, verify that the user is authenticated.
	if serveServerAuth(backend, w, req) {
		return
	}

	if serveProgress(backend, w, req) {
		return
	}

	if serveStatic(backend, w, req) {
		return
	}

	conn := backend.Connect()
	if conn == nil {
		respond(log, w, req, "Couldn't connect to backend server", http.StatusServiceUnavailable)
		return
	}
	defer conn.Close()

	director := func(req *http.Request) {
		req.URL.Path = backend.GetInfo().Backend.BasePath + strings.TrimPrefix(req.URL.Path, backend.GetInfo().Prefix)
		req.URL.Scheme = "http"
		req.URL.Host = backend.GetInfo().Backend.Address
	}

	var revProxy http.Handler
	if isWebsocket(req) {
		revProxy = &websocketReverseProxy{
			Backend:  backend,
			Director: director,
			Dial: func(network, addr string) (net.Conn, error) {
				return conn, nil
			},
		}

	} else {
		revProxy = &httputil.ReverseProxy{
			Director: director,
			Transport: &http.Transport{
				Dial: func(network, addr string) (net.Conn, error) {
					return conn, nil
				},
			},
		}
	}
	revProxy.ServeHTTP(w, req)
}
<|endoftext|>


<commit_before>// vim: ts=8 sw=8 noet ai

package perigee

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
)

// The UnexpectedResponseCodeError structure represents a mismatch in understanding between server and client in terms of response codes.
// Most often, this is due to an actual error condition (e.g., getting a 404 for a resource when you expect a 200).
// However, it needn't always be the case (e.g., getting a 204 (No Content) response back when a 200 is expected).
type UnexpectedResponseCodeError struct {
	Expected []int
	Actual   int
}

func (err *UnexpectedResponseCodeError) Error() string {
	return fmt.Sprintf("Expected HTTP response code %d; got %d instead", err.Expected, err.Actual)
}

// request is the procedure that does the ditch-work of making the request, marshaling parameters, and unmarshaling results.
func request(method string, url string, opts Options) error {
	var body io.Reader

	acceptableResponseCodes := opts.OkCodes
	if len(acceptableResponseCodes) == 0 {
		acceptableResponseCodes = []int{200}
	}

	client := opts.CustomClient
	if client == nil {
		client = new(http.Client)
	}

	body = nil
	if opts.ReqBody != nil {
		bodyText, err := json.Marshal(opts.ReqBody)
		if err != nil {
			return err
		}
		body = strings.NewReader(string(bodyText))
		if opts.DumpReqJson {
			log.Printf("Making request:\n%#v\n", string(bodyText))
		}
	}

	req, err := http.NewRequest(method, url, body)
	if err != nil {
		return err
	}
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("Accept", "application/json")
	if opts.MoreHeaders != nil {
		for k, v := range opts.MoreHeaders {
			req.Header.Add(k, v)
		}
	}

	response, err := client.Do(req)
	if err != nil {
		return err
	}
	if not_in(response.StatusCode, acceptableResponseCodes) {
		return &UnexpectedResponseCodeError{
			Expected: acceptableResponseCodes,
			Actual:   response.StatusCode,
		}
	}
	defer response.Body.Close()
	if opts.Results != nil {
		jsonResult, err := ioutil.ReadAll(response.Body)
		if err != nil {
			return err
		}

		err = json.Unmarshal(jsonResult, opts.Results)
		if opts.ResponseJson != nil {
			*opts.ResponseJson = jsonResult
		}
	}
	return err
}

// not_in returns false if, and only if, the provided needle is _not_
// in the given set of integers.
func not_in(needle int, haystack []int) bool {
	for _, straw := range haystack {
		if needle == straw {
			return false
		}
	}
	return true
}

// Post makes a POST request against a server using the provided HTTP client.
// The url must be a fully-formed URL string.
func Post(url string, opts Options) error {
	return request("POST", url, opts)
}

// Get makes a GET request against a server using the provided HTTP client.
// The url must be a fully-formed URL string.
func Get(url string, opts Options) error {
	return request("GET", url, opts)
}

// Delete makes a DELETE request against a server using the provided HTTP client.
// The url must be a fully-formed URL string.
func Delete(url string, opts Options) error {
	return request("DELETE", url, opts)
}

// Put makes a PUT request against a server using the provided HTTP client.
// The url must be a fully-formed URL string.
func Put(url string, opts Options) error {
	return request("PUT", url, opts)
}

// Options describes a set of optional parameters to the various request calls.
//
// The custom client can be used for a variety of purposes beyond selecting encrypted versus unencrypted channels.
// Transports can be defined to provide augmented logging, header manipulation, et. al.
//
// If the ReqBody field is provided, it will be embedded as a JSON object.
// Otherwise, provide nil.
//
// If JSON output is to be expected from the response,
// provide either a pointer to the container structure in Results,
// or a pointer to a nil-initialized pointer variable.
// The latter method will cause the unmarshaller to allocate the container type for you.
// If no response is expected, provide a nil Results value.
//
// The MoreHeaders map, if non-nil or empty, provides a set of headers to add to those
// already present in the request.  At present, only Accepted and Content-Type are set
// by default.
type Options struct {
	CustomClient *http.Client
	ReqBody      interface{}
	Results      interface{}
	MoreHeaders  map[string]string
	OkCodes      []int
	DumpReqJson  bool
	ResponseJson *[]byte
}
<commit_msg>Add StatusCode return path.<commit_after>// vim: ts=8 sw=8 noet ai

package perigee

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
)

// The UnexpectedResponseCodeError structure represents a mismatch in understanding between server and client in terms of response codes.
// Most often, this is due to an actual error condition (e.g., getting a 404 for a resource when you expect a 200).
// However, it needn't always be the case (e.g., getting a 204 (No Content) response back when a 200 is expected).
type UnexpectedResponseCodeError struct {
	Expected []int
	Actual   int
}

func (err *UnexpectedResponseCodeError) Error() string {
	return fmt.Sprintf("Expected HTTP response code %d; got %d instead", err.Expected, err.Actual)
}

// request is the procedure that does the ditch-work of making the request, marshaling parameters, and unmarshaling results.
func request(method string, url string, opts Options) error {
	var body io.Reader

	acceptableResponseCodes := opts.OkCodes
	if len(acceptableResponseCodes) == 0 {
		acceptableResponseCodes = []int{200}
	}

	client := opts.CustomClient
	if client == nil {
		client = new(http.Client)
	}

	body = nil
	if opts.ReqBody != nil {
		bodyText, err := json.Marshal(opts.ReqBody)
		if err != nil {
			return err
		}
		body = strings.NewReader(string(bodyText))
		if opts.DumpReqJson {
			log.Printf("Making request:\n%#v\n", string(bodyText))
		}
	}

	req, err := http.NewRequest(method, url, body)
	if err != nil {
		return err
	}
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("Accept", "application/json")
	if opts.MoreHeaders != nil {
		for k, v := range opts.MoreHeaders {
			req.Header.Add(k, v)
		}
	}

	response, err := client.Do(req)
	if err != nil {
		return err
	}
	if opts.StatusCode != nil {
		*opts.StatusCode = response.StatusCode
	}
	if not_in(response.StatusCode, acceptableResponseCodes) {
		return &UnexpectedResponseCodeError{
			Expected: acceptableResponseCodes,
			Actual:   response.StatusCode,
		}
	}
	defer response.Body.Close()
	if opts.Results != nil {
		jsonResult, err := ioutil.ReadAll(response.Body)
		if err != nil {
			return err
		}

		err = json.Unmarshal(jsonResult, opts.Results)
		if opts.ResponseJson != nil {
			*opts.ResponseJson = jsonResult
		}
	}
	return err
}

// not_in returns false if, and only if, the provided needle is _not_
// in the given set of integers.
func not_in(needle int, haystack []int) bool {
	for _, straw := range haystack {
		if needle == straw {
			return false
		}
	}
	return true
}

// Post makes a POST request against a server using the provided HTTP client.
// The url must be a fully-formed URL string.
func Post(url string, opts Options) error {
	return request("POST", url, opts)
}

// Get makes a GET request against a server using the provided HTTP client.
// The url must be a fully-formed URL string.
func Get(url string, opts Options) error {
	return request("GET", url, opts)
}

// Delete makes a DELETE request against a server using the provided HTTP client.
// The url must be a fully-formed URL string.
func Delete(url string, opts Options) error {
	return request("DELETE", url, opts)
}

// Put makes a PUT request against a server using the provided HTTP client.
// The url must be a fully-formed URL string.
func Put(url string, opts Options) error {
	return request("PUT", url, opts)
}

// Options describes a set of optional parameters to the various request calls.
//
// The custom client can be used for a variety of purposes beyond selecting encrypted versus unencrypted channels.
// Transports can be defined to provide augmented logging, header manipulation, et. al.
//
// If the ReqBody field is provided, it will be embedded as a JSON object.
// Otherwise, provide nil.
//
// If JSON output is to be expected from the response,
// provide either a pointer to the container structure in Results,
// or a pointer to a nil-initialized pointer variable.
// The latter method will cause the unmarshaller to allocate the container type for you.
// If no response is expected, provide a nil Results value.
//
// The MoreHeaders map, if non-nil or empty, provides a set of headers to add to those
// already present in the request.  At present, only Accepted and Content-Type are set
// by default.
//
// OkCodes provides a set of acceptable, positive responses.
//
// If provided, StatusCode specifies a pointer to an integer, which will receive the
// returned HTTP status code, successful or not.
//
// ResponseJson, if specified, provides a means for returning the raw JSON.  This is
// most useful for diagnostics.
type Options struct {
	CustomClient *http.Client
	ReqBody      interface{}
	Results      interface{}
	MoreHeaders  map[string]string
	OkCodes      []int
	StatusCode *int
	DumpReqJson  bool
	ResponseJson *[]byte
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"strconv"

	"github.com/gin-gonic/gin"
	"github.com/mdeheij/gitlegram/gitlab"
	log "github.com/mdeheij/logwrap"
)

func main() {
	c := NewConfig()
	//gin.SetMode(gin.ReleaseMode)
	r := gin.Default()

	r.POST("/", func(c *gin.Context) {
		body, ioerr := ioutil.ReadAll(c.Request.Body)
		if ioerr != nil {
			c.String(400, "Could not read request body")
			log.Critical(ioerr)
			return
		}

		//TODO: Request can be ambiguous
		request, err := gitlab.Parse(string(body))
		if err != nil {
			c.String(400, "Could not parse request body")
			log.Critical(err)
			return
		}
		c.JSON(200, getMessage(request))

	})
	address := c.Address + ":" + strconv.FormatInt(c.Port, 10)
	r.Run(address)
}
<commit_msg>Add TODO<commit_after>package main

import (
	"io/ioutil"
	"strconv"

	"github.com/gin-gonic/gin"
	"github.com/mdeheij/gitlegram/gitlab"
	log "github.com/mdeheij/logwrap"
)

func main() {
	c := NewConfig()
	//gin.SetMode(gin.ReleaseMode)
	r := gin.Default()

	r.POST("/", func(c *gin.Context) {
		body, ioerr := ioutil.ReadAll(c.Request.Body)
		if ioerr != nil {
			c.String(400, "Could not read request body")
			log.Critical(ioerr)
			return
		}

		//TODO: Request can be ambiguous
		//TODO: move this code to seperate func
		request, err := gitlab.Parse(string(body))
		if err != nil {
			c.String(400, "Could not parse request body")
			log.Critical(err)
			return
		}
		c.JSON(200, getMessage(request))

	})
	address := c.Address + ":" + strconv.FormatInt(c.Port, 10)
	r.Run(address)
}
<|endoftext|>


<commit_before>package main

import (
	docker "github.com/fsouza/go-dockerclient"
	gin "github.com/gin-gonic/gin"
)

func errorReponse(err error, c *gin.Context) {
	result := map[string]string{"error": err.Error()}
	c.JSON(400, result)
}

func HandleRun(c *gin.Context) {
	req, err := ParseRequest(c.Request)
	if err != nil {
		errorReponse(err, c)
		return
	}

	config, err := c.Get("config")
	if err != nil {
		errorReponse(err, c)
		return
	}

	client, err := c.Get("client")
	if err != nil {
		errorReponse(err, c)
		return
	}

	run := NewRun(config.(*Config), client.(*docker.Client), req)
	defer run.Destroy()

	if err := run.Setup(); err != nil {
		errorReponse(err, c)
		return
	}

	result, err := run.Start()
	if err != nil {
		errorReponse(err, c)
		return
	}

	c.JSON(200, result)
}

func RunApi(config *Config, client *docker.Client) {
	router := gin.Default()
	router.Use(func(c *gin.Context) {
		c.Set("config", config)
		c.Set("client", client)
	})

	router.POST("/run", HandleRun)
	router.Run("127.0.0.1:5000")
}
<commit_msg>Return strings instead of json in response<commit_after>package main

import (
	"strconv"

	docker "github.com/fsouza/go-dockerclient"
	gin "github.com/gin-gonic/gin"
)

func errorReponse(err error, c *gin.Context) {
	result := map[string]string{"error": err.Error()}
	c.JSON(400, result)
}

func HandleRun(c *gin.Context) {
	req, err := ParseRequest(c.Request)
	if err != nil {
		errorReponse(err, c)
		return
	}

	config, err := c.Get("config")
	if err != nil {
		errorReponse(err, c)
		return
	}

	client, err := c.Get("client")
	if err != nil {
		errorReponse(err, c)
		return
	}

	run := NewRun(config.(*Config), client.(*docker.Client), req)
	defer run.Destroy()

	if err := run.Setup(); err != nil {
		errorReponse(err, c)
		return
	}

	result, err := run.Start()
	if err != nil {
		errorReponse(err, c)
		return
	}

	c.Writer.Header().Set("X-Exit-Code", strconv.Itoa(result.ExitCode))
	c.Writer.Header().Set("X-Duration", result.Duration)

	c.String(200, result.Output)
}

func RunApi(config *Config, client *docker.Client) {
	router := gin.Default()
	router.Use(func(c *gin.Context) {
		c.Set("config", config)
		c.Set("client", client)
	})

	router.POST("/run", HandleRun)
	router.Run("127.0.0.1:5000")
}
<|endoftext|>


<commit_before>package main

import (
	"strconv"

	docker "github.com/fsouza/go-dockerclient"
	gin "github.com/gin-gonic/gin"
)

func errorReponse(err error, c *gin.Context) {
	result := map[string]string{"error": err.Error()}
	c.JSON(400, result)
}

func HandleRun(c *gin.Context) {
	req, err := ParseRequest(c.Request)
	if err != nil {
		errorReponse(err, c)
		return
	}

	config, err := c.Get("config")
	if err != nil {
		errorReponse(err, c)
		return
	}

	client, err := c.Get("client")
	if err != nil {
		errorReponse(err, c)
		return
	}

	run := NewRun(config.(*Config), client.(*docker.Client), req)
	defer run.Destroy()

	if err := run.Setup(); err != nil {
		errorReponse(err, c)
		return
	}

	result, err := run.Start()
	if err != nil {
		errorReponse(err, c)
		return
	}

	c.Writer.Header().Set("X-Exit-Code", strconv.Itoa(result.ExitCode))
	c.Writer.Header().Set("X-Duration", result.Duration)

	c.String(200, result.Output)
}

func RunApi(config *Config, client *docker.Client) {
	router := gin.Default()
	router.Use(func(c *gin.Context) {
		c.Set("config", config)
		c.Set("client", client)
	})

	router.POST("/run", HandleRun)
	router.Run("127.0.0.1:5000")
}
<commit_msg>Set CORS headers for run endpoint<commit_after>package main

import (
	"strconv"

	docker "github.com/fsouza/go-dockerclient"
	gin "github.com/gin-gonic/gin"
)

func errorReponse(err error, c *gin.Context) {
	result := map[string]string{"error": err.Error()}
	c.JSON(400, result)
}

func setCorsHeaders(c *gin.Context) {
	c.Writer.Header().Add("Access-Control-Allow-Origin", "*")
	c.Writer.Header().Add("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
}

func HandleRun(c *gin.Context) {
	// We only need CORS for this endpoint
	setCorsHeaders(c)

	req, err := ParseRequest(c.Request)
	if err != nil {
		errorReponse(err, c)
		return
	}

	config, err := c.Get("config")
	if err != nil {
		errorReponse(err, c)
		return
	}

	client, err := c.Get("client")
	if err != nil {
		errorReponse(err, c)
		return
	}

	run := NewRun(config.(*Config), client.(*docker.Client), req)
	defer run.Destroy()

	if err := run.Setup(); err != nil {
		errorReponse(err, c)
		return
	}

	result, err := run.Start()
	if err != nil {
		errorReponse(err, c)
		return
	}

	c.Writer.Header().Set("X-Exit-Code", strconv.Itoa(result.ExitCode))
	c.Writer.Header().Set("X-Duration", result.Duration)

	c.String(200, result.Output)
}

func RunApi(config *Config, client *docker.Client) {
	router := gin.Default()
	router.Use(func(c *gin.Context) {
		c.Set("config", config)
		c.Set("client", client)
	})

	router.POST("/run", HandleRun)
	router.Run("127.0.0.1:5000")
}
<|endoftext|>


<commit_before>package telebot

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
)

func sendCommand(method string, token string, params url.Values) ([]byte, error) {
	url := fmt.Sprintf("https://api.telegram.org/bot%s/%s?%s",
		token, method, params.Encode())

	resp, err := http.Get(url)
	if err != nil {
		return []byte{}, err
	}

	defer resp.Body.Close()
	json, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return []byte{}, err
	}

	return json, nil
}

func sendFile(method, token, name, path string, params url.Values) ([]byte, error) {
	file, err := os.Open(path)
	if err != nil {
		return []byte{}, err
	}
	defer file.Close()

	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	part, err := writer.CreateFormFile(name, filepath.Base(path))
	if err != nil {
		return []byte{}, err
	}

	if _, err = io.Copy(part, file); err != nil {
		return []byte{}, err
	}

	for field, values := range params {
		if len(values) > 0 {
			writer.WriteField(field, values[0])
		}
	}

	if err = writer.Close(); err != nil {
		return []byte{}, err
	}

	url := fmt.Sprintf("https://api.telegram.org/bot%s/%s", token, method)
	req, err := http.NewRequest("POST", url, body)
	if err != nil {
		return []byte{}, err
	}

	req.Header.Add("Content-Type", writer.FormDataContentType())

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return []byte{}, err
	}

	if resp.StatusCode == http.StatusInternalServerError {
		return []byte{}, fmt.Errorf("Telegram API returned 500 internal server error")
	}

	json, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return []byte{}, err
	}

	return json, nil
}

func embedSendOptions(params *url.Values, options *SendOptions) {
	if params == nil || options == nil {
		return
	}

	if options.ReplyTo.ID != 0 {
		params.Set("reply_to_message_id", strconv.Itoa(options.ReplyTo.ID))
	}

	if options.DisableWebPagePreview {
		params.Set("disable_web_page_preview", "true")
	}

	if options.ParseMode != ModeDefault {
		params.Set("parse_mode", string(options.ParseMode))
	}

	// process reply_markup
	if options.ReplyMarkup.ForceReply || options.ReplyMarkup.CustomKeyboard != nil || options.ReplyMarkup.HideCustomKeyboard {
		replyMarkup, _ := json.Marshal(options.ReplyMarkup)
		params.Set("reply_markup", string(replyMarkup))
	}
}

func getMe(token string) (User, error) {
	meJSON, err := sendCommand("getMe", token, url.Values{})
	if err != nil {
		return User{}, err
	}

	var botInfo struct {
		Ok          bool
		Result      User
		Description string
	}

	err = json.Unmarshal(meJSON, &botInfo)
	if err != nil {
		return User{}, fmt.Errorf("telebot: invalid token")
	}

	if botInfo.Ok {
		return botInfo.Result, nil
	}

	return User{}, fmt.Errorf("telebot: %s", botInfo.Description)
}

func getUpdates(token string, offset int, timeout int) (updates []Update, err error) {
	params := url.Values{}
	params.Set("offset", strconv.Itoa(offset))
	params.Set("timeout", strconv.Itoa(timeout))
	updatesJSON, err := sendCommand("getUpdates", token, params)
	if err != nil {
		return
	}

	var updatesRecieved struct {
		Ok          bool
		Result      []Update
		Description string
	}

	err = json.Unmarshal(updatesJSON, &updatesRecieved)
	if err != nil {
		return
	}

	if !updatesRecieved.Ok {
		err = fmt.Errorf("telebot: %s", updatesRecieved.Description)
		return
	}

	updates = updatesRecieved.Result
	return
}
<commit_msg>Close connection after usage.<commit_after>package telebot

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
)

func sendCommand(method string, token string, params url.Values) ([]byte, error) {
	url := fmt.Sprintf("https://api.telegram.org/bot%s/%s?%s",
		token, method, params.Encode())

	resp, err := http.Get(url)
	if err != nil {
		return []byte{}, err
	}
	resp.Close = true
	defer resp.Body.Close()
	json, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return []byte{}, err
	}

	return json, nil
}

func sendFile(method, token, name, path string, params url.Values) ([]byte, error) {
	file, err := os.Open(path)
	if err != nil {
		return []byte{}, err
	}
	defer file.Close()

	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	part, err := writer.CreateFormFile(name, filepath.Base(path))
	if err != nil {
		return []byte{}, err
	}

	if _, err = io.Copy(part, file); err != nil {
		return []byte{}, err
	}

	for field, values := range params {
		if len(values) > 0 {
			writer.WriteField(field, values[0])
		}
	}

	if err = writer.Close(); err != nil {
		return []byte{}, err
	}

	url := fmt.Sprintf("https://api.telegram.org/bot%s/%s", token, method)
	req, err := http.NewRequest("POST", url, body)
	if err != nil {
		return []byte{}, err
	}

	req.Header.Add("Content-Type", writer.FormDataContentType())

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return []byte{}, err
	}

	if resp.StatusCode == http.StatusInternalServerError {
		return []byte{}, fmt.Errorf("Telegram API returned 500 internal server error")
	}

	json, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return []byte{}, err
	}

	return json, nil
}

func embedSendOptions(params *url.Values, options *SendOptions) {
	if params == nil || options == nil {
		return
	}

	if options.ReplyTo.ID != 0 {
		params.Set("reply_to_message_id", strconv.Itoa(options.ReplyTo.ID))
	}

	if options.DisableWebPagePreview {
		params.Set("disable_web_page_preview", "true")
	}

	if options.ParseMode != ModeDefault {
		params.Set("parse_mode", string(options.ParseMode))
	}

	// process reply_markup
	if options.ReplyMarkup.ForceReply || options.ReplyMarkup.CustomKeyboard != nil || options.ReplyMarkup.HideCustomKeyboard {
		replyMarkup, _ := json.Marshal(options.ReplyMarkup)
		params.Set("reply_markup", string(replyMarkup))
	}
}

func getMe(token string) (User, error) {
	meJSON, err := sendCommand("getMe", token, url.Values{})
	if err != nil {
		return User{}, err
	}

	var botInfo struct {
		Ok          bool
		Result      User
		Description string
	}

	err = json.Unmarshal(meJSON, &botInfo)
	if err != nil {
		return User{}, fmt.Errorf("telebot: invalid token")
	}

	if botInfo.Ok {
		return botInfo.Result, nil
	}

	return User{}, fmt.Errorf("telebot: %s", botInfo.Description)
}

func getUpdates(token string, offset int, timeout int) (updates []Update, err error) {
	params := url.Values{}
	params.Set("offset", strconv.Itoa(offset))
	params.Set("timeout", strconv.Itoa(timeout))
	updatesJSON, err := sendCommand("getUpdates", token, params)
	if err != nil {
		return
	}

	var updatesRecieved struct {
		Ok          bool
		Result      []Update
		Description string
	}

	err = json.Unmarshal(updatesJSON, &updatesRecieved)
	if err != nil {
		return
	}

	if !updatesRecieved.Ok {
		err = fmt.Errorf("telebot: %s", updatesRecieved.Description)
		return
	}

	updates = updatesRecieved.Result
	return
}
<|endoftext|>


<commit_before>// Copyright 2014 The Azul3D Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"flag"
	"io"
	"log"
	"net/http"
	"net/url"
	"path"
	"strings"
	"time"

	"azul3d.org/semver.v1"
)

const (
	githubOrg = "azul3d"
	host      = "azul3d.org"
	fileHost  = "azul3d.github.io"
	certFile  = "azul3d.org.pem"
	keyFile   = "azul3d.org.key"
)

var (
	lastIdlePurge = time.Now()
	pkgHandler    = &semver.Handler{
		Matcher:  semver.MatcherFunc(compatMatcher),
		Host:     host,
		NoSecure: true,
	}
	githubMatcher = semver.GitHub(githubOrg)
)

func compatMatcher(u *url.URL) (r *semver.Repo, err error) {
	// Special case just for glfw.v3.1 -- we made a bad mistake here and this
	// is purely for backwards compatability so you can still write the (uihhhhhhhhhhgbad)
	// import:
	//
	//  "azul3d.org/native/glfw.v3.1"
	//
	// All imports like this should be updated to just ".v3" instead.
	if u.Path == "/native/glfw.v3.1" {
		u.Path = "/native/glfw.v3"
	}

	// Special case just for .dev paths -- previously we suggested importing
	// in-development packages under the .dev extension (which violates semver
	// specification):
	//
	//  "azul3d.org/tmx.dev"
	//
	// Now we redirect those to .v0 -- which has the *same effect*, but only a
	// different meaning under semver specification. In-development packages
	// today are imported under either .v0 (no stable release yet) or .vN-dev:
	//
	//  "azul3d.org/tmx.v2-dev"
	//
	if strings.HasSuffix(u.Path, ".dev") {
		u.Path = strings.TrimSuffix(u.Path, ".dev") + ".v0"
	}

	// Now let the github matcher perform the matching.
	return githubMatcher.Match(u)
}

func handler(w http.ResponseWriter, r *http.Request) {
	log.Printf("%v %v\n", r.Method, r.URL)

	// Purge idle connections.
	if time.Since(lastIdlePurge) > 2*time.Hour {
		lastIdlePurge = time.Now()
		http.DefaultTransport.(*http.Transport).CloseIdleConnections()
	}

	// Give our semver handler the ability to handle the request.
	status, err := pkgHandler.Handle(w, r)
	if err != nil {
		log.Println(err)
	}
	if status == semver.Handled {
		return
	}
	if status == semver.PkgPage {
		// Package page, redirect them to godoc.org documentation.
		tmp := *r.URL
		tmp.Scheme = "https"
		tmp.Host = "godoc.org"
		tmp.Path = path.Join(pkgHandler.Host, tmp.Path)
		http.Redirect(w, r, tmp.String(), http.StatusSeeOther)
		return
	}

	// Default to HTTPS scheme.
	if r.URL.Scheme == "" {
		r.URL.Scheme = "https"
	}

	// Just proxy the request to the file host then (it's an actual user -- not
	// the Go tool).
	r.RequestURI = ""
	delete(r.Header, "Content-Length")

	// Change Host in URL so that the request goes to the file host.
	r.URL.Host = fileHost

	// Force the Host header to be the file host (github.io uses this header).
	r.Host = fileHost

	resp, err := http.DefaultClient.Do(r)
	if err != nil {
		log.Printf("GET error: %v\n", err)
		w.WriteHeader(http.StatusNotFound)
		return
	}
	defer resp.Body.Close()

	// Copy headers over.
	hdr := w.Header()
	for k, v := range resp.Header {
		hdr[k] = v
	}

	// Peek to detect the content type.
	br := bufio.NewReaderSize(resp.Body, 1024)
	if r.Method != "HEAD" && len(resp.Header.Get("If-Modified-Since")) != 0 {
		ident, err := br.Peek(512)
		if err != nil {
			log.Printf("Proxy peek error: %v\n", err)
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		hdr["Content-Type"] = []string{http.DetectContentType(ident)}
	}

	// Write the header / status code.
	w.WriteHeader(resp.StatusCode)

	// Copy the response to the user.
	_, err = io.Copy(w, br)
	if err != nil {
		log.Printf("Proxy copy error: %v\n", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	return
}

var (
	addr    = flag.String("http", ":80", "HTTP address to serve on")
	tlsaddr = flag.String("https", ":443", "HTTPS address to serve on")
)

func main() {
	flag.Parse()
	http.HandleFunc("/", handler)

	// Start HTTPS server:
	go func() {
		if len(*tlsaddr) > 0 {
			log.Println("Serving on", *tlsaddr)
			err := http.ListenAndServeTLS(*tlsaddr, certFile, keyFile, nil)
			if err != nil {
				log.Fatal(err)
			}
		}
	}()

	// Start HTTP server:
	log.Println("Serving on", *addr)
	err := http.ListenAndServe(*addr, nil)
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Remove incorrect HTTP-only flag.<commit_after>// Copyright 2014 The Azul3D Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"flag"
	"io"
	"log"
	"net/http"
	"net/url"
	"path"
	"strings"
	"time"

	"azul3d.org/semver.v1"
)

const (
	githubOrg = "azul3d"
	host      = "azul3d.org"
	fileHost  = "azul3d.github.io"
	certFile  = "azul3d.org.pem"
	keyFile   = "azul3d.org.key"
)

var (
	lastIdlePurge = time.Now()
	pkgHandler    = &semver.Handler{
		Matcher:  semver.MatcherFunc(compatMatcher),
		Host:     host,
	}
	githubMatcher = semver.GitHub(githubOrg)
)

func compatMatcher(u *url.URL) (r *semver.Repo, err error) {
	// Special case just for glfw.v3.1 -- we made a bad mistake here and this
	// is purely for backwards compatability so you can still write the (uihhhhhhhhhhgbad)
	// import:
	//
	//  "azul3d.org/native/glfw.v3.1"
	//
	// All imports like this should be updated to just ".v3" instead.
	if u.Path == "/native/glfw.v3.1" {
		u.Path = "/native/glfw.v3"
	}

	// Special case just for .dev paths -- previously we suggested importing
	// in-development packages under the .dev extension (which violates semver
	// specification):
	//
	//  "azul3d.org/tmx.dev"
	//
	// Now we redirect those to .v0 -- which has the *same effect*, but only a
	// different meaning under semver specification. In-development packages
	// today are imported under either .v0 (no stable release yet) or .vN-dev:
	//
	//  "azul3d.org/tmx.v2-dev"
	//
	if strings.HasSuffix(u.Path, ".dev") {
		u.Path = strings.TrimSuffix(u.Path, ".dev") + ".v0"
	}

	// Now let the github matcher perform the matching.
	return githubMatcher.Match(u)
}

func handler(w http.ResponseWriter, r *http.Request) {
	log.Printf("%v %v\n", r.Method, r.URL)

	// Purge idle connections.
	if time.Since(lastIdlePurge) > 2*time.Hour {
		lastIdlePurge = time.Now()
		http.DefaultTransport.(*http.Transport).CloseIdleConnections()
	}

	// Give our semver handler the ability to handle the request.
	status, err := pkgHandler.Handle(w, r)
	if err != nil {
		log.Println(err)
	}
	if status == semver.Handled {
		return
	}
	if status == semver.PkgPage {
		// Package page, redirect them to godoc.org documentation.
		tmp := *r.URL
		tmp.Scheme = "https"
		tmp.Host = "godoc.org"
		tmp.Path = path.Join(pkgHandler.Host, tmp.Path)
		http.Redirect(w, r, tmp.String(), http.StatusSeeOther)
		return
	}

	// Default to HTTPS scheme.
	if r.URL.Scheme == "" {
		r.URL.Scheme = "https"
	}

	// Just proxy the request to the file host then (it's an actual user -- not
	// the Go tool).
	r.RequestURI = ""
	delete(r.Header, "Content-Length")

	// Change Host in URL so that the request goes to the file host.
	r.URL.Host = fileHost

	// Force the Host header to be the file host (github.io uses this header).
	r.Host = fileHost

	resp, err := http.DefaultClient.Do(r)
	if err != nil {
		log.Printf("GET error: %v\n", err)
		w.WriteHeader(http.StatusNotFound)
		return
	}
	defer resp.Body.Close()

	// Copy headers over.
	hdr := w.Header()
	for k, v := range resp.Header {
		hdr[k] = v
	}

	// Peek to detect the content type.
	br := bufio.NewReaderSize(resp.Body, 1024)
	if r.Method != "HEAD" && len(resp.Header.Get("If-Modified-Since")) != 0 {
		ident, err := br.Peek(512)
		if err != nil {
			log.Printf("Proxy peek error: %v\n", err)
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		hdr["Content-Type"] = []string{http.DetectContentType(ident)}
	}

	// Write the header / status code.
	w.WriteHeader(resp.StatusCode)

	// Copy the response to the user.
	_, err = io.Copy(w, br)
	if err != nil {
		log.Printf("Proxy copy error: %v\n", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	return
}

var (
	addr    = flag.String("http", ":80", "HTTP address to serve on")
	tlsaddr = flag.String("https", ":443", "HTTPS address to serve on")
)

func main() {
	flag.Parse()
	http.HandleFunc("/", handler)

	// Start HTTPS server:
	go func() {
		if len(*tlsaddr) > 0 {
			log.Println("Serving on", *tlsaddr)
			err := http.ListenAndServeTLS(*tlsaddr, certFile, keyFile, nil)
			if err != nil {
				log.Fatal(err)
			}
		}
	}()

	// Start HTTP server:
	log.Println("Serving on", *addr)
	err := http.ListenAndServe(*addr, nil)
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	"os"

	"github.com/Financial-Times/base-ft-rw-app-go/baseftrwapp"
	"github.com/Financial-Times/go-fthealth/v1a"
	"github.com/Financial-Times/http-handlers-go/httphandlers"
	"github.com/Financial-Times/public-concordances-api/concordances"

	"fmt"
	"strconv"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/gorilla/mux"
	"github.com/jawher/mow.cli"
	"github.com/jmcvetta/neoism"
	"github.com/rcrowley/go-metrics"
)

func main() {
	log.Infof("Application starting with args %s", os.Args)
	app := cli.App("public-concordances-api-neo4j", "A public RESTful API for accessing concordances in neo4j")
	neoURL := app.StringOpt("neo-url", "http://localhost:7474/db/data", "neo4j endpoint URL")
	//neoURL := app.StringOpt("neo-url", "http://ftper60304-law1a-eu-t:8080/db/data", "neo4j endpoint URL")
	port := app.StringOpt("port", "8080", "Port to listen on")
	env := app.StringOpt("env", "local", "environment this app is running in")
	graphiteTCPAddress := app.StringOpt("graphiteTCPAddress", "",
		"Graphite TCP address, e.g. graphite.ft.com:2003. Leave as default if you do NOT want to output to graphite (e.g. if running locally)")
	graphitePrefix := app.StringOpt("graphitePrefix", "",
		"Prefix to use. Should start with content, include the environment, and the host name. e.g. content.test.public.concordances.api.ftaps59382-law1a-eu-t")
	logMetrics := app.BoolOpt("logMetrics", false, "Whether to log metrics. Set to true if running locally and you want metrics output")
	cacheDuration := app.StringOpt("cache-duration", "1h", "Duration Get requests should be cached for. e.g. 2h45m would set the max-age value to '7440' seconds")

	app.Action = func() {
		baseftrwapp.OutputMetricsIfRequired(*graphiteTCPAddress, *graphitePrefix, *logMetrics)
		if *env != "local" {
			f, err := os.OpenFile("/var/log/apps/public-concordances-api-go-app.log", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0755)
			if err == nil {
				log.SetOutput(f)

			} else {
				log.Fatalf("Failed to initialise log file, %v", err)
			}
			defer f.Close()
		}

		log.Infof("public-concordances-api will listen on port: %s, connecting to: %s", *port, *neoURL)

		runServer(*neoURL, *port, *cacheDuration, *env)

	}
	log.SetFormatter(&log.TextFormatter{DisableColors: true})
	log.SetLevel(log.InfoLevel)
	log.Infof("Application started with args %s", os.Args)
	app.Run(os.Args)
}

func runServer(neoURL string, port string, cacheDuration string, env string) {

	if duration, durationErr := time.ParseDuration(cacheDuration); durationErr != nil {
		log.Fatalf("Failed to parse cache duration string, %v", durationErr)
	} else {
		concordances.CacheControlHeader = fmt.Sprintf("max-age=%s, public", strconv.FormatFloat(duration.Seconds(), 'f', 0, 64))
	}

	db, err := neoism.Connect(neoURL)
	db.Session.Client = &http.Client{Transport: &http.Transport{MaxIdleConnsPerHost: 100}}
	if err != nil {
		log.Fatalf("Error connecting to neo4j %s", err)
	}

	concordances.ConcordanceDriver = concordances.NewCypherDriver(db, env)

	servicesRouter := mux.NewRouter()

	// Healthchecks and standards first
	servicesRouter.HandleFunc("/__health", v1a.Handler("PublicConcordancesRead Healthchecks",
		"Checks for accessing neo4j", concordances.HealthCheck()))
	servicesRouter.HandleFunc("/ping", concordances.Ping)
	servicesRouter.HandleFunc("/__ping", concordances.Ping)
	servicesRouter.HandleFunc("/__gtg", concordances.GoodToGo)

	// Then API specific ones:
	servicesRouter.HandleFunc("/concordances", concordances.GetConcordances).Methods("GET")

	servicesRouter.HandleFunc("/concordances", concordances.MethodNotAllowedHandler)

	var monitoringRouter http.Handler = servicesRouter
	monitoringRouter = httphandlers.TransactionAwareRequestLoggingHandler(log.StandardLogger(), monitoringRouter)
	monitoringRouter = httphandlers.HTTPMetricsHandler(metrics.DefaultRegistry, monitoringRouter)

	// The top one of these feels more correct, but the lower one matches what we have in Dropwizard,
	// so it's what apps expect currently same as ping, the content of build-info needs more definition
	//using http router here to be able to catch "/"
	http.HandleFunc("/__build-info", concordances.BuildInfoHandler)
	http.HandleFunc("/build-info", concordances.BuildInfoHandler)
	http.Handle("/", monitoringRouter)

	if err := http.ListenAndServe(":"+port, nil); err != nil {
		log.Fatalf("Unable to start server: %v", err)
	}

}
<commit_msg>Added build info.<commit_after>package main

import (
	"net/http"
	"os"

	"github.com/Financial-Times/base-ft-rw-app-go/baseftrwapp"
	"github.com/Financial-Times/go-fthealth/v1a"
	"github.com/Financial-Times/http-handlers-go/httphandlers"
	"github.com/Financial-Times/public-concordances-api/concordances"
	status "github.com/Financial-Times/service-status-go/httphandlers"

	"fmt"
	"strconv"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/gorilla/mux"
	"github.com/jawher/mow.cli"
	"github.com/jmcvetta/neoism"
	"github.com/rcrowley/go-metrics"
)

func main() {
	log.Infof("Application starting with args %s", os.Args)
	app := cli.App("public-concordances-api-neo4j", "A public RESTful API for accessing concordances in neo4j")
	neoURL := app.StringOpt("neo-url", "http://localhost:7474/db/data", "neo4j endpoint URL")
	//neoURL := app.StringOpt("neo-url", "http://ftper60304-law1a-eu-t:8080/db/data", "neo4j endpoint URL")
	port := app.StringOpt("port", "8080", "Port to listen on")
	env := app.StringOpt("env", "local", "environment this app is running in")
	graphiteTCPAddress := app.StringOpt("graphiteTCPAddress", "",
		"Graphite TCP address, e.g. graphite.ft.com:2003. Leave as default if you do NOT want to output to graphite (e.g. if running locally)")
	graphitePrefix := app.StringOpt("graphitePrefix", "",
		"Prefix to use. Should start with content, include the environment, and the host name. e.g. content.test.public.concordances.api.ftaps59382-law1a-eu-t")
	logMetrics := app.BoolOpt("logMetrics", false, "Whether to log metrics. Set to true if running locally and you want metrics output")
	cacheDuration := app.StringOpt("cache-duration", "1h", "Duration Get requests should be cached for. e.g. 2h45m would set the max-age value to '7440' seconds")

	app.Action = func() {
		baseftrwapp.OutputMetricsIfRequired(*graphiteTCPAddress, *graphitePrefix, *logMetrics)
		if *env != "local" {
			f, err := os.OpenFile("/var/log/apps/public-concordances-api-go-app.log", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0755)
			if err == nil {
				log.SetOutput(f)

			} else {
				log.Fatalf("Failed to initialise log file, %v", err)
			}
			defer f.Close()
		}

		log.Infof("public-concordances-api will listen on port: %s, connecting to: %s", *port, *neoURL)

		runServer(*neoURL, *port, *cacheDuration, *env)

	}
	log.SetFormatter(&log.TextFormatter{DisableColors: true})
	log.SetLevel(log.InfoLevel)
	log.Infof("Application started with args %s", os.Args)
	app.Run(os.Args)
}

func runServer(neoURL string, port string, cacheDuration string, env string) {

	if duration, durationErr := time.ParseDuration(cacheDuration); durationErr != nil {
		log.Fatalf("Failed to parse cache duration string, %v", durationErr)
	} else {
		concordances.CacheControlHeader = fmt.Sprintf("max-age=%s, public", strconv.FormatFloat(duration.Seconds(), 'f', 0, 64))
	}

	db, err := neoism.Connect(neoURL)
	db.Session.Client = &http.Client{Transport: &http.Transport{MaxIdleConnsPerHost: 100}}
	if err != nil {
		log.Fatalf("Error connecting to neo4j %s", err)
	}

	concordances.ConcordanceDriver = concordances.NewCypherDriver(db, env)

	servicesRouter := mux.NewRouter()

	// Healthchecks and standards first
	servicesRouter.HandleFunc("/__health", v1a.Handler("PublicConcordancesRead Healthchecks",
		"Checks for accessing neo4j", concordances.HealthCheck()))

	// Then API specific ones:
	servicesRouter.HandleFunc("/concordances", concordances.GetConcordances).Methods("GET")

	servicesRouter.HandleFunc("/concordances", concordances.MethodNotAllowedHandler)

	var monitoringRouter http.Handler = servicesRouter
	monitoringRouter = httphandlers.TransactionAwareRequestLoggingHandler(log.StandardLogger(), monitoringRouter)
	monitoringRouter = httphandlers.HTTPMetricsHandler(metrics.DefaultRegistry, monitoringRouter)

	// The top one of these feels more correct, but the lower one matches what we have in Dropwizard,
	// so it's what apps expect currently same as ping, the content of build-info needs more definition
	//using http router here to be able to catch "/"
	http.HandleFunc(status.PingPath, status.PingHandler)
	http.HandleFunc(status.PingPathDW, status.PingHandler)
	http.HandleFunc(status.BuildInfoPath, status.BuildInfoHandler)
	http.HandleFunc(status.BuildInfoPathDW, status.BuildInfoHandler)
	http.HandleFunc("/__gtg", concordances.GoodToGo)
	http.Handle("/", monitoringRouter)

	if err := http.ListenAndServe(":"+port, nil); err != nil {
		log.Fatalf("Unable to start server: %v", err)
	}

}
<|endoftext|>


<commit_before>package simplyput

// TODO: PropertyList can support nested objects with named properties like "A.B.C", support nested JSON objects.
// TODO: Add rudimentary single-property queries, pagination, sorting, etc.
// TODO: Add memcache
// TODO: Support ETags, If-Modified-Since, etc. (http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html)
// TODO: PUT requests
// TODO: HEAD requests
// TODO: PATCH requests/semantics
// TODO: Batch requests (via multipart?)
// TODO: User POSTs a JSON schema, future requests are validated against that schema. Would anybody use that?

import (
	"appengine"
	"appengine/datastore"
	"appengine/urlfetch"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const (
	idKey        = "_id"
	createdKey   = "_created"
	updatedKey   = "_updated"
	defaultLimit = 10
)

func init() {
	http.HandleFunc("/datastore/v1dev/objects/", handle)
}

type userQuery struct {
	Limit, Offset int
	FilterKey, FilterType, FilterValue,
	StartCursor, EndCursor string
}

// getUserID gets the Google User ID for an access token.
func getUserID(accessToken string, client http.Client) (string, error) {
	resp, err := client.Get("https://www.googleapis.com/oauth2/v1/userinfo?access_token=" + accessToken)
	if err != nil {
		return "", err
	}
	var info struct {
		ID string
	}
	if err = json.NewDecoder(resp.Body).Decode(&info); err != nil {
		return "", err
	}
	resp.Body.Close()
	id := info.ID
	if id == "" {
		return "", errors.New("invalid auth")
	}
	return id, nil
}

// getKindAndID parses the kind and ID from a request path.
func getKindAndID(path string) (string, int64, error) {
	if match, err := regexp.MatchString("/datastore/v1dev/objects/[a-zA-Z]+/[0-9]+", path); err != nil {
		return "", int64(0), err
	} else if match {
		kind := path[len("/datastore/v1dev/objects/"):strings.LastIndex(path, "/")]
		idStr := path[strings.LastIndex(path, "/")+1:]
		id, err := strconv.ParseInt(idStr, 10, 64)
		if err != nil {
			return "", int64(0), err
		}
		return kind, id, nil
	}
	if match, err := regexp.MatchString("/datastore/v1dev/objects/[a-zA-Z]+", path); err != nil {
		return "", int64(0), err
	} else if match {
		kind := path[len("/datastore/v1dev/objects/"):]
		return kind, int64(0), nil
	}
	return "", int64(0), errors.New("invalid path")
}

// handle dispatches requests to the relevant API method and arranges certain common state
func handle(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	w.Header().Add("Access-Control-Allow-Origin", "*")

	r.ParseForm()
	client := urlfetch.Client(c)

	// Get the access_token from the request and turn it into a user ID with which we will namespace Kinds in the datastore.
	accessToken := r.Form.Get("access_token")
	if accessToken == "" {
		h := r.Header.Get("Authorization")
		if strings.HasPrefix(h, "Bearer ") {
			accessToken = h[len("Bearer "):]
		}
	}
	if accessToken == "" {
		http.Error(w, "Unauthorized", http.StatusUnauthorized)
		return
	}
	userID, err := getUserID(accessToken, *client)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	kind, id, err := getKindAndID(r.URL.Path)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	dsKind := fmt.Sprintf("%s--%s", userID, kind)

	resp := make(map[string]interface{}, 0)
	errCode := http.StatusOK
	if id == int64(0) {
		switch r.Method {
		case "POST":
			resp, errCode = insert(c, dsKind, r.Body)
			r.Body.Close()
		case "GET":
			resp, errCode = list(c, dsKind, newUserQuery(r))
		default:
			http.Error(w, "Unsupported Method", http.StatusMethodNotAllowed)
			return
		}
	} else {
		switch r.Method {
		case "GET":
			resp, errCode = get(c, dsKind, id)
		case "DELETE":
			errCode = delete(c, dsKind, id)
		case "POST":
			// This is strictly "replace all properties/values", not "add new properties, update existing"
			resp, errCode = update(c, dsKind, id, r.Body)
			r.Body.Close()
		default:
			http.Error(w, "Unsupported Method", http.StatusMethodNotAllowed)
			return
		}
	}
	if errCode != http.StatusOK {
		http.Error(w, "", errCode)
		return
	}
	if err := json.NewEncoder(w).Encode(&resp); err != nil {
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}
	w.Header().Add("Content-Type", "application/json")
}

func newUserQuery(r *http.Request) userQuery {
	uq := userQuery{
		StartCursor: r.FormValue("start"),
		EndCursor:   r.FormValue("end"),
	}
	// TODO: MustParse for limit/offset, else panic
	uq.Limit, _ = strconv.Atoi(r.FormValue("limit"))
	uq.Offset, _ = strconv.Atoi(r.FormValue("offset"))

	// TODO: Support ?where=foo<bar queries (which may or may not be annoying to scope for users...)
	_ = r.FormValue("where")
	return uq
}

func delete(c appengine.Context, kind string, id int64) int {
	k := datastore.NewKey(c, kind, "", id, nil)
	if err := datastore.Delete(c, k); err != nil {
		if err == datastore.ErrNoSuchEntity {
			return http.StatusNotFound
		} else {
			c.Errorf("%v", err)
			return http.StatusInternalServerError
		}
	}
	return http.StatusOK
}

func get(c appengine.Context, kind string, id int64) (map[string]interface{}, int) {
	k := datastore.NewKey(c, kind, "", id, nil)
	var pl plist
	if err := datastore.Get(c, k, &pl); err != nil {
		if err == datastore.ErrNoSuchEntity {
			return nil, http.StatusNotFound
		}
		c.Errorf("%v", err)
		return nil, http.StatusInternalServerError
	}
	m := plistToMap(pl, k.IntID())
	m[idKey] = k.IntID()
	return m, http.StatusOK
}

func insert(c appengine.Context, kind string, r io.Reader) (map[string]interface{}, int) {
	var m map[string]interface{}
	if err := json.NewDecoder(r).Decode(&m); err != nil {
		c.Errorf("%v", err)
		return nil, http.StatusInternalServerError
	}
	m[createdKey] = time.Now().Unix()

	pl := mapToPlist(m)

	k := datastore.NewIncompleteKey(c, kind, nil)
	k, err := datastore.Put(c, k, &pl)
	if err != nil {
		c.Errorf("%v", err)
		return nil, http.StatusInternalServerError
	}
	m[idKey] = k.IntID()
	return m, http.StatusOK
}

type prop struct {
	Name string
	Value interface{}
	Multiple bool
	NoIndex bool
}
type plist []prop

// plistToMap transforms a plist such as you would get from the datastore into a map[string]interface{} suitable for JSON-encoding.
func plistToMap(pl plist, id int64) map[string]interface{} {
	m := make(map[string]interface{})
	for _, p := range pl {
		if _, exists := m[p.Name]; exists {
			if _, isArr := m[p.Name].([]interface{}); isArr {
				m[p.Name] = append(m[p.Name].([]interface{}), p.Value)
			} else {
				m[p.Name] = []interface{}{m[p.Name], p.Value}
			}
		} else {
			m[p.Name] = p.Value
		}
	}
	m[idKey] = id
	return m
}

// mapToPlist transforms a map[string]interface{} such as you would get from decoding JSON into a plist to store in the datastore.
func mapToPlist(m map[string]interface{}) plist {
	pl := make(plist, 0, len(m))
	for k, v := range m {
		if _, mult := v.([]interface{}); mult {
			for _, mv := range v.([]interface{}) {
				pl = append(pl, prop{
					Name:     k,
					Value:    mv,
					Multiple: true,
				})
			}
		} else {
			pl = append(pl, prop{
				Name:  k,
				Value: v,
			})
		}
	}
	return pl
}

func list(c appengine.Context, kind string, uq userQuery) (map[string]interface{}, int) {
	q := datastore.NewQuery(kind)

	if uq.Limit != 0 {
		q = q.Limit(uq.Limit)
	}
	if c, err := datastore.DecodeCursor(uq.StartCursor); err == nil {
		q.Start(c)
	}
	if c, err := datastore.DecodeCursor(uq.EndCursor); err == nil {
		q.End(c)
	}

	items := make([]map[string]interface{}, 0)

	var crs datastore.Cursor
	for t := q.Run(c); ; {
		var pl plist
		k, err := t.Next(&pl)
		if err == datastore.Done {
			break
		}
		if err != nil {
			c.Errorf("%v", err)
			return nil, http.StatusInternalServerError
		}
		m := plistToMap(pl, k.IntID())
		items = append(items, m)
		if crs, err = t.Cursor(); err != nil {
			c.Errorf("%v", err)
			return nil, http.StatusInternalServerError
		}
	}
	r := map[string]interface{}{
		"items":          items,
		"nextStartToken": crs.String(),
	}
	return r, http.StatusOK
}

func update(c appengine.Context, kind string, id int64, r io.Reader) (map[string]interface{}, int) {
	var m map[string]interface{}
	if err := json.NewDecoder(r).Decode(&m); err != nil {
		c.Errorf("%v", err)
		return nil, http.StatusInternalServerError
	}
	m[updatedKey] = time.Now().Unix()

	pl := mapToPlist(m)

	k := datastore.NewKey(c, kind, "", id, nil)
	if _, err := datastore.Put(c, k, &pl); err != nil {
		c.Errorf("%v", err)
		return nil, http.StatusInternalServerError
	}
	m[idKey] = id
	return m, http.StatusOK
}
<commit_msg>Factor out now for overriding in tests<commit_after>package simplyput

// TODO: PropertyList can support nested objects with named properties like "A.B.C", support nested JSON objects.
// TODO: Add rudimentary single-property queries, pagination, sorting, etc.
// TODO: Add memcache
// TODO: Support ETags, If-Modified-Since, etc. (http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html)
// TODO: PUT requests
// TODO: HEAD requests
// TODO: PATCH requests/semantics
// TODO: Batch requests (via multipart?)
// TODO: User POSTs a JSON schema, future requests are validated against that schema. Would anybody use that?

import (
	"appengine"
	"appengine/datastore"
	"appengine/urlfetch"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const (
	idKey        = "_id"
	createdKey   = "_created"
	updatedKey   = "_updated"
	defaultLimit = 10
)

// To override for testing
var now = time.Now

func init() {
	http.HandleFunc("/datastore/v1dev/objects/", handle)
}

type userQuery struct {
	Limit, Offset int
	FilterKey, FilterType, FilterValue,
	StartCursor, EndCursor string
}

// getUserID gets the Google User ID for an access token.
func getUserID(accessToken string, client http.Client) (string, error) {
	resp, err := client.Get("https://www.googleapis.com/oauth2/v1/userinfo?access_token=" + accessToken)
	if err != nil {
		return "", err
	}
	var info struct {
		ID string
	}
	if err = json.NewDecoder(resp.Body).Decode(&info); err != nil {
		return "", err
	}
	resp.Body.Close()
	id := info.ID
	if id == "" {
		return "", errors.New("invalid auth")
	}
	return id, nil
}

// getKindAndID parses the kind and ID from a request path.
func getKindAndID(path string) (string, int64, error) {
	if match, err := regexp.MatchString("/datastore/v1dev/objects/[a-zA-Z]+/[0-9]+", path); err != nil {
		return "", int64(0), err
	} else if match {
		kind := path[len("/datastore/v1dev/objects/"):strings.LastIndex(path, "/")]
		idStr := path[strings.LastIndex(path, "/")+1:]
		id, err := strconv.ParseInt(idStr, 10, 64)
		if err != nil {
			return "", int64(0), err
		}
		return kind, id, nil
	}
	if match, err := regexp.MatchString("/datastore/v1dev/objects/[a-zA-Z]+", path); err != nil {
		return "", int64(0), err
	} else if match {
		kind := path[len("/datastore/v1dev/objects/"):]
		return kind, int64(0), nil
	}
	return "", int64(0), errors.New("invalid path")
}

// handle dispatches requests to the relevant API method and arranges certain common state
func handle(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	w.Header().Add("Access-Control-Allow-Origin", "*")

	r.ParseForm()
	client := urlfetch.Client(c)

	// Get the access_token from the request and turn it into a user ID with which we will namespace Kinds in the datastore.
	accessToken := r.Form.Get("access_token")
	if accessToken == "" {
		h := r.Header.Get("Authorization")
		if strings.HasPrefix(h, "Bearer ") {
			accessToken = h[len("Bearer "):]
		}
	}
	if accessToken == "" {
		http.Error(w, "Unauthorized", http.StatusUnauthorized)
		return
	}
	userID, err := getUserID(accessToken, *client)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	kind, id, err := getKindAndID(r.URL.Path)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	dsKind := fmt.Sprintf("%s--%s", userID, kind)

	resp := make(map[string]interface{}, 0)
	errCode := http.StatusOK
	if id == int64(0) {
		switch r.Method {
		case "POST":
			resp, errCode = insert(c, dsKind, r.Body)
			r.Body.Close()
		case "GET":
			resp, errCode = list(c, dsKind, newUserQuery(r))
		default:
			http.Error(w, "Unsupported Method", http.StatusMethodNotAllowed)
			return
		}
	} else {
		switch r.Method {
		case "GET":
			resp, errCode = get(c, dsKind, id)
		case "DELETE":
			errCode = delete(c, dsKind, id)
		case "POST":
			// This is strictly "replace all properties/values", not "add new properties, update existing"
			resp, errCode = update(c, dsKind, id, r.Body)
			r.Body.Close()
		default:
			http.Error(w, "Unsupported Method", http.StatusMethodNotAllowed)
			return
		}
	}
	if errCode != http.StatusOK {
		http.Error(w, "", errCode)
		return
	}
	if err := json.NewEncoder(w).Encode(&resp); err != nil {
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}
	w.Header().Add("Content-Type", "application/json")
}

func newUserQuery(r *http.Request) userQuery {
	uq := userQuery{
		StartCursor: r.FormValue("start"),
		EndCursor:   r.FormValue("end"),
	}
	// TODO: MustParse for limit/offset, else panic
	uq.Limit, _ = strconv.Atoi(r.FormValue("limit"))
	uq.Offset, _ = strconv.Atoi(r.FormValue("offset"))

	// TODO: Support ?where=foo<bar queries (which may or may not be annoying to scope for users...)
	_ = r.FormValue("where")
	return uq
}

func delete(c appengine.Context, kind string, id int64) int {
	k := datastore.NewKey(c, kind, "", id, nil)
	if err := datastore.Delete(c, k); err != nil {
		if err == datastore.ErrNoSuchEntity {
			return http.StatusNotFound
		} else {
			c.Errorf("%v", err)
			return http.StatusInternalServerError
		}
	}
	return http.StatusOK
}

func get(c appengine.Context, kind string, id int64) (map[string]interface{}, int) {
	k := datastore.NewKey(c, kind, "", id, nil)
	var pl plist
	if err := datastore.Get(c, k, &pl); err != nil {
		if err == datastore.ErrNoSuchEntity {
			return nil, http.StatusNotFound
		}
		c.Errorf("%v", err)
		return nil, http.StatusInternalServerError
	}
	m := plistToMap(pl, k.IntID())
	m[idKey] = k.IntID()
	return m, http.StatusOK
}

func insert(c appengine.Context, kind string, r io.Reader) (map[string]interface{}, int) {
	var m map[string]interface{}
	if err := json.NewDecoder(r).Decode(&m); err != nil {
		c.Errorf("%v", err)
		return nil, http.StatusInternalServerError
	}
	m[createdKey] = now.Unix()

	pl := mapToPlist(m)

	k := datastore.NewIncompleteKey(c, kind, nil)
	k, err := datastore.Put(c, k, &pl)
	if err != nil {
		c.Errorf("%v", err)
		return nil, http.StatusInternalServerError
	}
	m[idKey] = k.IntID()
	return m, http.StatusOK
}

type prop struct {
	Name string
	Value interface{}
	Multiple bool
	NoIndex bool
}
type plist []prop

// plistToMap transforms a plist such as you would get from the datastore into a map[string]interface{} suitable for JSON-encoding.
func plistToMap(pl plist, id int64) map[string]interface{} {
	m := make(map[string]interface{})
	for _, p := range pl {
		if _, exists := m[p.Name]; exists {
			if _, isArr := m[p.Name].([]interface{}); isArr {
				m[p.Name] = append(m[p.Name].([]interface{}), p.Value)
			} else {
				m[p.Name] = []interface{}{m[p.Name], p.Value}
			}
		} else {
			m[p.Name] = p.Value
		}
	}
	m[idKey] = id
	return m
}

// mapToPlist transforms a map[string]interface{} such as you would get from decoding JSON into a plist to store in the datastore.
func mapToPlist(m map[string]interface{}) plist {
	pl := make(plist, 0, len(m))
	for k, v := range m {
		if _, mult := v.([]interface{}); mult {
			for _, mv := range v.([]interface{}) {
				pl = append(pl, prop{
					Name:     k,
					Value:    mv,
					Multiple: true,
				})
			}
		} else {
			pl = append(pl, prop{
				Name:  k,
				Value: v,
			})
		}
	}
	return pl
}

func list(c appengine.Context, kind string, uq userQuery) (map[string]interface{}, int) {
	q := datastore.NewQuery(kind)

	if uq.Limit != 0 {
		q = q.Limit(uq.Limit)
	}
	if c, err := datastore.DecodeCursor(uq.StartCursor); err == nil {
		q.Start(c)
	}
	if c, err := datastore.DecodeCursor(uq.EndCursor); err == nil {
		q.End(c)
	}

	items := make([]map[string]interface{}, 0)

	var crs datastore.Cursor
	for t := q.Run(c); ; {
		var pl plist
		k, err := t.Next(&pl)
		if err == datastore.Done {
			break
		}
		if err != nil {
			c.Errorf("%v", err)
			return nil, http.StatusInternalServerError
		}
		m := plistToMap(pl, k.IntID())
		items = append(items, m)
		if crs, err = t.Cursor(); err != nil {
			c.Errorf("%v", err)
			return nil, http.StatusInternalServerError
		}
	}
	r := map[string]interface{}{
		"items":          items,
		"nextStartToken": crs.String(),
	}
	return r, http.StatusOK
}

func update(c appengine.Context, kind string, id int64, r io.Reader) (map[string]interface{}, int) {
	var m map[string]interface{}
	if err := json.NewDecoder(r).Decode(&m); err != nil {
		c.Errorf("%v", err)
		return nil, http.StatusInternalServerError
	}
	m[updatedKey] = now.Unix()

	pl := mapToPlist(m)

	k := datastore.NewKey(c, kind, "", id, nil)
	if _, err := datastore.Put(c, k, &pl); err != nil {
		c.Errorf("%v", err)
		return nil, http.StatusInternalServerError
	}
	m[idKey] = id
	return m, http.StatusOK
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/maniksurtani/protodocparser/impl"
	"encoding/json"
	"io/ioutil"
	"strings"
	"regexp"
)

func main() {
	// Read a proto file from StdIn
	protoContents := readFromStdIn()
	j := parse(protoContents)
	fmt.Println(j)
}

func readFromStdIn() []string {
	//todo read from stdin and convert this into a proper test
	dat, err := ioutil.ReadFile("./sample.proto")
	check(err)
	return strings.Split(string(dat), "\n")
}

func isStartComment(line string) bool {
	return checkRegex("\\s*/\\*\\*", line)
}

func isEndComment(line string) bool {
	return checkRegex("\\s*\\*\\\\\\s*$", line)
}

func isRpc(line string) bool {
	return checkRegex("\\s*RPC\\s+", line)
}

func isService(line string) bool {
	return checkRegex("\\s*Service\\s+", line)
}

func addRpcToLastService(services []*impl.Service, commentBlock *impl.CommentBlock, lines []string) {
	// TODO
}

func addServiceToServices(services []*impl.Service, commentBlock *impl.CommentBlock, lines []string) []*impl.Service {
	// TODO
	//return &impl.Service{}
	return nil;
}

func parse(lines []string) string {
	fmt.Printf("Got %v lines of txt\n", len(lines))
	// Create an array of services.
	services := make([]*impl.Service, 0)

	var currentBlock *impl.CommentBlock

	for ln, line := range lines {
		if isStartComment(line) && currentBlock == nil {
			// Create a new comment block.
			currentBlock = &impl.CommentBlock{}
			currentBlock.Start = ln
			currentBlock.Type = impl.OtherComment
		} else if isEndComment(line) && currentBlock != nil {
			currentBlock.End = ln
		} else if isRpc(line) && currentBlock != nil && currentBlock.End > 0 {
			// Mark block as an RPC type.
			currentBlock.Type = impl.RpcComment
			addRpcToLastService(services, currentBlock, lines)
			currentBlock = nil
		} else if isService(line) && currentBlock != nil && currentBlock.End > 0 {
			// Mark block as a Service type.
			currentBlock.Type = impl.ServiceComment
			services = addServiceToServices(services, currentBlock, lines)
			currentBlock = nil
		}
	}

	bytes, err := json.Marshal(services)
	if err != nil {
		panic(fmt.Sprintf("Caught error %v trying to serialize %v into JSON.", err, services))
	}
	return string(bytes)
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func checkRegex(regex string, s string) bool {
	out, err := regexp.MatchString(regex, s)
	if err != nil {
		return false
	}
	return out
}<commit_msg>Read from stdin<commit_after>package main

import (
	"fmt"
	"github.com/maniksurtani/protodocparser/impl"
	"encoding/json"
	"regexp"
	"bufio"
	"os"
)

func main() {
	// Read a proto file from StdIn
	protoContents := readFromStdIn()
	j := parse(protoContents)
	fmt.Println(j)
}

func readFromStdIn() []string {
	s := bufio.NewScanner(os.Stdin)
	txt := make([]string, 0)
	for s.Scan() {
		txt = append(txt, s.Text())
	}
	return txt
}

func isStartComment(line string) bool {
	return checkRegex("\\s*/\\*\\*", line)
}

func isEndComment(line string) bool {
	return checkRegex("\\s*\\*\\\\\\s*$", line)
}

func isRpc(line string) bool {
	return checkRegex("\\s*RPC\\s+", line)
}

func isService(line string) bool {
	return checkRegex("\\s*Service\\s+", line)
}

func addRpcToLastService(services []*impl.Service, commentBlock *impl.CommentBlock, lines []string) {
	// TODO
}

func addServiceToServices(services []*impl.Service, commentBlock *impl.CommentBlock, lines []string) []*impl.Service {
	// TODO
	//return &impl.Service{}
	return nil;
}

func parse(lines []string) string {
	fmt.Printf("Got %v lines of txt\n", len(lines))
	// Create an array of services.
	services := make([]*impl.Service, 0)

	var currentBlock *impl.CommentBlock

	for ln, line := range lines {
		if isStartComment(line) && currentBlock == nil {
			// Create a new comment block.
			currentBlock = &impl.CommentBlock{}
			currentBlock.Start = ln
			currentBlock.Type = impl.OtherComment
		} else if isEndComment(line) && currentBlock != nil {
			currentBlock.End = ln
		} else if isRpc(line) && currentBlock != nil && currentBlock.End > 0 {
			// Mark block as an RPC type.
			currentBlock.Type = impl.RpcComment
			addRpcToLastService(services, currentBlock, lines)
			currentBlock = nil
		} else if isService(line) && currentBlock != nil && currentBlock.End > 0 {
			// Mark block as a Service type.
			currentBlock.Type = impl.ServiceComment
			services = addServiceToServices(services, currentBlock, lines)
			currentBlock = nil
		}
	}

	bytes, err := json.Marshal(services)
	if err != nil {
		panic(fmt.Sprintf("Caught error %v trying to serialize %v into JSON.", err, services))
	}
	return string(bytes)
}

func checkRegex(regex string, s string) bool {
	out, err := regexp.MatchString(regex, s)
	if err != nil {
		return false
	}
	return out
}<|endoftext|>