<commit_before>package ipfix

import (
	"bufio"
	"encoding/binary"
	"errors"
	"io"
	"runtime"
)

// The version field in IPFIX messages should always have the value 10. If it
// does not, you get this error. It's probably a sign of a bug in the parser or
// the exporter and that we have lost synchronization with the data stream.
// Reestablishing the session is the only way forward at this point.
var ErrVersion = errors.New("incorrect version field in message header - out of sync?")

// A Message is the top level construct representing an IPFIX message. A well
// formed message contains one or more sets of data or template information.
type Message struct {
	Header       MessageHeader
	DataSets     []DataSet
	TemplateSets []TemplateSet
}

// The MessageHeader provides metadata for the entire Message. The sequence
// number and domain ID can be used to gain knowledge of messages lost on an
// unreliable transport such as UDP.
type MessageHeader struct {
	Version        uint16 // Always 0x0a
	Length         uint16
	ExportTime     uint32 // Epoch seconds
	SequenceNumber uint32
	DomainId       uint32
}

type setHeader struct {
	SetId  uint16
	Length uint16
}

type templateHeader struct {
	TemplateId uint16
	FieldCount uint16
}

// The DataSet represents a single exported flow. The Records each describe
// different aspects of the flow (source and destination address, counters,
// service, etc.).
type DataSet struct {
	TemplateId uint16
	Records    [][]byte
}

// The TemplateSet describes a data template, as used by DataSets.
type TemplateSet struct {
	TemplateId uint16
	Records    []TemplateRecord
}

// The TemplateRecord describes the ID and size of the corresponding Records in a DataSet.
type TemplateRecord struct {
	EnterpriseId uint32
	FieldId      uint16
	Length       uint16
}

// The Session is the context for IPFIX messages.
type Session struct {
	templates  [][]TemplateRecord
	reader     *bufio.Reader
	minRecord  []uint16
	dictionary dictionary
}

// NewSession initializes a new Session based on the provided io.Reader.
func NewSession(reader io.Reader) *Session {
	s := Session{}
	s.templates = make([][]TemplateRecord, 65536)
	s.reader = bufio.NewReaderSize(reader, 32768)
	s.minRecord = make([]uint16, 65536)
	s.dictionary = builtinDictionary
	return &s
}

// ReadMessage extracts and returns one message from the IPFIX stream. As long
// as err is nil, further messages can be read from the stream. Errors are not
// recoverable -- once an error has been returned, ReadMessage should not be
// called again on the same session.
func (s *Session) ReadMessage() (msg *Message, err error) {
	defer func() {
		if r := recover(); r != nil {
			if _, ok := r.(runtime.Error); ok {
				panic(r)
			}

			msg = nil
			err = r.(error)
		}
	}()

	msg = &Message{}
	msg.DataSets = make([]DataSet, 0)
	msg.TemplateSets = make([]TemplateSet, 0)

	msgHdr := MessageHeader{}
	err = binary.Read(s.reader, binary.BigEndian, &msgHdr)
	s.errorIf(err)
	if msgHdr.Version != 10 {
		s.errorIf(ErrVersion)
	}
	read := binary.Size(msgHdr)
	msg.Header = msgHdr

	for read < int(msgHdr.Length) {
		tsets, dsets, tr := s.readSet()
		read += tr
		msg.TemplateSets = append(msg.TemplateSets, tsets...)
		msg.DataSets = append(msg.DataSets, dsets...)
	}
	return
}

func (s *Session) errorIf(err error) {
	if err != nil {
		panic(err)
	}
}

func (s *Session) readSet() (tsets []TemplateSet, dsets []DataSet, read int) {
	tsets = make([]TemplateSet, 0)
	dsets = make([]DataSet, 0)

	setHdr := setHeader{}
	err := binary.Read(s.reader, binary.BigEndian, &setHdr)
	s.errorIf(err)
	read += binary.Size(setHdr)

	end := int(setHdr.Length)
	for read < end {
		if end-read < int(s.minRecord[setHdr.SetId]) {
			// Padding
			_, err = io.ReadFull(s.reader, make([]byte, end-read))
			s.errorIf(err)
			read += end - read
		} else if setHdr.SetId == 2 {
			// Template Set
			var ts *TemplateSet
			var tsRead int
			ts, tsRead = s.readTemplateSet()
			read += tsRead
			tsets = append(tsets, *ts)

			tid := ts.TemplateId
			s.templates[tid] = ts.Records

			var minLength uint16
			for i := range ts.Records {
				if ts.Records[i].Length == 65535 {
					minLength += 1
				} else {
					minLength += ts.Records[i].Length
				}
				s.minRecord[i] = minLength
			}
		} else if setHdr.SetId == 3 {
			// Options Template Set, not handled
			_, err = io.ReadFull(s.reader, make([]byte, end-read))
			s.errorIf(err)
			read += end - read
		} else {
			// Data Set
			ds := DataSet{}
			ds.TemplateId = setHdr.SetId

			if tpl := s.templates[setHdr.SetId]; tpl != nil {
				ds.Records = make([][]byte, len(tpl))
				for i := range tpl {
					var bs []byte
					if tpl[i].Length == 65535 {
						var bsRead int
						bs, bsRead = s.readVariableLength()
						read += bsRead
					} else {
						bs = make([]byte, tpl[i].Length)
						_, err = io.ReadFull(s.reader, bs)
						s.errorIf(err)
						read += len(bs)
					}
					ds.Records[i] = bs
				}
			} else {
				// Data set with unknown template
				_, err = io.ReadFull(s.reader, make([]byte, end-read))
				s.errorIf(err)
				read += end - read
			}

			dsets = append(dsets, ds)
		}
	}
	return
}

func (s *Session) readTemplateSet() (ts *TemplateSet, read int) {
	ts = &TemplateSet{}
	th := templateHeader{}
	err := binary.Read(s.reader, binary.BigEndian, &th)
	s.errorIf(err)
	read = binary.Size(th)

	ts.TemplateId = th.TemplateId
	ts.Records = make([]TemplateRecord, th.FieldCount)
	for i := 0; i < int(th.FieldCount); i++ {
		f := TemplateRecord{}
		err = binary.Read(s.reader, binary.BigEndian, &f.FieldId)
		s.errorIf(err)
		read += binary.Size(f.FieldId)
		err = binary.Read(s.reader, binary.BigEndian, &f.Length)
		s.errorIf(err)
		read += binary.Size(f.Length)
		if f.FieldId >= 0x8000 {
			f.FieldId -= 0x8000
			err = binary.Read(s.reader, binary.BigEndian, &f.EnterpriseId)
			s.errorIf(err)
			read += binary.Size(f.EnterpriseId)
		}
		ts.Records[i] = f
	}

	return
}

// Reads a variable length information element. Returns a slice with the data
// in it, the total number of bytes read.
func (s *Session) readVariableLength() (bs []byte, r int) {
	var l int

	var l0 uint8
	err := binary.Read(s.reader, binary.BigEndian, &l0)
	s.errorIf(err)
	r += 1

	if l0 < 255 {
		l = int(l0)
	} else {
		var l1 uint16
		err = binary.Read(s.reader, binary.BigEndian, &l1)
		s.errorIf(err)
		r += 2
		l = int(l1)
	}

	bs = make([]byte, l)
	_, err = io.ReadFull(s.reader, bs)
	s.errorIf(err)
	r += len(bs)

	return
}
<commit_msg>Precompute sizes<commit_after>package ipfix

import (
	"bufio"
	"encoding/binary"
	"errors"
	"io"
	"runtime"
)

// The version field in IPFIX messages should always have the value 10. If it
// does not, you get this error. It's probably a sign of a bug in the parser or
// the exporter and that we have lost synchronization with the data stream.
// Reestablishing the session is the only way forward at this point.
var ErrVersion = errors.New("incorrect version field in message header - out of sync?")

// A Message is the top level construct representing an IPFIX message. A well
// formed message contains one or more sets of data or template information.
type Message struct {
	Header       MessageHeader
	DataSets     []DataSet
	TemplateSets []TemplateSet
}

// The MessageHeader provides metadata for the entire Message. The sequence
// number and domain ID can be used to gain knowledge of messages lost on an
// unreliable transport such as UDP.
type MessageHeader struct {
	Version        uint16 // Always 0x0a
	Length         uint16
	ExportTime     uint32 // Epoch seconds
	SequenceNumber uint32
	DomainId       uint32
}

type setHeader struct {
	SetId  uint16
	Length uint16
}

type templateHeader struct {
	TemplateId uint16
	FieldCount uint16
}

// The DataSet represents a single exported flow. The Records each describe
// different aspects of the flow (source and destination address, counters,
// service, etc.).
type DataSet struct {
	TemplateId uint16
	Records    [][]byte
}

// The TemplateSet describes a data template, as used by DataSets.
type TemplateSet struct {
	TemplateId uint16
	Records    []TemplateRecord
}

// The TemplateRecord describes the ID and size of the corresponding Records in a DataSet.
type TemplateRecord struct {
	EnterpriseId uint32
	FieldId      uint16
	Length       uint16
}

// The Session is the context for IPFIX messages.
type Session struct {
	templates  [][]TemplateRecord
	reader     *bufio.Reader
	minRecord  []uint16
	dictionary dictionary
}

// NewSession initializes a new Session based on the provided io.Reader.
func NewSession(reader io.Reader) *Session {
	s := Session{}
	s.templates = make([][]TemplateRecord, 65536)
	s.reader = bufio.NewReaderSize(reader, 32768)
	s.minRecord = make([]uint16, 65536)
	s.dictionary = builtinDictionary
	return &s
}

var msgHeaderLength = binary.Size(MessageHeader{})
var setHeaderLength = binary.Size(setHeader{})
var templateHeaderLength = binary.Size(templateHeader{})

// ReadMessage extracts and returns one message from the IPFIX stream. As long
// as err is nil, further messages can be read from the stream. Errors are not
// recoverable -- once an error has been returned, ReadMessage should not be
// called again on the same session.
func (s *Session) ReadMessage() (msg *Message, err error) {
	defer func() {
		if r := recover(); r != nil {
			if _, ok := r.(runtime.Error); ok {
				panic(r)
			}

			msg = nil
			err = r.(error)
		}
	}()

	msg = &Message{}
	msg.DataSets = make([]DataSet, 0)
	msg.TemplateSets = make([]TemplateSet, 0)

	msgHdr := MessageHeader{}
	err = binary.Read(s.reader, binary.BigEndian, &msgHdr)
	s.errorIf(err)
	if msgHdr.Version != 10 {
		s.errorIf(ErrVersion)
	}
	read := msgHeaderLength
	msg.Header = msgHdr

	for read < int(msgHdr.Length) {
		tsets, dsets, tr := s.readSet()
		read += tr
		msg.TemplateSets = append(msg.TemplateSets, tsets...)
		msg.DataSets = append(msg.DataSets, dsets...)
	}
	return
}

func (s *Session) errorIf(err error) {
	if err != nil {
		panic(err)
	}
}

func (s *Session) readSet() (tsets []TemplateSet, dsets []DataSet, read int) {
	tsets = make([]TemplateSet, 0)
	dsets = make([]DataSet, 0)

	setHdr := setHeader{}
	err := binary.Read(s.reader, binary.BigEndian, &setHdr)
	s.errorIf(err)
	read += setHeaderLength

	end := int(setHdr.Length)
	for read < end {
		if end-read < int(s.minRecord[setHdr.SetId]) {
			// Padding
			_, err = io.ReadFull(s.reader, make([]byte, end-read))
			s.errorIf(err)
			read += end - read
		} else if setHdr.SetId == 2 {
			// Template Set
			var ts *TemplateSet
			var tsRead int
			ts, tsRead = s.readTemplateSet()
			read += tsRead
			tsets = append(tsets, *ts)

			tid := ts.TemplateId
			s.templates[tid] = ts.Records

			var minLength uint16
			for i := range ts.Records {
				if ts.Records[i].Length == 65535 {
					minLength += 1
				} else {
					minLength += ts.Records[i].Length
				}
				s.minRecord[i] = minLength
			}
		} else if setHdr.SetId == 3 {
			// Options Template Set, not handled
			_, err = io.ReadFull(s.reader, make([]byte, end-read))
			s.errorIf(err)
			read += end - read
		} else {
			// Data Set
			ds := DataSet{}
			ds.TemplateId = setHdr.SetId

			if tpl := s.templates[setHdr.SetId]; tpl != nil {
				ds.Records = make([][]byte, len(tpl))
				for i := range tpl {
					var bs []byte
					if tpl[i].Length == 65535 {
						var bsRead int
						bs, bsRead = s.readVariableLength()
						read += bsRead
					} else {
						bs = make([]byte, tpl[i].Length)
						_, err = io.ReadFull(s.reader, bs)
						s.errorIf(err)
						read += len(bs)
					}
					ds.Records[i] = bs
				}
			} else {
				// Data set with unknown template
				_, err = io.ReadFull(s.reader, make([]byte, end-read))
				s.errorIf(err)
				read += end - read
			}

			dsets = append(dsets, ds)
		}
	}
	return
}

func (s *Session) readTemplateSet() (ts *TemplateSet, read int) {
	ts = &TemplateSet{}
	th := templateHeader{}
	err := binary.Read(s.reader, binary.BigEndian, &th)
	s.errorIf(err)
	read = templateHeaderLength

	ts.TemplateId = th.TemplateId
	ts.Records = make([]TemplateRecord, th.FieldCount)
	for i := 0; i < int(th.FieldCount); i++ {
		f := TemplateRecord{}
		err = binary.Read(s.reader, binary.BigEndian, &f.FieldId)
		s.errorIf(err)
		read += 2
		err = binary.Read(s.reader, binary.BigEndian, &f.Length)
		s.errorIf(err)
		read += 2
		if f.FieldId >= 0x8000 {
			f.FieldId -= 0x8000
			err = binary.Read(s.reader, binary.BigEndian, &f.EnterpriseId)
			s.errorIf(err)
			read += 4
		}
		ts.Records[i] = f
	}

	return
}

// Reads a variable length information element. Returns a slice with the data
// in it, the total number of bytes read.
func (s *Session) readVariableLength() (bs []byte, r int) {
	var l int

	var l0 uint8
	err := binary.Read(s.reader, binary.BigEndian, &l0)
	s.errorIf(err)
	r += 1

	if l0 < 255 {
		l = int(l0)
	} else {
		var l1 uint16
		err = binary.Read(s.reader, binary.BigEndian, &l1)
		s.errorIf(err)
		r += 2
		l = int(l1)
	}

	bs = make([]byte, l)
	_, err = io.ReadFull(s.reader, bs)
	s.errorIf(err)
	r += len(bs)

	return
}
<|endoftext|>


<commit_before>package main

import (
	"os"

	"github.com/emicklei/artreyu/command"
	"github.com/emicklei/artreyu/model"
	"github.com/spf13/cobra"
)

var VERSION string = "dev"
var BUILDDATE string = "now"

func main() {
	model.Printf("artreyu-nexus - artreyu Sonatype Nexus plugin (build:%s, commit:%s)\n", BUILDDATE, VERSION)

	cmd, settings, artifact := command.NewPluginCommand()
	cmd.Use = "artreyu-nexus"
	cmd.Short = "archives and fetches from a Sonatype Nexus Repository"
	cmd.PersistentPreRun = func(cmd *cobra.Command, args []string) {
		// TODO refactor this
		model.Verbose = settings.Verbose
		if settings.Verbose {
			dir, _ := os.Getwd()
			model.Printf("working directory = [%s]", dir)
		}
	}

	// Need closures because only after cmd.Execute() the model data is populated.
	getArtifact := func() model.Artifact {
		return *artifact
	}
	getRepo := func() model.Repository {
		return NewRepository(model.RepositoryConfigNamed(settings, "nexus"), settings)
	}

	cmd.AddCommand(command.NewArchiveCommand(getArtifact, getRepo))
	cmd.AddCommand(command.NewFetchCommand(getArtifact, getRepo))
	cmd.Execute()
}
<commit_msg>use targetrepo from settings i.o fixed "nexus"<commit_after>package main

import (
	"os"

	"github.com/emicklei/artreyu/command"
	"github.com/emicklei/artreyu/model"
	"github.com/spf13/cobra"
)

var VERSION string = "dev"
var BUILDDATE string = "now"

func main() {
	model.Printf("artreyu-nexus - artreyu Sonatype Nexus plugin (build:%s, commit:%s)\n", BUILDDATE, VERSION)

	cmd, settings, artifact := command.NewPluginCommand()
	cmd.Use = "artreyu-nexus"
	cmd.Short = "archives and fetches from a Sonatype Nexus Repository"
	cmd.PersistentPreRun = func(cmd *cobra.Command, args []string) {
		// TODO refactor this
		model.Verbose = settings.Verbose
		if settings.Verbose {
			dir, _ := os.Getwd()
			model.Printf("working directory = [%s]", dir)
		}
	}

	// Need closures because only after cmd.Execute() the model data is populated.
	getArtifact := func() model.Artifact {
		return *artifact
	}
	getRepo := func() model.Repository {
		return NewRepository(model.RepositoryConfigNamed(settings, settings.TargetRepository), settings)
	}

	cmd.AddCommand(command.NewArchiveCommand(getArtifact, getRepo))
	cmd.AddCommand(command.NewFetchCommand(getArtifact, getRepo))
	cmd.Execute()
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/alecthomas/kingpin"
)

type Severity string

// Linter message severity levels.
const (
	Warning Severity = "warning"
	Error   Severity = "error"
)

type Linter string

func (l Linter) Command() string {
	return string(l[0:strings.Index(string(l), ":")])
}

func (l Linter) Pattern() string {
	return string(l[strings.Index(string(l), ":"):])
}

var (
	predefinedPatterns = map[string]string{
		"PATH:LINE:COL:MESSAGE": `(?P<path>[^:]+):(?P<line>\d+):(?P<col>\d+):\s*(?P<message>.*)`,
		"PATH:LINE:MESSAGE":     `(?P<path>[^:]+):(?P<line>\d+):\s*(?P<message>.*)`,
	}
	lintersFlag = map[string]string{
		// main.go:8:10: should omit type map[string]string from declaration of var linters; it will be inferred from the right-hand side
		"golint": "golint {path}:PATH:LINE:COL:MESSAGE",
		// test/stutter.go:19: missing argument for Printf("%d"): format reads arg 1, have only 0 args
		"vet":         "go vet {path}:PATH:LINE:MESSAGE",
		"gotype":      "gotype {path}:PATH:LINE:COL:MESSAGE",
		"errcheck":    `errcheck {path}:(?P<path>[^:]+):(?P<line>\d+):(?P<col>\d+)\t(?P<message>.*)`,
		"varcheck":    "varcheck {path}:PATH:LINE:MESSAGE",
		"structcheck": "structcheck {path}:PATH:LINE:MESSAGE",
		"defercheck":  "defercheck {path}:PATH:LINE:MESSAGE",
		"deadcode":    `deadcode {path}:deadcode: (?P<path>[^:]+):(?P<line>\d+):(?P<col>\d+):\s*(?P<message>.*)`,
		"gocyclo":     `gocyclo -over {mincyclo} {path}:(?P<cyclo>\d+)\s+\S+\s\S+\s+(?P<path>[^:]+):(?P<line>\d+):(?P<col>\d+)`,
	}
	linterMessageOverrideFlag = map[string]string{
		"errcheck":    "error return value not checked ({message})",
		"varcheck":    "unused global variable {message}",
		"structcheck": "unused struct field {message}",
		"gocyclo":     "cyclomatic complexity {cyclo} is high (> {mincyclo})",
	}
	linterSeverityFlag = map[string]string{
		"errcheck":    "warning",
		"golint":      "warning",
		"varcheck":    "warning",
		"structcheck": "warning",
		"deadcode":    "warning",
	}
	installMap = map[string]string{
		"golint":      "github.com/golang/lint/golint",
		"gotype":      "golang.org/x/tools/cmd/gotype",
		"errcheck":    "github.com/kisielk/errcheck",
		"defercheck":  "github.com/opennota/check/cmd/defercheck",
		"varcheck":    "github.com/opennota/check/cmd/varcheck",
		"structcheck": "github.com/opennota/check/cmd/structcheck",
		"vet":         "golang.org/x/tools/cmd/vet",
		"deadcode":    "github.com/remyoudompheng/go-misc/deadcode",
		"gocyclo":     "github.com/fzipp/gocyclo",
	}
	slowLinters = []string{"structcheck", "varcheck", "errcheck"}

	pathArg            = kingpin.Arg("path", "Directory to lint.").Default(".").String()
	fastFlag           = kingpin.Flag("fast", "Only run fast linters.").Bool()
	installFlag        = kingpin.Flag("install", "Attempt to install all known linters.").Short('i').Bool()
	updateFlag         = kingpin.Flag("update", "Pass -u to go tool when installing.").Short('u').Bool()
	disableLintersFlag = kingpin.Flag("disable", "List of linters to disable.").PlaceHolder("LINTER").Short('D').Strings()
	debugFlag          = kingpin.Flag("debug", "Display messages for failed linters, etc.").Short('d').Bool()
	concurrencyFlag    = kingpin.Flag("concurrency", "Number of concurrent linters to run.").Default("16").Short('j').Int()
	excludeFlag        = kingpin.Flag("exclude", "Exclude messages matching this regular expression.").PlaceHolder("REGEXP").String()
	cycloFlag          = kingpin.Flag("cyclo-over", "Report functions with cyclomatic complexity over N (using gocyclo).").Default("10").String()
)

func init() {
	kingpin.Flag("linter", "Specify a linter.").PlaceHolder("NAME:COMMAND:PATTERN").StringMapVar(&lintersFlag)
	kingpin.Flag("message-overrides", "Override message from linter. {message} will be expanded to the original message.").PlaceHolder("LINTER:MESSAGE").StringMapVar(&linterMessageOverrideFlag)
	kingpin.Flag("severity", "Map of linter severities.").PlaceHolder("LINTER:SEVERITY").StringMapVar(&linterSeverityFlag)
}

type Issue struct {
	severity Severity
	path     string
	line     int
	col      int
	message  string
}

func (m *Issue) String() string {
	col := ""
	if m.col != 0 {
		col = fmt.Sprintf("%d", m.col)
	}
	return fmt.Sprintf("%s:%d:%s:%s: %s", m.path, m.line, col, m.severity, m.message)
}

type Issues []*Issue

func (m Issues) Len() int      { return len(m) }
func (m Issues) Swap(i, j int) { m[i], m[j] = m[j], m[i] }
func (m Issues) Less(i, j int) bool {
	return m[i].path < m[j].path || m[i].line < m[j].line || m[i].col < m[j].col
}

func debug(format string, args ...interface{}) {
	if *debugFlag {
		fmt.Fprintf(os.Stderr, "DEBUG: "+format+"\n", args...)
	}
}

func formatLinters() string {
	w := bytes.NewBuffer(nil)
	for command, description := range lintersFlag {
		linter := Linter(description)
		fmt.Fprintf(w, "    %s -> %s -> %s\n", command, linter.Command(), linter.Pattern())
	}
	return w.String()
}

func formatSeverity() string {
	w := bytes.NewBuffer(nil)
	for name, severity := range linterSeverityFlag {
		fmt.Fprintf(w, "    %s -> %s\n", name, severity)
	}
	return w.String()
}

func exArgs() (arg0 string, arg1 string) {
	if runtime.GOOS == "windows" {
		arg0 = "cmd"
		arg1 = "/C"
	} else {
		arg0 = "/bin/sh"
		arg1 = "-c"
	}
	return
}

type Vars map[string]string

func (v Vars) Replace(s string) string {
	for k, v := range v {
		s = strings.Replace(s, fmt.Sprintf("{%s}", k), v, -1)
	}
	return s
}

func main() {
	kingpin.CommandLine.Help = fmt.Sprintf(`Aggregate and normalise the output of a whole bunch of Go linters.

Default linters:

%s

Severity override map (default is "error"):

%s
`, formatLinters(), formatSeverity())
	kingpin.Parse()
	var filter *regexp.Regexp
	if *excludeFlag != "" {
		filter = regexp.MustCompile(*excludeFlag)
	}

	if *fastFlag {
		*disableLintersFlag = append(*disableLintersFlag, slowLinters...)
	}

	if *installFlag {
		for name, target := range installMap {
			cmd := "go get"
			if *debugFlag {
				cmd += " -v"
			}
			if *updateFlag {
				cmd += " -u"
			}
			cmd += " " + target
			fmt.Printf("Installing %s -> %s\n", name, cmd)
			arg0, arg1 := exArgs()
			c := exec.Command(arg0, arg1, cmd)
			c.Stdout = os.Stdout
			c.Stderr = os.Stderr
			err := c.Run()
			if err != nil {
				kingpin.CommandLine.Errorf(os.Stderr, "failed to install %s: %s", name, err)
			}
		}
		return
	}

	runtime.GOMAXPROCS(*concurrencyFlag)

	disable := map[string]bool{}
	for _, linter := range *disableLintersFlag {
		disable[linter] = true
	}

	start := time.Now()
	paths := *pathArg
	concurrency := make(chan bool, *concurrencyFlag)
	issues := make(chan *Issue, 100000)
	wg := &sync.WaitGroup{}
	for name, description := range lintersFlag {
		if _, ok := disable[name]; ok {
			debug("linter %s disabled", name)
			continue
		}
		parts := strings.SplitN(description, ":", 2)
		command := parts[0]
		pattern := parts[1]

		wg.Add(1)
		vars := Vars{
			"mincyclo": *cycloFlag,
		}
		go func(name, command, pattern string) {
			concurrency <- true
			executeLinter(issues, name, command, pattern, paths, vars)
			<-concurrency
			wg.Done()
		}(name, command, pattern)
	}

	wg.Wait()
	close(issues)
	for issue := range issues {
		if filter != nil && filter.MatchString(issue.String()) {
			continue
		}
		fmt.Printf("%s\n", issue)
	}
	elapsed := time.Now().Sub(start)
	debug("total elapsed time %s", elapsed)
}

func executeLinter(issues chan *Issue, name, command, pattern, paths string, vars Vars) {
	debug("linting with %s: %s", name, command)

	start := time.Now()
	if p, ok := predefinedPatterns[pattern]; ok {
		pattern = p
	}
	regexp.Compile(pattern)
	re, err := regexp.Compile(pattern)
	kingpin.FatalIfError(err, "invalid pattern for '"+command+"'")

	vars["path"] = paths
	command = vars.Replace(command)
	debug("executing %s", command)
	arg0, arg1 := exArgs()
	cmd := exec.Command(arg0, arg1, command)
	out, err := cmd.CombinedOutput()
	if err != nil {
		if _, ok := err.(*exec.ExitError); !ok {
			debug("warning: %s failed: %s", command, err)
			return
		}
		debug("warning: %s returned %s", command, err)
	}

	for _, line := range bytes.Split(out, []byte("\n")) {
		groups := re.FindAllSubmatch(line, -1)
		if groups == nil {
			debug("%s (didn't match): '%s'", name, line)
			continue
		}
		issue := &Issue{}
		for i, name := range re.SubexpNames() {
			part := string(groups[0][i])
			if name != "" {
				vars[name] = part
			}
			switch name {
			case "path":
				issue.path = part

			case "line":
				n, err := strconv.ParseInt(part, 10, 32)
				kingpin.FatalIfError(err, "line matched invalid integer")
				issue.line = int(n)

			case "col":
				n, err := strconv.ParseInt(part, 10, 32)
				kingpin.FatalIfError(err, "col matched invalid integer")
				issue.col = int(n)

			case "message":
				issue.message = part

			case "":
			}
		}
		if m, ok := linterMessageOverrideFlag[name]; ok {
			issue.message = vars.Replace(m)
		}
		if sev, ok := linterSeverityFlag[name]; ok {
			issue.severity = Severity(sev)
		} else {
			issue.severity = "error"
		}
		issues <- issue
	}

	elapsed := time.Now().Sub(start)
	debug("%s linter took %s", name, elapsed)
}
<commit_msg>Make gocyclo issues warnings.<commit_after>package main

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/alecthomas/kingpin"
)

type Severity string

// Linter message severity levels.
const (
	Warning Severity = "warning"
	Error   Severity = "error"
)

type Linter string

func (l Linter) Command() string {
	return string(l[0:strings.Index(string(l), ":")])
}

func (l Linter) Pattern() string {
	return string(l[strings.Index(string(l), ":"):])
}

var (
	predefinedPatterns = map[string]string{
		"PATH:LINE:COL:MESSAGE": `(?P<path>[^:]+):(?P<line>\d+):(?P<col>\d+):\s*(?P<message>.*)`,
		"PATH:LINE:MESSAGE":     `(?P<path>[^:]+):(?P<line>\d+):\s*(?P<message>.*)`,
	}
	lintersFlag = map[string]string{
		// main.go:8:10: should omit type map[string]string from declaration of var linters; it will be inferred from the right-hand side
		"golint": "golint {path}:PATH:LINE:COL:MESSAGE",
		// test/stutter.go:19: missing argument for Printf("%d"): format reads arg 1, have only 0 args
		"vet":         "go vet {path}:PATH:LINE:MESSAGE",
		"gotype":      "gotype {path}:PATH:LINE:COL:MESSAGE",
		"errcheck":    `errcheck {path}:(?P<path>[^:]+):(?P<line>\d+):(?P<col>\d+)\t(?P<message>.*)`,
		"varcheck":    "varcheck {path}:PATH:LINE:MESSAGE",
		"structcheck": "structcheck {path}:PATH:LINE:MESSAGE",
		"defercheck":  "defercheck {path}:PATH:LINE:MESSAGE",
		"deadcode":    `deadcode {path}:deadcode: (?P<path>[^:]+):(?P<line>\d+):(?P<col>\d+):\s*(?P<message>.*)`,
		"gocyclo":     `gocyclo -over {mincyclo} {path}:(?P<cyclo>\d+)\s+\S+\s\S+\s+(?P<path>[^:]+):(?P<line>\d+):(?P<col>\d+)`,
	}
	linterMessageOverrideFlag = map[string]string{
		"errcheck":    "error return value not checked ({message})",
		"varcheck":    "unused global variable {message}",
		"structcheck": "unused struct field {message}",
		"gocyclo":     "cyclomatic complexity {cyclo} is high (> {mincyclo})",
	}
	linterSeverityFlag = map[string]string{
		"errcheck":    "warning",
		"golint":      "warning",
		"varcheck":    "warning",
		"structcheck": "warning",
		"deadcode":    "warning",
		"gocyclo":     "warning",
	}
	installMap = map[string]string{
		"golint":      "github.com/golang/lint/golint",
		"gotype":      "golang.org/x/tools/cmd/gotype",
		"errcheck":    "github.com/kisielk/errcheck",
		"defercheck":  "github.com/opennota/check/cmd/defercheck",
		"varcheck":    "github.com/opennota/check/cmd/varcheck",
		"structcheck": "github.com/opennota/check/cmd/structcheck",
		"vet":         "golang.org/x/tools/cmd/vet",
		"deadcode":    "github.com/remyoudompheng/go-misc/deadcode",
		"gocyclo":     "github.com/fzipp/gocyclo",
	}
	slowLinters = []string{"structcheck", "varcheck", "errcheck"}

	pathArg            = kingpin.Arg("path", "Directory to lint.").Default(".").String()
	fastFlag           = kingpin.Flag("fast", "Only run fast linters.").Bool()
	installFlag        = kingpin.Flag("install", "Attempt to install all known linters.").Short('i').Bool()
	updateFlag         = kingpin.Flag("update", "Pass -u to go tool when installing.").Short('u').Bool()
	disableLintersFlag = kingpin.Flag("disable", "List of linters to disable.").PlaceHolder("LINTER").Short('D').Strings()
	debugFlag          = kingpin.Flag("debug", "Display messages for failed linters, etc.").Short('d').Bool()
	concurrencyFlag    = kingpin.Flag("concurrency", "Number of concurrent linters to run.").Default("16").Short('j').Int()
	excludeFlag        = kingpin.Flag("exclude", "Exclude messages matching this regular expression.").PlaceHolder("REGEXP").String()
	cycloFlag          = kingpin.Flag("cyclo-over", "Report functions with cyclomatic complexity over N (using gocyclo).").Default("10").String()
)

func init() {
	kingpin.Flag("linter", "Specify a linter.").PlaceHolder("NAME:COMMAND:PATTERN").StringMapVar(&lintersFlag)
	kingpin.Flag("message-overrides", "Override message from linter. {message} will be expanded to the original message.").PlaceHolder("LINTER:MESSAGE").StringMapVar(&linterMessageOverrideFlag)
	kingpin.Flag("severity", "Map of linter severities.").PlaceHolder("LINTER:SEVERITY").StringMapVar(&linterSeverityFlag)
}

type Issue struct {
	severity Severity
	path     string
	line     int
	col      int
	message  string
}

func (m *Issue) String() string {
	col := ""
	if m.col != 0 {
		col = fmt.Sprintf("%d", m.col)
	}
	return fmt.Sprintf("%s:%d:%s:%s: %s", m.path, m.line, col, m.severity, m.message)
}

type Issues []*Issue

func (m Issues) Len() int      { return len(m) }
func (m Issues) Swap(i, j int) { m[i], m[j] = m[j], m[i] }
func (m Issues) Less(i, j int) bool {
	return m[i].path < m[j].path || m[i].line < m[j].line || m[i].col < m[j].col
}

func debug(format string, args ...interface{}) {
	if *debugFlag {
		fmt.Fprintf(os.Stderr, "DEBUG: "+format+"\n", args...)
	}
}

func formatLinters() string {
	w := bytes.NewBuffer(nil)
	for command, description := range lintersFlag {
		linter := Linter(description)
		fmt.Fprintf(w, "    %s -> %s -> %s\n", command, linter.Command(), linter.Pattern())
	}
	return w.String()
}

func formatSeverity() string {
	w := bytes.NewBuffer(nil)
	for name, severity := range linterSeverityFlag {
		fmt.Fprintf(w, "    %s -> %s\n", name, severity)
	}
	return w.String()
}

func exArgs() (arg0 string, arg1 string) {
	if runtime.GOOS == "windows" {
		arg0 = "cmd"
		arg1 = "/C"
	} else {
		arg0 = "/bin/sh"
		arg1 = "-c"
	}
	return
}

type Vars map[string]string

func (v Vars) Replace(s string) string {
	for k, v := range v {
		s = strings.Replace(s, fmt.Sprintf("{%s}", k), v, -1)
	}
	return s
}

func main() {
	kingpin.CommandLine.Help = fmt.Sprintf(`Aggregate and normalise the output of a whole bunch of Go linters.

Default linters:

%s

Severity override map (default is "error"):

%s
`, formatLinters(), formatSeverity())
	kingpin.Parse()
	var filter *regexp.Regexp
	if *excludeFlag != "" {
		filter = regexp.MustCompile(*excludeFlag)
	}

	if *fastFlag {
		*disableLintersFlag = append(*disableLintersFlag, slowLinters...)
	}

	if *installFlag {
		for name, target := range installMap {
			cmd := "go get"
			if *debugFlag {
				cmd += " -v"
			}
			if *updateFlag {
				cmd += " -u"
			}
			cmd += " " + target
			fmt.Printf("Installing %s -> %s\n", name, cmd)
			arg0, arg1 := exArgs()
			c := exec.Command(arg0, arg1, cmd)
			c.Stdout = os.Stdout
			c.Stderr = os.Stderr
			err := c.Run()
			if err != nil {
				kingpin.CommandLine.Errorf(os.Stderr, "failed to install %s: %s", name, err)
			}
		}
		return
	}

	runtime.GOMAXPROCS(*concurrencyFlag)

	disable := map[string]bool{}
	for _, linter := range *disableLintersFlag {
		disable[linter] = true
	}

	start := time.Now()
	paths := *pathArg
	concurrency := make(chan bool, *concurrencyFlag)
	issues := make(chan *Issue, 100000)
	wg := &sync.WaitGroup{}
	for name, description := range lintersFlag {
		if _, ok := disable[name]; ok {
			debug("linter %s disabled", name)
			continue
		}
		parts := strings.SplitN(description, ":", 2)
		command := parts[0]
		pattern := parts[1]

		wg.Add(1)
		vars := Vars{
			"mincyclo": *cycloFlag,
		}
		go func(name, command, pattern string) {
			concurrency <- true
			executeLinter(issues, name, command, pattern, paths, vars)
			<-concurrency
			wg.Done()
		}(name, command, pattern)
	}

	wg.Wait()
	close(issues)
	for issue := range issues {
		if filter != nil && filter.MatchString(issue.String()) {
			continue
		}
		fmt.Printf("%s\n", issue)
	}
	elapsed := time.Now().Sub(start)
	debug("total elapsed time %s", elapsed)
}

func executeLinter(issues chan *Issue, name, command, pattern, paths string, vars Vars) {
	debug("linting with %s: %s", name, command)

	start := time.Now()
	if p, ok := predefinedPatterns[pattern]; ok {
		pattern = p
	}
	regexp.Compile(pattern)
	re, err := regexp.Compile(pattern)
	kingpin.FatalIfError(err, "invalid pattern for '"+command+"'")

	vars["path"] = paths
	command = vars.Replace(command)
	debug("executing %s", command)
	arg0, arg1 := exArgs()
	cmd := exec.Command(arg0, arg1, command)
	out, err := cmd.CombinedOutput()
	if err != nil {
		if _, ok := err.(*exec.ExitError); !ok {
			debug("warning: %s failed: %s", command, err)
			return
		}
		debug("warning: %s returned %s", command, err)
	}

	for _, line := range bytes.Split(out, []byte("\n")) {
		groups := re.FindAllSubmatch(line, -1)
		if groups == nil {
			debug("%s (didn't match): '%s'", name, line)
			continue
		}
		issue := &Issue{}
		for i, name := range re.SubexpNames() {
			part := string(groups[0][i])
			if name != "" {
				vars[name] = part
			}
			switch name {
			case "path":
				issue.path = part

			case "line":
				n, err := strconv.ParseInt(part, 10, 32)
				kingpin.FatalIfError(err, "line matched invalid integer")
				issue.line = int(n)

			case "col":
				n, err := strconv.ParseInt(part, 10, 32)
				kingpin.FatalIfError(err, "col matched invalid integer")
				issue.col = int(n)

			case "message":
				issue.message = part

			case "":
			}
		}
		if m, ok := linterMessageOverrideFlag[name]; ok {
			issue.message = vars.Replace(m)
		}
		if sev, ok := linterSeverityFlag[name]; ok {
			issue.severity = Severity(sev)
		} else {
			issue.severity = "error"
		}
		issues <- issue
	}

	elapsed := time.Now().Sub(start)
	debug("%s linter took %s", name, elapsed)
}
<|endoftext|>


<commit_before>package mains

import (
	"fmt"
	"os"
	"runtime"
	"unsafe"

	"../history"
	"../interpreter"
	"../lua"
	ole "../lua/ole"
	"../readline"
)

const REGKEY_INTERPRETER = "nyagos.interpreter"

func setRegInt(L lua.Lua, it *interpreter.Interpreter) {
	L.PushValue(lua.LUA_REGISTRYINDEX)
	L.PushLightUserData(unsafe.Pointer(it))
	L.SetField(-2, REGKEY_INTERPRETER)
	L.Pop(1)
}

func getRegInt(L lua.Lua) *interpreter.Interpreter {
	L.PushValue(lua.LUA_REGISTRYINDEX)
	L.GetField(-1, REGKEY_INTERPRETER)
	rc := (*interpreter.Interpreter)(L.ToUserData(-1))
	L.Pop(2)
	return rc
}

func NyagosCallLua(L lua.Lua, it *interpreter.Interpreter, nargs int, nresult int) error {
	save := getRegInt(L)
	setRegInt(L, it)
	err := L.Call(nargs, nresult)
	setRegInt(L, save)
	return err
}

var orgArgHook func(*interpreter.Interpreter, []string) ([]string, error)

var luaArgsFilter lua.Pushable = lua.TNil{}

func newArgHook(it *interpreter.Interpreter, args []string) ([]string, error) {
	L, err := NewNyagosLua()
	if err != nil {
		return nil, err
	}
	defer L.Close()
	L.Push(luaArgsFilter)
	if !L.IsFunction(-1) {
		return orgArgHook(it, args)
	}
	L.NewTable()
	for i := 0; i < len(args); i++ {
		L.PushString(args[i])
		L.RawSetI(-2, lua.Integer(i))
	}
	if err := L.Call(1, 1); err != nil {
		fmt.Fprintf(os.Stderr, "%s\n", err)
		return orgArgHook(it, args)
	}
	if L.GetType(-1) != lua.LUA_TTABLE {
		return orgArgHook(it, args)
	}
	newargs := []string{}
	for i := lua.Integer(0); true; i++ {
		L.PushInteger(i)
		L.GetTable(-2)
		if L.GetType(-1) == lua.LUA_TNIL {
			break
		}
		arg1, arg1err := L.ToString(-1)
		if arg1err == nil {
			newargs = append(newargs, arg1)
		} else {
			fmt.Fprintln(os.Stderr, arg1err.Error())
		}
		L.Pop(1)
	}
	return orgArgHook(it, newargs)
}

var orgOnCommandNotFound func(*interpreter.Interpreter, error) error

var luaOnCommandNotFound lua.Pushable = lua.TNil{}

func on_command_not_found(inte *interpreter.Interpreter, err error) error {
	L, err := NewNyagosLua()
	if err != nil {
		return err
	}
	defer L.Close()

	L.Push(luaOnCommandNotFound)
	if !L.IsFunction(-1) {
		L.Pop(1)
		return orgOnCommandNotFound(inte, err)
	}
	L.NewTable()
	for key, val := range inte.Args {
		L.PushString(val)
		L.RawSetI(-2, lua.Integer(key))
	}
	err1 := L.Call(1, 1)
	defer L.Pop(1)
	if err1 != nil {
		return err
	}
	if L.ToBool(-1) {
		return nil
	} else {
		return orgOnCommandNotFound(inte, err)
	}
}

func emptyToNil(s string) lua.Pushable {
	if s == "" {
		return &lua.TNil{}
	} else {
		return lua.TString(s)
	}
}

var option_table_member = map[string]IProperty{
	"glob": &lua.BoolProperty{&interpreter.WildCardExpansionAlways},
}

func getOption(L lua.Lua) int {
	key, key_err := L.ToString(2)
	if key_err != nil {
		return L.Push(nil, key_err)
	}
	val, val_ok := option_table_member[key]
	if !val_ok {
		return L.Push(nil)
	}
	return L.Push(val)
}

func setOption(L lua.Lua) int {
	key, key_err := L.ToString(2)
	if key_err != nil {
		return L.Push(nil, key_err)
	}
	opt, opt_ok := option_table_member[key]
	if !opt_ok {
		print(key, " not found\n")
		return L.Push(nil)
	}
	if err := opt.Set(L, 3); err != nil {
		return L.Push(nil, err.Error())
	} else {
		return L.Push(true)
	}
}

var nyagos_table_member map[string]lua.Pushable

func getNyagosTable(L lua.Lua) int {
	index, index_err := L.ToString(2)
	if index_err != nil {
		return L.Push(nil, index_err.Error())
	}
	if entry, entry_ok := nyagos_table_member[index]; entry_ok {
		return L.Push(entry)
	} else if index == "exe" {
		if exeName, exeNameErr := os.Executable(); exeNameErr != nil {
			return L.Push(nil, exeNameErr.Error())
		} else {
			L.PushString(exeName)
			return 1
		}
	} else {
		L.PushNil()
		return 1
	}
}

type IProperty interface {
	Push(lua.Lua) int
	Set(lua.Lua, int) error
}

func setNyagosTable(L lua.Lua) int {
	index, index_err := L.ToString(2)
	if index_err != nil {
		return L.Push(nil, index_err)
	}
	if current_value, exists := nyagos_table_member[index]; exists {
		if property, castOk := current_value.(IProperty); castOk {
			if err := property.Set(L, 3); err != nil {
				fmt.Fprintf(os.Stderr, "nyagos.%s: %s\n", index, err.Error())
				return L.Push(nil, err)
			} else {
				return L.Push(true)
			}
		} else {
			value, value_err := L.ToPushable(3)
			if value_err != nil {
				return L.Push(nil, value_err)
			}
			nyagos_table_member[index] = value
			return L.Push(true)
		}
	} else {
		fmt.Fprintf(os.Stderr, "nyagos.%s: reserved variable.\n", index)
		return L.Push(nil)
	}
}

var share_table = map[string]lua.Pushable{}

func getShareTable(L lua.Lua) int {
	key, keyErr := L.ToString(-1)
	if keyErr != nil {
		return L.Push(nil, keyErr)
	}
	if value, ok := share_table[key]; ok {
		return L.Push(value)
	} else {
		L.PushNil()
		return 1
	}
}

func setShareTable(L lua.Lua) int {
	key, keyErr := L.ToString(-2)
	if keyErr != nil {
		return L.Push(nil, keyErr)
	}
	value, valErr := L.ToPushable(-1)
	if valErr != nil {
		fmt.Fprintf(os.Stderr, "%s: %s\n", key, valErr.Error())
		return L.Push(nil, valErr)
	}
	share_table[key] = value
	return 1
}

var hook_setuped = false

func NewNyagosLua() (lua.Lua, error) {
	this, err := lua.New()
	if err != nil {
		return this, err
	}
	this.OpenLibs()

	this.Push(lua.NewVirtualTable("nyagos", getNyagosTable, setNyagosTable))
	this.SetGlobal("nyagos")

	this.Push(lua.NewVirtualTable("share", getShareTable, setShareTable))
	this.SetGlobal("share")

	if !hook_setuped {
		orgArgHook = interpreter.SetArgsHook(newArgHook)

		orgOnCommandNotFound = interpreter.OnCommandNotFound
		interpreter.OnCommandNotFound = on_command_not_found
		hook_setuped = true
	}
	return this, nil
}

var silentmode = false

func init() {
	nyagos_table_member = map[string]lua.Pushable{
		"access":               lua.TGoFunction(cmdAccess),
		"alias":                lua.NewVirtualTable("nyagos.alias", cmdGetAlias, cmdSetAlias),
		"antihistquot":         lua.StringProperty{&history.DisableMarks},
		"argsfilter":           lua.Property{&luaArgsFilter},
		"atou":                 lua.TGoFunction(cmdAtoU),
		"key":                  lua.NewVirtualTable("nyagos.key", cmdGetBindKey, cmdBindKey),
		"bindkey":              lua.TGoFunction(cmdBindKey),
		"chdir":                lua.TGoFunction(cmdChdir),
		"commit":               emptyToNil(Commit),
		"commonprefix":         lua.TGoFunction(cmdCommonPrefix),
		"completion_hook":      lua.Property{&completionHook},
		"create_object":        lua.TGoFunction(ole.CreateObject),
		"default_prompt":       lua.TGoFunction(nyagosPrompt),
		"env":                  lua.NewVirtualTable("nyagos.env", cmdGetEnv, cmdSetEnv),
		"eval":                 lua.TGoFunction(cmdEval),
		"exec":                 lua.TGoFunction(cmdExec),
		"filter":               lua.Property{&luaFilter},
		"getalias":             lua.TGoFunction(cmdGetAlias),
		"getenv":               lua.TGoFunction(cmdGetEnv),
		"gethistory":           lua.TGoFunction(cmdGetHistory),
		"getkey":               lua.TGoFunction(cmdGetKey),
		"getviewwidth":         lua.TGoFunction(cmdGetViewWidth),
		"getwd":                lua.TGoFunction(cmdGetwd),
		"glob":                 lua.TGoFunction(cmdGlob),
		"goarch":               lua.TString(runtime.GOARCH),
		"goversion":            lua.TString(runtime.Version()),
		"histchar":             lua.StringProperty{&history.Mark},
		"lines":                lua.TGoFunction(cmdLines),
		"loadfile":             lua.TGoFunction(cmdLoadFile),
		"netdrivetounc":        lua.TGoFunction(cmdNetDriveToUNC),
		"on_command_not_found": lua.Property{&luaOnCommandNotFound},
		"open":                 lua.TGoFunction(cmdOpenFile),
		"option":               lua.NewVirtualTable("nyagos.option", getOption, setOption),
		"pathjoin":             lua.TGoFunction(cmdPathJoin),
		"prompt":               lua.Property{&prompt_hook},
		"quotation":            lua.StringProperty{&readline.Delimiters},
		"raweval":              lua.TGoFunction(cmdRawEval),
		"rawexec":              lua.TGoFunction(cmdRawExec),
		"resetcharwidth":       lua.TGoFunction(cmdResetCharWidth),
		"setalias":             lua.TGoFunction(cmdSetAlias),
		"setenv":               lua.TGoFunction(cmdSetEnv),
		"setrunewidth":         lua.TGoFunction(cmdSetRuneWidth),
		"shellexecute":         lua.TGoFunction(cmdShellExecute),
		"silentmode":           &lua.BoolProperty{&silentmode},
		"stamp":                emptyToNil(Stamp),
		"stat":                 lua.TGoFunction(cmdStat),
		"utoa":                 lua.TGoFunction(cmdUtoA),
		"version":              emptyToNil(Version),
		"which":                lua.TGoFunction(cmdWhich),
		"write":                lua.TGoFunction(cmdWrite),
		"writerr":              lua.TGoFunction(cmdWriteErr),
	}
}
<commit_msg>Fix: lua: nyagos.version was always empty.<commit_after>package mains

import (
	"fmt"
	"os"
	"runtime"
	"unsafe"

	"../history"
	"../interpreter"
	"../lua"
	ole "../lua/ole"
	"../readline"
)

const REGKEY_INTERPRETER = "nyagos.interpreter"

func setRegInt(L lua.Lua, it *interpreter.Interpreter) {
	L.PushValue(lua.LUA_REGISTRYINDEX)
	L.PushLightUserData(unsafe.Pointer(it))
	L.SetField(-2, REGKEY_INTERPRETER)
	L.Pop(1)
}

func getRegInt(L lua.Lua) *interpreter.Interpreter {
	L.PushValue(lua.LUA_REGISTRYINDEX)
	L.GetField(-1, REGKEY_INTERPRETER)
	rc := (*interpreter.Interpreter)(L.ToUserData(-1))
	L.Pop(2)
	return rc
}

func NyagosCallLua(L lua.Lua, it *interpreter.Interpreter, nargs int, nresult int) error {
	save := getRegInt(L)
	setRegInt(L, it)
	err := L.Call(nargs, nresult)
	setRegInt(L, save)
	return err
}

var orgArgHook func(*interpreter.Interpreter, []string) ([]string, error)

var luaArgsFilter lua.Pushable = lua.TNil{}

func newArgHook(it *interpreter.Interpreter, args []string) ([]string, error) {
	L, err := NewNyagosLua()
	if err != nil {
		return nil, err
	}
	defer L.Close()
	L.Push(luaArgsFilter)
	if !L.IsFunction(-1) {
		return orgArgHook(it, args)
	}
	L.NewTable()
	for i := 0; i < len(args); i++ {
		L.PushString(args[i])
		L.RawSetI(-2, lua.Integer(i))
	}
	if err := L.Call(1, 1); err != nil {
		fmt.Fprintf(os.Stderr, "%s\n", err)
		return orgArgHook(it, args)
	}
	if L.GetType(-1) != lua.LUA_TTABLE {
		return orgArgHook(it, args)
	}
	newargs := []string{}
	for i := lua.Integer(0); true; i++ {
		L.PushInteger(i)
		L.GetTable(-2)
		if L.GetType(-1) == lua.LUA_TNIL {
			break
		}
		arg1, arg1err := L.ToString(-1)
		if arg1err == nil {
			newargs = append(newargs, arg1)
		} else {
			fmt.Fprintln(os.Stderr, arg1err.Error())
		}
		L.Pop(1)
	}
	return orgArgHook(it, newargs)
}

var orgOnCommandNotFound func(*interpreter.Interpreter, error) error

var luaOnCommandNotFound lua.Pushable = lua.TNil{}

func on_command_not_found(inte *interpreter.Interpreter, err error) error {
	L, err := NewNyagosLua()
	if err != nil {
		return err
	}
	defer L.Close()

	L.Push(luaOnCommandNotFound)
	if !L.IsFunction(-1) {
		L.Pop(1)
		return orgOnCommandNotFound(inte, err)
	}
	L.NewTable()
	for key, val := range inte.Args {
		L.PushString(val)
		L.RawSetI(-2, lua.Integer(key))
	}
	err1 := L.Call(1, 1)
	defer L.Pop(1)
	if err1 != nil {
		return err
	}
	if L.ToBool(-1) {
		return nil
	} else {
		return orgOnCommandNotFound(inte, err)
	}
}

var option_table_member = map[string]IProperty{
	"glob": &lua.BoolProperty{&interpreter.WildCardExpansionAlways},
}

func getOption(L lua.Lua) int {
	key, key_err := L.ToString(2)
	if key_err != nil {
		return L.Push(nil, key_err)
	}
	val, val_ok := option_table_member[key]
	if !val_ok {
		return L.Push(nil)
	}
	return L.Push(val)
}

func setOption(L lua.Lua) int {
	key, key_err := L.ToString(2)
	if key_err != nil {
		return L.Push(nil, key_err)
	}
	opt, opt_ok := option_table_member[key]
	if !opt_ok {
		print(key, " not found\n")
		return L.Push(nil)
	}
	if err := opt.Set(L, 3); err != nil {
		return L.Push(nil, err.Error())
	} else {
		return L.Push(true)
	}
}

var nyagos_table_member map[string]lua.Pushable

func getNyagosTable(L lua.Lua) int {
	index, index_err := L.ToString(2)
	if index_err != nil {
		return L.Push(nil, index_err.Error())
	}
	if entry, entry_ok := nyagos_table_member[index]; entry_ok {
		return L.Push(entry)
	} else if index == "exe" {
		if exeName, exeNameErr := os.Executable(); exeNameErr != nil {
			return L.Push(nil, exeNameErr.Error())
		} else {
			L.PushString(exeName)
			return 1
		}
	} else {
		L.PushNil()
		return 1
	}
}

type IProperty interface {
	Push(lua.Lua) int
	Set(lua.Lua, int) error
}

func setNyagosTable(L lua.Lua) int {
	index, index_err := L.ToString(2)
	if index_err != nil {
		return L.Push(nil, index_err)
	}
	if current_value, exists := nyagos_table_member[index]; exists {
		if property, castOk := current_value.(IProperty); castOk {
			if err := property.Set(L, 3); err != nil {
				fmt.Fprintf(os.Stderr, "nyagos.%s: %s\n", index, err.Error())
				return L.Push(nil, err)
			} else {
				return L.Push(true)
			}
		} else {
			value, value_err := L.ToPushable(3)
			if value_err != nil {
				return L.Push(nil, value_err)
			}
			nyagos_table_member[index] = value
			return L.Push(true)
		}
	} else {
		fmt.Fprintf(os.Stderr, "nyagos.%s: reserved variable.\n", index)
		return L.Push(nil)
	}
}

var share_table = map[string]lua.Pushable{}

func getShareTable(L lua.Lua) int {
	key, keyErr := L.ToString(-1)
	if keyErr != nil {
		return L.Push(nil, keyErr)
	}
	if value, ok := share_table[key]; ok {
		return L.Push(value)
	} else {
		L.PushNil()
		return 1
	}
}

func setShareTable(L lua.Lua) int {
	key, keyErr := L.ToString(-2)
	if keyErr != nil {
		return L.Push(nil, keyErr)
	}
	value, valErr := L.ToPushable(-1)
	if valErr != nil {
		fmt.Fprintf(os.Stderr, "%s: %s\n", key, valErr.Error())
		return L.Push(nil, valErr)
	}
	share_table[key] = value
	return 1
}

var hook_setuped = false

func NewNyagosLua() (lua.Lua, error) {
	this, err := lua.New()
	if err != nil {
		return this, err
	}
	this.OpenLibs()

	this.Push(lua.NewVirtualTable("nyagos", getNyagosTable, setNyagosTable))
	this.SetGlobal("nyagos")

	this.Push(lua.NewVirtualTable("share", getShareTable, setShareTable))
	this.SetGlobal("share")

	if !hook_setuped {
		orgArgHook = interpreter.SetArgsHook(newArgHook)

		orgOnCommandNotFound = interpreter.OnCommandNotFound
		interpreter.OnCommandNotFound = on_command_not_found
		hook_setuped = true
	}
	return this, nil
}

var silentmode = false

func init() {
	nyagos_table_member = map[string]lua.Pushable{
		"access":               lua.TGoFunction(cmdAccess),
		"alias":                lua.NewVirtualTable("nyagos.alias", cmdGetAlias, cmdSetAlias),
		"antihistquot":         lua.StringProperty{&history.DisableMarks},
		"argsfilter":           lua.Property{&luaArgsFilter},
		"atou":                 lua.TGoFunction(cmdAtoU),
		"key":                  lua.NewVirtualTable("nyagos.key", cmdGetBindKey, cmdBindKey),
		"bindkey":              lua.TGoFunction(cmdBindKey),
		"chdir":                lua.TGoFunction(cmdChdir),
		"commit":               lua.StringProperty{&Commit},
		"commonprefix":         lua.TGoFunction(cmdCommonPrefix),
		"completion_hook":      lua.Property{&completionHook},
		"create_object":        lua.TGoFunction(ole.CreateObject),
		"default_prompt":       lua.TGoFunction(nyagosPrompt),
		"env":                  lua.NewVirtualTable("nyagos.env", cmdGetEnv, cmdSetEnv),
		"eval":                 lua.TGoFunction(cmdEval),
		"exec":                 lua.TGoFunction(cmdExec),
		"filter":               lua.Property{&luaFilter},
		"getalias":             lua.TGoFunction(cmdGetAlias),
		"getenv":               lua.TGoFunction(cmdGetEnv),
		"gethistory":           lua.TGoFunction(cmdGetHistory),
		"getkey":               lua.TGoFunction(cmdGetKey),
		"getviewwidth":         lua.TGoFunction(cmdGetViewWidth),
		"getwd":                lua.TGoFunction(cmdGetwd),
		"glob":                 lua.TGoFunction(cmdGlob),
		"goarch":               lua.TString(runtime.GOARCH),
		"goversion":            lua.TString(runtime.Version()),
		"histchar":             lua.StringProperty{&history.Mark},
		"lines":                lua.TGoFunction(cmdLines),
		"loadfile":             lua.TGoFunction(cmdLoadFile),
		"netdrivetounc":        lua.TGoFunction(cmdNetDriveToUNC),
		"on_command_not_found": lua.Property{&luaOnCommandNotFound},
		"open":                 lua.TGoFunction(cmdOpenFile),
		"option":               lua.NewVirtualTable("nyagos.option", getOption, setOption),
		"pathjoin":             lua.TGoFunction(cmdPathJoin),
		"prompt":               lua.Property{&prompt_hook},
		"quotation":            lua.StringProperty{&readline.Delimiters},
		"raweval":              lua.TGoFunction(cmdRawEval),
		"rawexec":              lua.TGoFunction(cmdRawExec),
		"resetcharwidth":       lua.TGoFunction(cmdResetCharWidth),
		"setalias":             lua.TGoFunction(cmdSetAlias),
		"setenv":               lua.TGoFunction(cmdSetEnv),
		"setrunewidth":         lua.TGoFunction(cmdSetRuneWidth),
		"shellexecute":         lua.TGoFunction(cmdShellExecute),
		"silentmode":           &lua.BoolProperty{&silentmode},
		"stamp":                lua.StringProperty{&Stamp},
		"stat":                 lua.TGoFunction(cmdStat),
		"utoa":                 lua.TGoFunction(cmdUtoA),
		"version":              lua.StringProperty{&Version},
		"which":                lua.TGoFunction(cmdWhich),
		"write":                lua.TGoFunction(cmdWrite),
		"writerr":              lua.TGoFunction(cmdWriteErr),
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"sync"
)

// bonus is serialised to a POST payload
type bonus struct {
	Amount        uint   `json:"amount"`
	Reason        string `json:"reason"`
	ReceiverEmail string `json:"receiver_email"`
}

const (
	server   = "https://bonus.ly/api/v1"
	resource = "bonuses"
)

var (
	token  = flag.String("token", "", "Bonus.ly access token")
	reason = flag.String("reason", "", "reason. e.g. for nyancats #mateship")
	email  = flag.String("email", "", "email address of recipient")
	points = flag.Int("points", 0, "number of points to give")
)

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Give bonus.ly points, one at a time ;)\n")
		flag.PrintDefaults()

		fmt.Fprintln(os.Stderr, "\nUsage:")
		fmt.Fprintln(os.Stderr, "    bonusly -points 10 -email a@b.com -reason \"for the lulz #mateship\"")
	}
}

// validate user input, roughly.
func validate() bool {
	return *points > 0 && len(*email) > 0 && len(*reason) > 0 && len(*token) > 0
}

// give points
//
// This implementation is only called as a goroutine from main() so rather
// than making a nice api friendly method the method was designed to
// accept preconstructed values which would be used repeatedly.
//
// url : the url to POST to
// points : the number of points to give
// payload : the encoded payload
func give(url *string, points int, payload *[]byte) error {
	buf := bytes.NewBuffer(*payload)
	resp, err := http.Post(*url, "application/json", buf)

	if resp != nil {
		defer resp.Body.Close()
	}

	if err != nil {
		return fmt.Errorf("Post failed : %+v", err)
	}

	body, err := ioutil.ReadAll(resp.Body)

	var response map[string]interface{}

	if err = json.Unmarshal(body, &response); err != nil {
		return fmt.Errorf("Unable to unmarshal response")
	}

	if response["success"] != true {
		return fmt.Errorf("%v", response["message"])
	}

	return nil
}

func main() {
	flag.Parse()

	if !validate() {
		flag.Usage()
		os.Exit(1)
	}

	// create the http parameters
	params := fmt.Sprintf("access_token=%v", *token)

	// create the Bonus message, which will be the payload
	bonus := &bonus{
		Amount:        1,
		Reason:        *reason,
		ReceiverEmail: *email,
	}

	// create the url
	url := fmt.Sprintf("%v/%v?%v", server, resource, params)

	var payload []byte
	var err error

	if payload, err = json.Marshal(&bonus); err != nil {
		fmt.Printf("%v", err)
		os.Exit(2)
	}

	// we need to wait for the goroutines to finish
	var wg sync.WaitGroup

	// set the number of goroutines to wait for
	wg.Add(*points)

	// give, in parallel
	for i := 0; i < *points; i++ {
		go func() {
			defer wg.Done()
			if err := give(&url, 1, &payload); err != nil {
				fmt.Printf("%v\n", err)
			}
		}()
	}

	// don't exit until the goroutines have finished
	wg.Wait()
}
<commit_msg>Minor reorganise.<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"sync"
)

// bonus is serialised to a POST payload
type bonus struct {
	Amount        uint   `json:"amount"`
	Reason        string `json:"reason"`
	ReceiverEmail string `json:"receiver_email"`
}

const (
	server   = "https://bonus.ly/api/v1"
	resource = "bonuses"
)

var (
	token  = flag.String("token", "", "Bonus.ly access token")
	reason = flag.String("reason", "", "reason. e.g. for nyancats #mateship")
	email  = flag.String("email", "", "email address of recipient")
	points = flag.Int("points", 0, "number of points to give")
)

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Give bonus.ly points, one at a time ;)\n")
		flag.PrintDefaults()

		fmt.Fprintln(os.Stderr, "\nUsage:")
		fmt.Fprintln(os.Stderr, "    bonusly -points 10 -email a@b.com -reason \"for the lulz #mateship\"")
	}
}

// validate user input, roughly.
func validate() bool {
	return *points > 0 && len(*email) > 0 && len(*reason) > 0 && len(*token) > 0
}

// give points
//
// This implementation is only called as a goroutine from main() so rather
// than making a nice api friendly method the method was designed to
// accept preconstructed values which would be used repeatedly.
//
// url : the url to POST to
// points : the number of points to give
// payload : the encoded payload
func give(url *string, points int, payload *[]byte) error {
	buf := bytes.NewBuffer(*payload)
	resp, err := http.Post(*url, "application/json", buf)

	if resp != nil {
		defer resp.Body.Close()
	}

	if err != nil {
		return fmt.Errorf("Post failed : %+v", err)
	}

	body, err := ioutil.ReadAll(resp.Body)

	var response map[string]interface{}

	if err = json.Unmarshal(body, &response); err != nil {
		return fmt.Errorf("Unable to unmarshal response")
	}

	if response["success"] != true {
		return fmt.Errorf("%v", response["message"])
	}

	return nil
}

func main() {
	flag.Parse()

	if !validate() {
		flag.Usage()
		os.Exit(1)
	}

	// create the Bonus message, which will be the payload
	bonus := &bonus{
		Amount:        1,
		Reason:        *reason,
		ReceiverEmail: *email,
	}

	var payload []byte
	var err error

	if payload, err = json.Marshal(&bonus); err != nil {
		fmt.Printf("%v", err)
		os.Exit(2)
	}

	// create the http parameters
	params := fmt.Sprintf("access_token=%v", *token)

	// create the url
	url := fmt.Sprintf("%v/%v?%v", server, resource, params)

	// we need to wait for the goroutines to finish
	var wg sync.WaitGroup

	// set the number of goroutines to wait for
	wg.Add(*points)

	// give, in parallel
	for i := 0; i < *points; i++ {
		go func() {
			defer wg.Done()
			if err := give(&url, 1, &payload); err != nil {
				fmt.Printf("%v\n", err)
			}
		}()
	}

	// don't exit until the goroutines have finished
	wg.Wait()
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha1"
	"crypto/subtle"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/codegangsta/cli"
	"golang.org/x/net/websocket"
)

func main() {
	app := cli.NewApp()
	app.Usage = "turn webhooks into websockets"

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "bind, b",
			Value: ":8080",
			Usage: "address to listen on",
		},
	}

	app.Action = ActionMain

	app.RunAndExitOnError()
}

func ActionMain(c *cli.Context) {
	var (
		key           = os.Getenv("HOOKBOT_KEY")
		github_secret = os.Getenv("HOOKBOT_GITHUB_SECRET")
	)

	if key == "" || github_secret == "" {
		log.Fatalln("Error: HOOKBOT_KEY or HOOKBOT_GITHUB_SECRET not set")
	}

	hookbot := NewHookbot(key, github_secret)
	http.Handle("/", hookbot)
	http.HandleFunc("/status", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "OK")
	})

	log.Println("Listening on", c.String("bind"))
	err := http.ListenAndServe(c.String("bind"), nil)
	if err != nil {
		log.Fatal(err)
	}
}

type Message struct {
	Topic string
	Body  []byte
}

type Listener struct {
	Topic string
	c     chan []byte
}

type Hookbot struct {
	key, github_secret string

	http.Handler

	message                  chan Message
	addListener, delListener chan Listener
}

func NewHookbot(key, github_secret string) *Hookbot {
	h := &Hookbot{
		key: key, github_secret: github_secret,

		message:     make(chan Message, 1),
		addListener: make(chan Listener, 1),
		delListener: make(chan Listener, 1),
	}

	mux := http.NewServeMux()
	mux.Handle("/sub/", websocket.Handler(h.ServeSubscribe))
	mux.HandleFunc("/pub/", h.ServePublish)

	// Middlewares
	h.Handler = mux
	h.Handler = h.KeyChecker(h.Handler)

	go h.Loop()

	return h
}

var timeout = 1 * time.Second

func TimeoutSend(c chan []byte, m []byte) {
	select {
	case c <- m:
	case <-time.After(timeout):
	}
}

// Manage fanout from h.message onto listeners
func (h *Hookbot) Loop() {
	listeners := map[Listener]struct{}{}
	for {
		select {
		case m := <-h.message:
			for listener := range listeners {
				if listener.Topic == m.Topic {
					go TimeoutSend(listener.c, m.Body)
				}
			}

		case l := <-h.addListener:
			listeners[l] = struct{}{}
		case l := <-h.delListener:
			delete(listeners, l)
		}
	}
}

func (h *Hookbot) Add(topic string) Listener {
	l := Listener{Topic: topic, c: make(chan []byte)}
	h.addListener <- l
	return l
}

func (h *Hookbot) Del(l Listener) {
	h.delListener <- l
}

func SecureEqual(x, y string) bool {
	if subtle.ConstantTimeCompare([]byte(x), []byte(y)) == 1 {
		return true
	}
	return false
}

func (h *Hookbot) IsGithubKeyOK(w http.ResponseWriter, r *http.Request) bool {
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, "Not Authorized", http.StatusUnauthorized)
	}

	r.Body = ioutil.NopCloser(bytes.NewReader(body))

	mac := hmac.New(sha1.New, []byte(h.github_secret))
	mac.Reset()
	mac.Write(body)

	signature := fmt.Sprintf("sha1=%x", mac.Sum(nil))

	return SecureEqual(r.Header.Get("X-Hub-Signature"), signature)
}

func (h *Hookbot) IsKeyOK(w http.ResponseWriter, r *http.Request) bool {

	if _, ok := r.Header["X-Hub-Signature"]; ok {
		if !h.IsGithubKeyOK(w, r) {
			http.NotFound(w, r)
			return false
		}
		return true
	}

	lhs := r.Header.Get("Authorization")
	rhs := fmt.Sprintf("Bearer %v", h.key)

	if !SecureEqual(lhs, rhs) {
		http.NotFound(w, r)
		return false
	}

	return true
}

func (h *Hookbot) KeyChecker(wrapped http.Handler) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if !h.IsKeyOK(w, r) {
			return
		}

		wrapped.ServeHTTP(w, r)
	}
}

func Topic(url *url.URL) string {
	// The topic is everything after the "/pub/" or "/sub/"
	p := strings.SplitN(url.Path, "/", 3)
	topic := p[2]
        return topic
}

func (h *Hookbot) ServePublish(w http.ResponseWriter, r *http.Request) {
	body, err := ioutil.ReadAll(r.Body)

	if err != nil {
		log.Printf("Error serving %v: %v", r.URL, err)
		return
	}

	topic := Topic(r.URL)
	h.message <- Message{Topic: topic, Body: body}
}

func (h *Hookbot) ServeSubscribe(conn *websocket.Conn) {

	topic := Topic(conn.Request().URL)

	listener := h.Add(topic)
	defer h.Del(listener)

	closed := make(chan struct{})

	go func() {
		defer close(closed)
		_, _ = io.Copy(ioutil.Discard, conn)
	}()

	var message []byte

	for {
		select {
		case message = <-listener.c:
		case <-closed:
			log.Printf("Client disconnected")
			return
		}

		conn.SetWriteDeadline(time.Now().Add(90 * time.Second))
		n, err := conn.Write(message)
		switch {
		case n != len(message):
			log.Printf("Short write %d != %d", n, len(message))
			return // short write
		case err == io.EOF:
			return // done
		case err != nil:
			log.Printf("Error in conn.Write: %v", err)
			return // unknown error
		}
	}
}
<commit_msg>Use a Regexp instead of SplitN<commit_after>package main

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha1"
	"crypto/subtle"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"time"

	"github.com/codegangsta/cli"
	"golang.org/x/net/websocket"
)

func main() {
	app := cli.NewApp()
	app.Usage = "turn webhooks into websockets"

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "bind, b",
			Value: ":8080",
			Usage: "address to listen on",
		},
	}

	app.Action = ActionMain

	app.RunAndExitOnError()
}

func ActionMain(c *cli.Context) {
	var (
		key           = os.Getenv("HOOKBOT_KEY")
		github_secret = os.Getenv("HOOKBOT_GITHUB_SECRET")
	)

	if key == "" || github_secret == "" {
		log.Fatalln("Error: HOOKBOT_KEY or HOOKBOT_GITHUB_SECRET not set")
	}

	hookbot := NewHookbot(key, github_secret)
	http.Handle("/", hookbot)
	http.HandleFunc("/status", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "OK")
	})

	log.Println("Listening on", c.String("bind"))
	err := http.ListenAndServe(c.String("bind"), nil)
	if err != nil {
		log.Fatal(err)
	}
}

type Message struct {
	Topic string
	Body  []byte
}

type Listener struct {
	Topic string
	c     chan []byte
}

type Hookbot struct {
	key, github_secret string

	http.Handler

	message                  chan Message
	addListener, delListener chan Listener
}

func NewHookbot(key, github_secret string) *Hookbot {
	h := &Hookbot{
		key: key, github_secret: github_secret,

		message:     make(chan Message, 1),
		addListener: make(chan Listener, 1),
		delListener: make(chan Listener, 1),
	}

	mux := http.NewServeMux()
	mux.Handle("/sub/", websocket.Handler(h.ServeSubscribe))
	mux.HandleFunc("/pub/", h.ServePublish)

	// Middlewares
	h.Handler = mux
	h.Handler = h.KeyChecker(h.Handler)

	go h.Loop()

	return h
}

var timeout = 1 * time.Second

func TimeoutSend(c chan []byte, m []byte) {
	select {
	case c <- m:
	case <-time.After(timeout):
	}
}

// Manage fanout from h.message onto listeners
func (h *Hookbot) Loop() {
	listeners := map[Listener]struct{}{}
	for {
		select {
		case m := <-h.message:
			for listener := range listeners {
				if listener.Topic == m.Topic {
					go TimeoutSend(listener.c, m.Body)
				}
			}

		case l := <-h.addListener:
			listeners[l] = struct{}{}
		case l := <-h.delListener:
			delete(listeners, l)
		}
	}
}

func (h *Hookbot) Add(topic string) Listener {
	l := Listener{Topic: topic, c: make(chan []byte)}
	h.addListener <- l
	return l
}

func (h *Hookbot) Del(l Listener) {
	h.delListener <- l
}

func SecureEqual(x, y string) bool {
	if subtle.ConstantTimeCompare([]byte(x), []byte(y)) == 1 {
		return true
	}
	return false
}

func (h *Hookbot) IsGithubKeyOK(w http.ResponseWriter, r *http.Request) bool {
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, "Not Authorized", http.StatusUnauthorized)
	}

	r.Body = ioutil.NopCloser(bytes.NewReader(body))

	mac := hmac.New(sha1.New, []byte(h.github_secret))
	mac.Reset()
	mac.Write(body)

	signature := fmt.Sprintf("sha1=%x", mac.Sum(nil))

	return SecureEqual(r.Header.Get("X-Hub-Signature"), signature)
}

func (h *Hookbot) IsKeyOK(w http.ResponseWriter, r *http.Request) bool {

	if _, ok := r.Header["X-Hub-Signature"]; ok {
		if !h.IsGithubKeyOK(w, r) {
			http.NotFound(w, r)
			return false
		}
		return true
	}

	lhs := r.Header.Get("Authorization")
	rhs := fmt.Sprintf("Bearer %v", h.key)

	if !SecureEqual(lhs, rhs) {
		http.NotFound(w, r)
		return false
	}

	return true
}

func (h *Hookbot) KeyChecker(wrapped http.Handler) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if !h.IsKeyOK(w, r) {
			return
		}

		wrapped.ServeHTTP(w, r)
	}
}

// The topic is everything after the "/pub/" or "/sub/"
var TopicRE *regexp.Regexp = regexp.MustCompile("/[^/]+/(.*)")

func Topic(url *url.URL) string {
	m := TopicRE.FindStringSubmatch(url.Path)
	if m == nil {
		return ""
	}
	return m[1]
}

func (h *Hookbot) ServePublish(w http.ResponseWriter, r *http.Request) {
	body, err := ioutil.ReadAll(r.Body)

	if err != nil {
		log.Printf("Error serving %v: %v", r.URL, err)
		return
	}

	topic := Topic(r.URL)
	h.message <- Message{Topic: topic, Body: body}
}

func (h *Hookbot) ServeSubscribe(conn *websocket.Conn) {

	topic := Topic(conn.Request().URL)

	listener := h.Add(topic)
	defer h.Del(listener)

	closed := make(chan struct{})

	go func() {
		defer close(closed)
		_, _ = io.Copy(ioutil.Discard, conn)
	}()

	var message []byte

	for {
		select {
		case message = <-listener.c:
		case <-closed:
			log.Printf("Client disconnected")
			return
		}

		conn.SetWriteDeadline(time.Now().Add(90 * time.Second))
		n, err := conn.Write(message)
		switch {
		case n != len(message):
			log.Printf("Short write %d != %d", n, len(message))
			return // short write
		case err == io.EOF:
			return // done
		case err != nil:
			log.Printf("Error in conn.Write: %v", err)
			return // unknown error
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"runtime"
	"sort"
	"strings"
	"sync/atomic"
	"time"
)

const (
	helloVersion = "0.2"
)

var knownPaths []string
var boottime time.Time
var banner string
var requests int64

func inc() int64 {
	return atomic.AddInt64(&requests, 1)
}

func get() int64 {
	return atomic.LoadInt64(&requests)
}

func main() {

	boottime = time.Now()

	tls := true

	log.Print("version: ", helloVersion)
	log.Print("runtime: ", runtime.Version())

	currDir, err := os.Getwd()
	if err != nil {
		log.Fatalf("Getwd: %s", err)
	}

	var addr, httpsAddr, key, cert string
	var disableKeepalive bool

	flag.StringVar(&key, "key", "key.pem", "TLS key file")
	flag.StringVar(&cert, "cert", "cert.pem", "TLS cert file")
	flag.StringVar(&addr, "addr", ":8080", "HTTP listen address")
	flag.StringVar(&httpsAddr, "httpsAddr", ":8443", "HTTPS listen address")
	flag.StringVar(&banner, "banner", "deploy #4", "banner will be displayed")
	flag.BoolVar(&disableKeepalive, "disableKeepalive", false, "disable keepalive")
	flag.Parse()

	keepalive := !disableKeepalive

	log.Print("banner: ", banner)
	log.Print("keepalive: ", keepalive)

	if !fileExists(key) {
		log.Printf("TLS key file not found: %s - disabling TLS", key)
		tls = false
	}

	if !fileExists(cert) {
		log.Printf("TLS cert file not found: %s - disabling TLS", cert)
		tls = false
	}

	// default handler
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { rootHandler(w, r, keepalive) })

	registerStatic("/www/", currDir)

	log.Printf("using TCP ports HTTP=%s HTTPS=%s TLS=%v", addr, httpsAddr, tls)

	if tls {
		serveHTTPS(addr, httpsAddr, cert, key, keepalive)
		return
	}

	log.Printf("serving HTTP on TCP %s", addr)
	if err := listenAndServe(addr, nil, keepalive); err != nil {
		log.Fatalf("listenAndServe: %s: %v", addr, err)
	}
}

func serveHTTPS(addr, httpsAddr, cert, key string, keepalive bool) {
	httpPort := "80"
	h := strings.Split(addr, ":")
	if len(h) > 1 {
		httpPort = h[1]
	}

	httpsPort := "443"
	hs := strings.Split(httpsAddr, ":")
	if len(hs) > 1 {
		httpsPort = hs[1]
	}

	if httpPort != httpsPort {
		log.Printf("installing redirect from HTTP=%s to HTTPS=%s", addr, httpsPort)

		redirectTLS := func(w http.ResponseWriter, r *http.Request) {
			host := strings.Split(r.Host, ":")[0]
			//code := http.StatusMovedPermanently // 301 permanent
			code := http.StatusFound // 302 temporary
			http.Redirect(w, r, "https://"+host+":"+httpsPort+r.RequestURI, code)
		}

		// http-to-https redirect server
		go func() {
			if err := listenAndServe(addr, http.HandlerFunc(redirectTLS), keepalive); err != nil {
				log.Fatalf("redirect: listenAndServe: %s: %v", addr, err)
			}
		}()
	}

	log.Printf("serving HTTPS on TCP %s", httpsAddr)
	if err := listenAndServeTLS(httpsAddr, cert, key, nil, keepalive); err != nil {
		log.Fatalf("listenAndServeTLS: %s: %v", httpsAddr, err)
	}
}

func listenAndServe(addr string, handler http.Handler, keepalive bool) error {
	server := &http.Server{Addr: addr, Handler: handler}
	server.SetKeepAlivesEnabled(keepalive)
	return server.ListenAndServe()
}

func listenAndServeTLS(addr, certFile, keyFile string, handler http.Handler, keepalive bool) error {
	server := &http.Server{Addr: addr, Handler: handler}
	server.SetKeepAlivesEnabled(keepalive)
	return server.ListenAndServeTLS(certFile, keyFile)
}

func fileExists(path string) bool {
	_, err := os.Stat(path)
	return err == nil
}

type staticHandler struct {
	innerHandler http.Handler
}

func registerStatic(path, dir string) {
	http.Handle(path, staticHandler{http.StripPrefix(path, http.FileServer(http.Dir(dir)))})
	knownPaths = append(knownPaths, path)
	log.Printf("mapping www path %s to directory %s", path, dir)
}

func (handler staticHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	count := inc()
	log.Printf("staticHandler.ServeHTTP req=%d url=%s from=%s", count, r.URL.Path, r.RemoteAddr)
	handler.innerHandler.ServeHTTP(w, r)
}

func rootHandler(w http.ResponseWriter, r *http.Request, keepalive bool) {
	count := inc()
	msg := fmt.Sprintf("rootHandler: req=%d url=%s from=%s", count, r.URL.Path, r.RemoteAddr)
	log.Print(msg)

	var paths string
	for _, p := range knownPaths {
		paths += fmt.Sprintf("%s
", p, p)
	}

	var errMsg string
	if r.URL.Path != "/" {
		errMsg = fmt.Sprintf("
Path not found!
Path not found: [%s]", r.URL.Path)
	}

	header :=
		`<!DOCTYPE html>

    <title>gowebhello root page</title>

gowebhello root page

    https://github.com/udhos/gowebhello is a simple golang replacement for 'python -m SimpleHTTPServer'.

`
	bodyTempl :=
		`
Welcome!

	gowebhello version %s runtime %s

	Application banner: %s

	Application arguments: %v

	Application dir: %s

	Process: %d

	Server hostname: %s

	Your address: %s

	Current time: %s

	Uptime: %s

	Requests: %d

    %s

All known paths:

    %s
`

	footer :=
		`

`

	cwd, errCwd := os.Getwd()
	if errCwd != nil {
		cwd = cwd + " (error: " + errCwd.Error() + ")"
	}

	host, errHost := os.Hostname()
	if errHost != nil {
		host = host + " (error: " + errHost.Error() + ")"
	}

	now := time.Now()

	body := fmt.Sprintf(bodyTempl, helloVersion, runtime.Version(), banner, os.Args, cwd, os.Getpid(), host, r.RemoteAddr, now, time.Since(boottime), get(), errMsg, paths)

	if !keepalive {
		w.Header().Set("Connection", "close")
	}

	io.WriteString(w, header)
	io.WriteString(w, body)
	showHeaders(w, r)
	io.WriteString(w, footer)
}

func showHeaders(w http.ResponseWriter, r *http.Request) {
	io.WriteString(w, "
Headers
\n")

	var headers []string
	for k, v := range r.Header {
		for _, vv := range v {
			headers = append(headers, k+": "+vv+"
\n")
		}
	}

	sort.Strings(headers)

	for _, h := range headers {
		io.WriteString(w, h)
	}
}
<commit_msg>show pid on the console<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"runtime"
	"sort"
	"strings"
	"sync/atomic"
	"time"
)

const (
	helloVersion = "0.2"
)

var knownPaths []string
var boottime time.Time
var banner string
var requests int64

func inc() int64 {
	return atomic.AddInt64(&requests, 1)
}

func get() int64 {
	return atomic.LoadInt64(&requests)
}

func main() {

	boottime = time.Now()

	tls := true

	log.Print("version: ", helloVersion)
	log.Print("runtime: ", runtime.Version())
	log.Print("pid: ", os.Getpid())

	currDir, err := os.Getwd()
	if err != nil {
		log.Fatalf("Getwd: %s", err)
	}

	var addr, httpsAddr, key, cert string
	var disableKeepalive bool

	flag.StringVar(&key, "key", "key.pem", "TLS key file")
	flag.StringVar(&cert, "cert", "cert.pem", "TLS cert file")
	flag.StringVar(&addr, "addr", ":8080", "HTTP listen address")
	flag.StringVar(&httpsAddr, "httpsAddr", ":8443", "HTTPS listen address")
	flag.StringVar(&banner, "banner", "deploy #4", "banner will be displayed")
	flag.BoolVar(&disableKeepalive, "disableKeepalive", false, "disable keepalive")
	flag.Parse()

	keepalive := !disableKeepalive

	log.Print("banner: ", banner)
	log.Print("keepalive: ", keepalive)

	if !fileExists(key) {
		log.Printf("TLS key file not found: %s - disabling TLS", key)
		tls = false
	}

	if !fileExists(cert) {
		log.Printf("TLS cert file not found: %s - disabling TLS", cert)
		tls = false
	}

	// default handler
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { rootHandler(w, r, keepalive) })

	registerStatic("/www/", currDir)

	log.Printf("using TCP ports HTTP=%s HTTPS=%s TLS=%v", addr, httpsAddr, tls)

	if tls {
		serveHTTPS(addr, httpsAddr, cert, key, keepalive)
		return
	}

	log.Printf("serving HTTP on TCP %s", addr)
	if err := listenAndServe(addr, nil, keepalive); err != nil {
		log.Fatalf("listenAndServe: %s: %v", addr, err)
	}
}

func serveHTTPS(addr, httpsAddr, cert, key string, keepalive bool) {
	httpPort := "80"
	h := strings.Split(addr, ":")
	if len(h) > 1 {
		httpPort = h[1]
	}

	httpsPort := "443"
	hs := strings.Split(httpsAddr, ":")
	if len(hs) > 1 {
		httpsPort = hs[1]
	}

	if httpPort != httpsPort {
		log.Printf("installing redirect from HTTP=%s to HTTPS=%s", addr, httpsPort)

		redirectTLS := func(w http.ResponseWriter, r *http.Request) {
			host := strings.Split(r.Host, ":")[0]
			//code := http.StatusMovedPermanently // 301 permanent
			code := http.StatusFound // 302 temporary
			http.Redirect(w, r, "https://"+host+":"+httpsPort+r.RequestURI, code)
		}

		// http-to-https redirect server
		go func() {
			if err := listenAndServe(addr, http.HandlerFunc(redirectTLS), keepalive); err != nil {
				log.Fatalf("redirect: listenAndServe: %s: %v", addr, err)
			}
		}()
	}

	log.Printf("serving HTTPS on TCP %s", httpsAddr)
	if err := listenAndServeTLS(httpsAddr, cert, key, nil, keepalive); err != nil {
		log.Fatalf("listenAndServeTLS: %s: %v", httpsAddr, err)
	}
}

func listenAndServe(addr string, handler http.Handler, keepalive bool) error {
	server := &http.Server{Addr: addr, Handler: handler}
	server.SetKeepAlivesEnabled(keepalive)
	return server.ListenAndServe()
}

func listenAndServeTLS(addr, certFile, keyFile string, handler http.Handler, keepalive bool) error {
	server := &http.Server{Addr: addr, Handler: handler}
	server.SetKeepAlivesEnabled(keepalive)
	return server.ListenAndServeTLS(certFile, keyFile)
}

func fileExists(path string) bool {
	_, err := os.Stat(path)
	return err == nil
}

type staticHandler struct {
	innerHandler http.Handler
}

func registerStatic(path, dir string) {
	http.Handle(path, staticHandler{http.StripPrefix(path, http.FileServer(http.Dir(dir)))})
	knownPaths = append(knownPaths, path)
	log.Printf("mapping www path %s to directory %s", path, dir)
}

func (handler staticHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	count := inc()
	log.Printf("staticHandler.ServeHTTP req=%d url=%s from=%s", count, r.URL.Path, r.RemoteAddr)
	handler.innerHandler.ServeHTTP(w, r)
}

func rootHandler(w http.ResponseWriter, r *http.Request, keepalive bool) {
	count := inc()
	msg := fmt.Sprintf("rootHandler: req=%d url=%s from=%s", count, r.URL.Path, r.RemoteAddr)
	log.Print(msg)

	var paths string
	for _, p := range knownPaths {
		paths += fmt.Sprintf("%s
", p, p)
	}

	var errMsg string
	if r.URL.Path != "/" {
		errMsg = fmt.Sprintf("
Path not found!
Path not found: [%s]", r.URL.Path)
	}

	header :=
		`<!DOCTYPE html>

    <title>gowebhello root page</title>

gowebhello root page

    https://github.com/udhos/gowebhello is a simple golang replacement for 'python -m SimpleHTTPServer'.

`
	bodyTempl :=
		`
Welcome!

	gowebhello version %s runtime %s

	Application banner: %s

	Application arguments: %v

	Application dir: %s

	Process: %d

	Server hostname: %s

	Your address: %s

	Current time: %s

	Uptime: %s

	Requests: %d

    %s

All known paths:

    %s
`

	footer :=
		`

`

	cwd, errCwd := os.Getwd()
	if errCwd != nil {
		cwd = cwd + " (error: " + errCwd.Error() + ")"
	}

	host, errHost := os.Hostname()
	if errHost != nil {
		host = host + " (error: " + errHost.Error() + ")"
	}

	now := time.Now()

	body := fmt.Sprintf(bodyTempl, helloVersion, runtime.Version(), banner, os.Args, cwd, os.Getpid(), host, r.RemoteAddr, now, time.Since(boottime), get(), errMsg, paths)

	if !keepalive {
		w.Header().Set("Connection", "close")
	}

	io.WriteString(w, header)
	io.WriteString(w, body)
	showHeaders(w, r)
	io.WriteString(w, footer)
}

func showHeaders(w http.ResponseWriter, r *http.Request) {
	io.WriteString(w, "
Headers
\n")

	var headers []string
	for k, v := range r.Header {
		for _, vv := range v {
			headers = append(headers, k+": "+vv+"
\n")
		}
	}

	sort.Strings(headers)

	for _, h := range headers {
		io.WriteString(w, h)
	}
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2013 Matt Jibson <matt.jibson@gmail.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package goapp

import (
	"encoding/json"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"net/url"
	"time"

	"github.com/mjibson/goread/_third_party/github.com/MiniProfiler/go/miniprofiler"
	mpg "github.com/mjibson/goread/_third_party/github.com/MiniProfiler/go/miniprofiler_gae"
	"github.com/mjibson/goread/_third_party/github.com/gorilla/mux"
	"github.com/mjibson/goread/_third_party/github.com/mjibson/goon"

	"appengine"
	"appengine/datastore"
)

var router = new(mux.Router)
var templates *template.Template

func init() {
	var err error
	if templates, err = template.New("").Funcs(funcs).
		ParseFiles(
		"templates/base.html",
		"templates/admin-all-feeds.html",
		"templates/admin-date-formats.html",
		"templates/admin-feed.html",
		"templates/admin-stats.html",
		"templates/admin-user.html",
	); err != nil {
		log.Fatal(err)
	}

	miniprofiler.ToggleShortcut = "Alt+C"
	miniprofiler.Position = "bottomleft"
}

func RegisterHandlers(r *mux.Router) {
	router = r
	router.Handle("/", mpg.NewHandler(Main)).Name("main")
	router.Handle("/login/google", mpg.NewHandler(LoginGoogle)).Name("login-google")
	router.Handle("/logout", mpg.NewHandler(Logout)).Name("logout")
	router.Handle("/push", mpg.NewHandler(SubscribeCallback)).Name("subscribe-callback")
	router.Handle("/tasks/import-opml", mpg.NewHandler(ImportOpmlTask)).Name("import-opml-task")
	router.Handle("/tasks/subscribe-feed", mpg.NewHandler(SubscribeFeed)).Name("subscribe-feed")
	router.Handle("/tasks/update-feed-last", mpg.NewHandler(UpdateFeedLast)).Name("update-feed-last")
	router.Handle("/tasks/update-feed-manual", mpg.NewHandler(UpdateFeed)).Name("update-feed-manual")
	router.Handle("/tasks/update-feed", mpg.NewHandler(UpdateFeed)).Name("update-feed")
	router.Handle("/tasks/update-feeds", mpg.NewHandler(UpdateFeeds)).Name("update-feeds")
	router.Handle("/tasks/delete-old-feeds", mpg.NewHandler(DeleteOldFeeds)).Name("delete-old-feeds")
	router.Handle("/tasks/delete-old-feed", mpg.NewHandler(DeleteOldFeed)).Name("delete-old-feed")
	router.Handle("/user/add-subscription", mpg.NewHandler(AddSubscription)).Name("add-subscription")
	router.Handle("/user/delete-account", mpg.NewHandler(DeleteAccount)).Name("delete-account")
	router.Handle("/user/export-opml", mpg.NewHandler(ExportOpml)).Name("export-opml")
	router.Handle("/user/feed-history", mpg.NewHandler(FeedHistory)).Name("feed-history")
	router.Handle("/user/get-contents", mpg.NewHandler(GetContents)).Name("get-contents")
	router.Handle("/user/get-feed", mpg.NewHandler(GetFeed)).Name("get-feed")
	router.Handle("/user/get-stars", mpg.NewHandler(GetStars)).Name("get-stars")
	router.Handle("/user/import/get-url", mpg.NewHandler(UploadUrl)).Name("upload-url")
	router.Handle("/user/import/opml", mpg.NewHandler(ImportOpml)).Name("import-opml")
	router.Handle("/user/list-feeds", mpg.NewHandler(ListFeeds)).Name("list-feeds")
	router.Handle("/user/mark-read", mpg.NewHandler(MarkRead)).Name("mark-read")
	router.Handle("/user/mark-unread", mpg.NewHandler(MarkUnread)).Name("mark-unread")
	router.Handle("/user/save-options", mpg.NewHandler(SaveOptions)).Name("save-options")
	router.Handle("/user/set-star", mpg.NewHandler(SetStar)).Name("set-star")
	router.Handle("/user/upload-opml", mpg.NewHandler(UploadOpml)).Name("upload-opml")

	router.Handle("/admin/all-feeds", mpg.NewHandler(AllFeeds)).Name("all-feeds")
	router.Handle("/admin/all-feeds-opml", mpg.NewHandler(AllFeedsOpml)).Name("all-feeds-opml")
	router.Handle("/admin/user", mpg.NewHandler(AdminUser)).Name("admin-user")
	router.Handle("/date-formats", mpg.NewHandler(AdminDateFormats)).Name("admin-date-formats")
	router.Handle("/admin/feed", mpg.NewHandler(AdminFeed)).Name("admin-feed")
	router.Handle("/admin/subhub", mpg.NewHandler(AdminSubHub)).Name("admin-subhub-feed")
	router.Handle("/admin/stats", mpg.NewHandler(AdminStats)).Name("admin-stats")
	router.Handle("/admin/update-feed", mpg.NewHandler(AdminUpdateFeed)).Name("admin-update-feed")
	router.Handle("/user/charge", mpg.NewHandler(Charge)).Name("charge")
	router.Handle("/user/account", mpg.NewHandler(Account)).Name("account")
	router.Handle("/user/uncheckout", mpg.NewHandler(Uncheckout)).Name("uncheckout")

	//router.Handle("/tasks/delete-blobs", mpg.NewHandler(DeleteBlobs)).Name("delete-blobs")

	if len(PUBSUBHUBBUB_HOST) > 0 {
		u := url.URL{
			Scheme:   "http",
			Host:     PUBSUBHUBBUB_HOST,
			Path:     routeUrl("add-subscription"),
			RawQuery: url.Values{"url": {"{url}"}}.Encode(),
		}
		subURL = u.String()
	}

	if !isDevServer {
		return
	}
	router.Handle("/user/clear-feeds", mpg.NewHandler(ClearFeeds)).Name("clear-feeds")
	router.Handle("/user/clear-read", mpg.NewHandler(ClearRead)).Name("clear-read")
	router.Handle("/test/atom.xml", mpg.NewHandler(TestAtom)).Name("test-atom")
}

func Main(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	if err := templates.ExecuteTemplate(w, "base.html", includes(c, w, r)); err != nil {
		c.Errorf("%v", err)
		serveError(w, err)
	}
	return
}

func addFeed(c mpg.Context, userid string, outline *OpmlOutline) error {
	gn := goon.FromContext(c)
	o := outline.Outline[0]
	c.Infof("adding feed %v to user %s", o.XmlUrl, userid)
	fu, ferr := url.Parse(o.XmlUrl)
	if ferr != nil {
		return ferr
	}
	fu.Fragment = ""
	o.XmlUrl = fu.String()

	f := Feed{Url: o.XmlUrl}
	if err := gn.Get(&f); err == datastore.ErrNoSuchEntity {
		if feed, stories, err := fetchFeed(c, o.XmlUrl, o.XmlUrl); err != nil {
			return fmt.Errorf("could not add feed %s: %v", o.XmlUrl, err)
		} else {
			f = *feed
			f.Updated = time.Time{}
			f.Checked = f.Updated
			f.NextUpdate = f.Updated
			f.LastViewed = time.Now()
			gn.Put(&f)
			for _, s := range stories {
				s.Created = s.Published
			}
			if err := updateFeed(c, f.Url, feed, stories, false, false, false); err != nil {
				return err
			}

			o.XmlUrl = feed.Url
			o.HtmlUrl = feed.Link
			if o.Title == "" {
				o.Title = feed.Title
			}
		}
	} else if err != nil {
		return err
	} else {
		o.HtmlUrl = f.Link
		if o.Title == "" {
			o.Title = f.Title
		}
	}
	o.Text = ""

	return nil
}

func mergeUserOpml(c appengine.Context, ud *UserData, outlines ...*OpmlOutline) error {
	var fs Opml
	json.Unmarshal(ud.Opml, &fs)
	urls := make(map[string]bool)

	for _, o := range fs.Outline {
		if o.XmlUrl != "" {
			urls[o.XmlUrl] = true
		} else {
			for _, so := range o.Outline {
				urls[so.XmlUrl] = true
			}
		}
	}

	mergeOutline := func(label string, outline *OpmlOutline) {
		if _, present := urls[outline.XmlUrl]; present {
			return
		} else {
			urls[outline.XmlUrl] = true

			if label == "" {
				fs.Outline = append(fs.Outline, outline)
			} else {
				done := false
				for _, ol := range fs.Outline {
					if ol.Title == label && ol.XmlUrl == "" {
						ol.Outline = append(ol.Outline, outline)
						done = true
						break
					}
				}
				if !done {
					fs.Outline = append(fs.Outline, &OpmlOutline{
						Title:   label,
						Outline: []*OpmlOutline{outline},
					})
				}
			}
		}
	}

	for _, outline := range outlines {
		if outline.XmlUrl != "" {
			mergeOutline("", outline)
		} else {
			for _, o := range outline.Outline {
				mergeOutline(outline.Title, o)
			}
		}
	}

	b, err := json.Marshal(&fs)
	if err != nil {
		return err
	}
	ud.Opml = b
	return nil
}
<commit_msg>gofmt<commit_after>/*
 * Copyright (c) 2013 Matt Jibson <matt.jibson@gmail.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package goapp

import (
	"encoding/json"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"net/url"
	"time"

	"github.com/mjibson/goread/_third_party/github.com/MiniProfiler/go/miniprofiler"
	mpg "github.com/mjibson/goread/_third_party/github.com/MiniProfiler/go/miniprofiler_gae"
	"github.com/mjibson/goread/_third_party/github.com/gorilla/mux"
	"github.com/mjibson/goread/_third_party/github.com/mjibson/goon"

	"appengine"
	"appengine/datastore"
)

var router = new(mux.Router)
var templates *template.Template

func init() {
	var err error
	if templates, err = template.New("").Funcs(funcs).
		ParseFiles(
			"templates/base.html",
			"templates/admin-all-feeds.html",
			"templates/admin-date-formats.html",
			"templates/admin-feed.html",
			"templates/admin-stats.html",
			"templates/admin-user.html",
		); err != nil {
		log.Fatal(err)
	}

	miniprofiler.ToggleShortcut = "Alt+C"
	miniprofiler.Position = "bottomleft"
}

func RegisterHandlers(r *mux.Router) {
	router = r
	router.Handle("/", mpg.NewHandler(Main)).Name("main")
	router.Handle("/login/google", mpg.NewHandler(LoginGoogle)).Name("login-google")
	router.Handle("/logout", mpg.NewHandler(Logout)).Name("logout")
	router.Handle("/push", mpg.NewHandler(SubscribeCallback)).Name("subscribe-callback")
	router.Handle("/tasks/import-opml", mpg.NewHandler(ImportOpmlTask)).Name("import-opml-task")
	router.Handle("/tasks/subscribe-feed", mpg.NewHandler(SubscribeFeed)).Name("subscribe-feed")
	router.Handle("/tasks/update-feed-last", mpg.NewHandler(UpdateFeedLast)).Name("update-feed-last")
	router.Handle("/tasks/update-feed-manual", mpg.NewHandler(UpdateFeed)).Name("update-feed-manual")
	router.Handle("/tasks/update-feed", mpg.NewHandler(UpdateFeed)).Name("update-feed")
	router.Handle("/tasks/update-feeds", mpg.NewHandler(UpdateFeeds)).Name("update-feeds")
	router.Handle("/tasks/delete-old-feeds", mpg.NewHandler(DeleteOldFeeds)).Name("delete-old-feeds")
	router.Handle("/tasks/delete-old-feed", mpg.NewHandler(DeleteOldFeed)).Name("delete-old-feed")
	router.Handle("/user/add-subscription", mpg.NewHandler(AddSubscription)).Name("add-subscription")
	router.Handle("/user/delete-account", mpg.NewHandler(DeleteAccount)).Name("delete-account")
	router.Handle("/user/export-opml", mpg.NewHandler(ExportOpml)).Name("export-opml")
	router.Handle("/user/feed-history", mpg.NewHandler(FeedHistory)).Name("feed-history")
	router.Handle("/user/get-contents", mpg.NewHandler(GetContents)).Name("get-contents")
	router.Handle("/user/get-feed", mpg.NewHandler(GetFeed)).Name("get-feed")
	router.Handle("/user/get-stars", mpg.NewHandler(GetStars)).Name("get-stars")
	router.Handle("/user/import/get-url", mpg.NewHandler(UploadUrl)).Name("upload-url")
	router.Handle("/user/import/opml", mpg.NewHandler(ImportOpml)).Name("import-opml")
	router.Handle("/user/list-feeds", mpg.NewHandler(ListFeeds)).Name("list-feeds")
	router.Handle("/user/mark-read", mpg.NewHandler(MarkRead)).Name("mark-read")
	router.Handle("/user/mark-unread", mpg.NewHandler(MarkUnread)).Name("mark-unread")
	router.Handle("/user/save-options", mpg.NewHandler(SaveOptions)).Name("save-options")
	router.Handle("/user/set-star", mpg.NewHandler(SetStar)).Name("set-star")
	router.Handle("/user/upload-opml", mpg.NewHandler(UploadOpml)).Name("upload-opml")

	router.Handle("/admin/all-feeds", mpg.NewHandler(AllFeeds)).Name("all-feeds")
	router.Handle("/admin/all-feeds-opml", mpg.NewHandler(AllFeedsOpml)).Name("all-feeds-opml")
	router.Handle("/admin/user", mpg.NewHandler(AdminUser)).Name("admin-user")
	router.Handle("/date-formats", mpg.NewHandler(AdminDateFormats)).Name("admin-date-formats")
	router.Handle("/admin/feed", mpg.NewHandler(AdminFeed)).Name("admin-feed")
	router.Handle("/admin/subhub", mpg.NewHandler(AdminSubHub)).Name("admin-subhub-feed")
	router.Handle("/admin/stats", mpg.NewHandler(AdminStats)).Name("admin-stats")
	router.Handle("/admin/update-feed", mpg.NewHandler(AdminUpdateFeed)).Name("admin-update-feed")
	router.Handle("/user/charge", mpg.NewHandler(Charge)).Name("charge")
	router.Handle("/user/account", mpg.NewHandler(Account)).Name("account")
	router.Handle("/user/uncheckout", mpg.NewHandler(Uncheckout)).Name("uncheckout")

	//router.Handle("/tasks/delete-blobs", mpg.NewHandler(DeleteBlobs)).Name("delete-blobs")

	if len(PUBSUBHUBBUB_HOST) > 0 {
		u := url.URL{
			Scheme:   "http",
			Host:     PUBSUBHUBBUB_HOST,
			Path:     routeUrl("add-subscription"),
			RawQuery: url.Values{"url": {"{url}"}}.Encode(),
		}
		subURL = u.String()
	}

	if !isDevServer {
		return
	}
	router.Handle("/user/clear-feeds", mpg.NewHandler(ClearFeeds)).Name("clear-feeds")
	router.Handle("/user/clear-read", mpg.NewHandler(ClearRead)).Name("clear-read")
	router.Handle("/test/atom.xml", mpg.NewHandler(TestAtom)).Name("test-atom")
}

func Main(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	if err := templates.ExecuteTemplate(w, "base.html", includes(c, w, r)); err != nil {
		c.Errorf("%v", err)
		serveError(w, err)
	}
	return
}

func addFeed(c mpg.Context, userid string, outline *OpmlOutline) error {
	gn := goon.FromContext(c)
	o := outline.Outline[0]
	c.Infof("adding feed %v to user %s", o.XmlUrl, userid)
	fu, ferr := url.Parse(o.XmlUrl)
	if ferr != nil {
		return ferr
	}
	fu.Fragment = ""
	o.XmlUrl = fu.String()

	f := Feed{Url: o.XmlUrl}
	if err := gn.Get(&f); err == datastore.ErrNoSuchEntity {
		if feed, stories, err := fetchFeed(c, o.XmlUrl, o.XmlUrl); err != nil {
			return fmt.Errorf("could not add feed %s: %v", o.XmlUrl, err)
		} else {
			f = *feed
			f.Updated = time.Time{}
			f.Checked = f.Updated
			f.NextUpdate = f.Updated
			f.LastViewed = time.Now()
			gn.Put(&f)
			for _, s := range stories {
				s.Created = s.Published
			}
			if err := updateFeed(c, f.Url, feed, stories, false, false, false); err != nil {
				return err
			}

			o.XmlUrl = feed.Url
			o.HtmlUrl = feed.Link
			if o.Title == "" {
				o.Title = feed.Title
			}
		}
	} else if err != nil {
		return err
	} else {
		o.HtmlUrl = f.Link
		if o.Title == "" {
			o.Title = f.Title
		}
	}
	o.Text = ""

	return nil
}

func mergeUserOpml(c appengine.Context, ud *UserData, outlines ...*OpmlOutline) error {
	var fs Opml
	json.Unmarshal(ud.Opml, &fs)
	urls := make(map[string]bool)

	for _, o := range fs.Outline {
		if o.XmlUrl != "" {
			urls[o.XmlUrl] = true
		} else {
			for _, so := range o.Outline {
				urls[so.XmlUrl] = true
			}
		}
	}

	mergeOutline := func(label string, outline *OpmlOutline) {
		if _, present := urls[outline.XmlUrl]; present {
			return
		} else {
			urls[outline.XmlUrl] = true

			if label == "" {
				fs.Outline = append(fs.Outline, outline)
			} else {
				done := false
				for _, ol := range fs.Outline {
					if ol.Title == label && ol.XmlUrl == "" {
						ol.Outline = append(ol.Outline, outline)
						done = true
						break
					}
				}
				if !done {
					fs.Outline = append(fs.Outline, &OpmlOutline{
						Title:   label,
						Outline: []*OpmlOutline{outline},
					})
				}
			}
		}
	}

	for _, outline := range outlines {
		if outline.XmlUrl != "" {
			mergeOutline("", outline)
		} else {
			for _, o := range outline.Outline {
				mergeOutline(outline.Title, o)
			}
		}
	}

	b, err := json.Marshal(&fs)
	if err != nil {
		return err
	}
	ud.Opml = b
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"net"
	"os"
	"os/signal"
	"unsafe"

	bpf "github.com/kinvolk/gobpf-elf-loader/bpf"
)

import "C"

type tcpEvent struct {
	// Timestamp must be the first field, the sorting depends on it
	Timestamp uint64

	Cpu   uint64
	Type  [12]C.char
	Pid   uint32
	Comm  [16]C.char
	SAddr uint32
	DAddr uint32
	SPort uint16
	DPort uint16
	NetNS uint32
}

var byteOrder binary.ByteOrder

// In lack of binary.HostEndian ...
func init() {
	var i int32 = 0x01020304
	u := unsafe.Pointer(&i)
	pb := (*byte)(u)
	b := *pb
	if b == 0x04 {
		byteOrder = binary.LittleEndian
	} else {
		byteOrder = binary.BigEndian
	}
}

var lastTimestamp uint64

func tcpEventCb(event tcpEvent) {
	timestamp := uint64(event.Timestamp)
	cpu := event.Cpu
	typ := C.GoString(&event.Type[0])
	pid := event.Pid & 0xffffffff

	saddrbuf := make([]byte, 4)
	daddrbuf := make([]byte, 4)

	binary.LittleEndian.PutUint32(saddrbuf, uint32(event.SAddr))
	binary.LittleEndian.PutUint32(daddrbuf, uint32(event.DAddr))

	sIP := net.IPv4(saddrbuf[0], saddrbuf[1], saddrbuf[2], saddrbuf[3])
	dIP := net.IPv4(daddrbuf[0], daddrbuf[1], daddrbuf[2], daddrbuf[3])

	sport := event.SPort
	dport := event.DPort
	netns := event.NetNS

	fmt.Printf("%v cpu#%d %s %v %v:%v %v:%v %v\n", timestamp, cpu, typ, pid, sIP, sport, dIP, dport, netns)

	if lastTimestamp > timestamp {
		fmt.Printf("WARNING: late event!\n")
		os.Exit(1)
	}

	lastTimestamp = timestamp
}

func main() {
	b := bpf.NewBpfPerfEvent("kernel/trace_output_kern.o")

	err := b.Load()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}

	fmt.Printf("Ready.\n")

	channel := make(chan []byte)

	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt, os.Kill)

	go func() {
		var event tcpEvent
		for {
			data := <-channel
			err := binary.Read(bytes.NewBuffer(data), byteOrder, &event)
			if err != nil {
				fmt.Printf("failed to decode received data: %s\n", err)
				continue
			}
			tcpEventCb(event)
		}
	}()

	b.PollStart("tcp_event", channel)
	<-sig
	b.PollStop("tcp_event")
}
<commit_msg>Require bpf object file as parameter<commit_after>package main

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"net"
	"os"
	"os/signal"
	"unsafe"

	bpf "github.com/kinvolk/gobpf-elf-loader/bpf"
)

import "C"

type tcpEvent struct {
	// Timestamp must be the first field, the sorting depends on it
	Timestamp uint64

	Cpu   uint64
	Type  [12]C.char
	Pid   uint32
	Comm  [16]C.char
	SAddr uint32
	DAddr uint32
	SPort uint16
	DPort uint16
	NetNS uint32
}

var byteOrder binary.ByteOrder

// In lack of binary.HostEndian ...
func init() {
	var i int32 = 0x01020304
	u := unsafe.Pointer(&i)
	pb := (*byte)(u)
	b := *pb
	if b == 0x04 {
		byteOrder = binary.LittleEndian
	} else {
		byteOrder = binary.BigEndian
	}
}

var lastTimestamp uint64

func tcpEventCb(event tcpEvent) {
	timestamp := uint64(event.Timestamp)
	cpu := event.Cpu
	typ := C.GoString(&event.Type[0])
	pid := event.Pid & 0xffffffff

	saddrbuf := make([]byte, 4)
	daddrbuf := make([]byte, 4)

	binary.LittleEndian.PutUint32(saddrbuf, uint32(event.SAddr))
	binary.LittleEndian.PutUint32(daddrbuf, uint32(event.DAddr))

	sIP := net.IPv4(saddrbuf[0], saddrbuf[1], saddrbuf[2], saddrbuf[3])
	dIP := net.IPv4(daddrbuf[0], daddrbuf[1], daddrbuf[2], daddrbuf[3])

	sport := event.SPort
	dport := event.DPort
	netns := event.NetNS

	fmt.Printf("%v cpu#%d %s %v %v:%v %v:%v %v\n", timestamp, cpu, typ, pid, sIP, sport, dIP, dport, netns)

	if lastTimestamp > timestamp {
		fmt.Printf("WARNING: late event!\n")
		os.Exit(1)
	}

	lastTimestamp = timestamp
}

func main() {
	if len(os.Args) != 2 {
		fmt.Fprintf(os.Stderr, "Usage: %s ${GOPATH}/src/github.com/kinvolk/tcptracer-bpf/ebpf/${DISTRO}/x86_64/$(uname -r)/ebpf.o\n", os.Args[0])
		os.Exit(1)
	}
	fileName := os.Args[1]
	b := bpf.NewBpfPerfEvent(fileName)

	err := b.Load()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}

	fmt.Printf("Ready.\n")

	channel := make(chan []byte)

	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt, os.Kill)

	go func() {
		var event tcpEvent
		for {
			data := <-channel
			err := binary.Read(bytes.NewBuffer(data), byteOrder, &event)
			if err != nil {
				fmt.Printf("failed to decode received data: %s\n", err)
				continue
			}
			tcpEventCb(event)
		}
	}()

	b.PollStart("tcp_event", channel)
	<-sig
	b.PollStop("tcp_event")
}
<|endoftext|>