<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rand

import (
	"bytes"
	"compress/flate"
	"io"
	"testing"
)

func TestRead(t *testing.T) {
	var n int = 4e6
	if testing.Short() {
		n = 1e5
	}
	b := make([]byte, n)
	n, err := io.ReadFull(Reader, b)
	if n != len(b) || err != nil {
		t.Fatalf("ReadFull(buf) = %d, %s", n, err)
	}

	var z bytes.Buffer
	f, _ := flate.NewWriter(&z, 5)
	f.Write(b)
	f.Close()
	if z.Len() < len(b)*99/100 {
		t.Fatalf("Compressed %d -> %d", len(b), z.Len())
	}
}

func TestReadEmpty(t *testing.T) {
	n, err := Reader.Read(make([]byte, 0))
	if n != 0 || err != nil {
		t.Fatalf("Read(make([]byte, 0)) = %d, %v", n, err)
	}
	n, err = Reader.Read(nil)
	if n != 0 || err != nil {
		t.Fatalf("Read(make(nil) = %d, %v", n, err)
	}
}
<commit_msg>crypto/rand: typo fix from a69e30463bf4<commit_after>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rand

import (
	"bytes"
	"compress/flate"
	"io"
	"testing"
)

func TestRead(t *testing.T) {
	var n int = 4e6
	if testing.Short() {
		n = 1e5
	}
	b := make([]byte, n)
	n, err := io.ReadFull(Reader, b)
	if n != len(b) || err != nil {
		t.Fatalf("ReadFull(buf) = %d, %s", n, err)
	}

	var z bytes.Buffer
	f, _ := flate.NewWriter(&z, 5)
	f.Write(b)
	f.Close()
	if z.Len() < len(b)*99/100 {
		t.Fatalf("Compressed %d -> %d", len(b), z.Len())
	}
}

func TestReadEmpty(t *testing.T) {
	n, err := Reader.Read(make([]byte, 0))
	if n != 0 || err != nil {
		t.Fatalf("Read(make([]byte, 0)) = %d, %v", n, err)
	}
	n, err = Reader.Read(nil)
	if n != 0 || err != nil {
		t.Fatalf("Read(nil) = %d, %v", n, err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package parser

import (
	"go/ast";
	"go/parser";
	"os";
	"testing";
)

var illegalInputs = []interface{} {
	nil,
	3.14,
	[]byte(nil),
	"foo!",
}

func TestParseIllegalInputs(t *testing.T) {
	for _, src := range illegalInputs {
		prog, err := ParseFile("", src, 0);
		if err == nil {
			t.Errorf("ParseFile(%v) should have failed", src);
		}
	}
}

var validPrograms = []interface{} {
	`package main`,
	`package main import "fmt" func main() { fmt.Println("Hello, World!") }`,
}

func TestParseValidPrograms(t *testing.T) {
	for _, src := range validPrograms {
		prog, err := ParseFile("", src, 0);
		if err != nil {
			t.Errorf("ParseFile(%q): %v", src, err);
		}
	}
}

var validFiles = []string {
	"parser.go",
	"parser_test.go",
}

func TestParse3(t *testing.T) {
	for _, filename := range validFiles {
		prog, err := ParseFile(filename, nil, 0);
		if err != nil {
			t.Errorf("ParseFile(%s): %v", filename, err);
		}
	}
}

func filter(filename string) bool {
	switch filename {
	case "parser.go":
	case "interface.go":
	case "parser_test.go":
	default:
		return false;
	}
	return true;
}

func TestParse4(t *testing.T) {
	path := ".";
	pkg, err := ParsePackage(path, filter, 0);
	if err != nil {
		t.Errorf("ParsePackage(%s): %v", path, err);
	}
	if pkg.Name != "parser" {
		t.Errorf("incorrect package name: %s", pkg.Name);
	}
	for filename, _ := range pkg.Files {
		if !filter(filename) {
			t.Errorf("unexpected package file: %s", filename);
		}
	}
}
<commit_msg>If we can't parse the package, call Fatalf so that the test fails without crashing.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package parser

import (
	"go/ast";
	"go/parser";
	"os";
	"testing";
)

var illegalInputs = []interface{} {
	nil,
	3.14,
	[]byte(nil),
	"foo!",
}

func TestParseIllegalInputs(t *testing.T) {
	for _, src := range illegalInputs {
		prog, err := ParseFile("", src, 0);
		if err == nil {
			t.Errorf("ParseFile(%v) should have failed", src);
		}
	}
}

var validPrograms = []interface{} {
	`package main`,
	`package main import "fmt" func main() { fmt.Println("Hello, World!") }`,
}

func TestParseValidPrograms(t *testing.T) {
	for _, src := range validPrograms {
		prog, err := ParseFile("", src, 0);
		if err != nil {
			t.Errorf("ParseFile(%q): %v", src, err);
		}
	}
}

var validFiles = []string {
	"parser.go",
	"parser_test.go",
}

func TestParse3(t *testing.T) {
	for _, filename := range validFiles {
		prog, err := ParseFile(filename, nil, 0);
		if err != nil {
			t.Errorf("ParseFile(%s): %v", filename, err);
		}
	}
}

func filter(filename string) bool {
	switch filename {
	case "parser.go":
	case "interface.go":
	case "parser_test.go":
	default:
		return false;
	}
	return true;
}

func TestParse4(t *testing.T) {
	path := ".";
	pkg, err := ParsePackage(path, filter, 0);
	if err != nil {
		t.Fatalf("ParsePackage(%s): %v", path, err);
	}
	if pkg.Name != "parser" {
		t.Errorf("incorrect package name: %s", pkg.Name);
	}
	for filename, _ := range pkg.Files {
		if !filter(filename) {
			t.Errorf("unexpected package file: %s", filename);
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package template

import (
	"fmt"
	"reflect"
)

// Strings of content from a trusted source.
type (
	// CSS encapsulates known safe content that matches any of:
	//   1. The CSS3 stylesheet production, such as `p { color: purple }`.
	//   2. The CSS3 rule production, such as `a[href=~"https:"].foo#bar`.
	//   3. CSS3 declaration productions, such as `color: red; margin: 2px`.
	//   4. The CSS3 value production, such as `rgba(0, 0, 255, 127)`.
	// See http://www.w3.org/TR/css3-syntax/#style
	CSS string

	// HTML encapsulates a known safe HTML document fragment.
	// It should not be used for HTML from a third-party, or HTML with
	// unclosed tags or comments. The outputs of a sound HTML sanitizer
	// and a template escaped by this package are fine for use with HTML.
	HTML string

	// HTMLAttr encapsulates an HTML attribute from a trusted source,
	// for example, ` dir="ltr"`.
	HTMLAttr string

	// JS encapsulates a known safe EcmaScript5 Expression, for example,
	// `(x + y * z())`.
	// Template authors are responsible for ensuring that typed expressions
	// do not break the intended precedence and that there is no
	// statement/expression ambiguity as when passing an expression like
	// "{ foo: bar() }\n['foo']()", which is both a valid Expression and a
	// valid Program with a very different meaning.
	JS string

	// JSStr encapsulates a sequence of characters meant to be embedded
	// between quotes in a JavaScript expression.
	// The string must match a series of StringCharacters:
	//   StringCharacter :: SourceCharacter but not `\` or LineTerminator
	//                    | EscapeSequence
	// Note that LineContinuations are not allowed.
	// JSStr("foo\\nbar") is fine, but JSStr("foo\\\nbar") is not.
	JSStr string

	// URL encapsulates a known safe URL as defined in RFC 3896.
	// A URL like `javascript:checkThatFormNotEditedBeforeLeavingPage()`
	// from a trusted source should go in the page, but by default dynamic
	// `javascript:` URLs are filtered out since they are a frequently
	// exploited injection vector.
	URL string
)

type contentType uint8

const (
	contentTypePlain contentType = iota
	contentTypeCSS
	contentTypeHTML
	contentTypeHTMLAttr
	contentTypeJS
	contentTypeJSStr
	contentTypeURL
	// contentTypeUnsafe is used in attr.go for values that affect how
	// embedded content and network messages are formed, vetted,
	// or interpreted; or which credentials network messages carry.
	contentTypeUnsafe
)

// indirect returns the value, after dereferencing as many times
// as necessary to reach the base type (or nil).
func indirect(a interface{}) interface{} {
	if t := reflect.TypeOf(a); t.Kind() != reflect.Ptr {
		// Avoid creating a reflect.Value if it's not a pointer.
		return a
	}
	v := reflect.ValueOf(a)
	for v.Kind() == reflect.Ptr && !v.IsNil() {
		v = v.Elem()
	}
	return v.Interface()
}

var (
	errorType       = reflect.TypeOf((*error)(nil)).Elem()
	fmtStringerType = reflect.TypeOf((*fmt.Stringer)(nil)).Elem()
)

// indirectToStringerOrError returns the value, after dereferencing as many times
// as necessary to reach the base type (or nil) or an implementation of fmt.Stringer
// or error,
func indirectToStringerOrError(a interface{}) interface{} {
	v := reflect.ValueOf(a)
	for !v.Type().Implements(fmtStringerType) && !v.Type().Implements(errorType) && v.Kind() == reflect.Ptr && !v.IsNil() {
		v = v.Elem()
	}
	return v.Interface()
}

// stringify converts its arguments to a string and the type of the content.
// All pointers are dereferenced, as in the text/template package.
func stringify(args ...interface{}) (string, contentType) {
	if len(args) == 1 {
		switch s := indirect(args[0]).(type) {
		case string:
			return s, contentTypePlain
		case CSS:
			return string(s), contentTypeCSS
		case HTML:
			return string(s), contentTypeHTML
		case HTMLAttr:
			return string(s), contentTypeHTMLAttr
		case JS:
			return string(s), contentTypeJS
		case JSStr:
			return string(s), contentTypeJSStr
		case URL:
			return string(s), contentTypeURL
		}
	}
	for i, arg := range args {
		args[i] = indirectToStringerOrError(arg)
	}
	return fmt.Sprint(args...), contentTypePlain
}
<commit_msg>html/template: fix URL doc<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package template

import (
	"fmt"
	"reflect"
)

// Strings of content from a trusted source.
type (
	// CSS encapsulates known safe content that matches any of:
	//   1. The CSS3 stylesheet production, such as `p { color: purple }`.
	//   2. The CSS3 rule production, such as `a[href=~"https:"].foo#bar`.
	//   3. CSS3 declaration productions, such as `color: red; margin: 2px`.
	//   4. The CSS3 value production, such as `rgba(0, 0, 255, 127)`.
	// See http://www.w3.org/TR/css3-syntax/#style
	CSS string

	// HTML encapsulates a known safe HTML document fragment.
	// It should not be used for HTML from a third-party, or HTML with
	// unclosed tags or comments. The outputs of a sound HTML sanitizer
	// and a template escaped by this package are fine for use with HTML.
	HTML string

	// HTMLAttr encapsulates an HTML attribute from a trusted source,
	// for example, ` dir="ltr"`.
	HTMLAttr string

	// JS encapsulates a known safe EcmaScript5 Expression, for example,
	// `(x + y * z())`.
	// Template authors are responsible for ensuring that typed expressions
	// do not break the intended precedence and that there is no
	// statement/expression ambiguity as when passing an expression like
	// "{ foo: bar() }\n['foo']()", which is both a valid Expression and a
	// valid Program with a very different meaning.
	JS string

	// JSStr encapsulates a sequence of characters meant to be embedded
	// between quotes in a JavaScript expression.
	// The string must match a series of StringCharacters:
	//   StringCharacter :: SourceCharacter but not `\` or LineTerminator
	//                    | EscapeSequence
	// Note that LineContinuations are not allowed.
	// JSStr("foo\\nbar") is fine, but JSStr("foo\\\nbar") is not.
	JSStr string

	// URL encapsulates a known safe URL or URL substring (see RFC 3986).
	// A URL like `javascript:checkThatFormNotEditedBeforeLeavingPage()`
	// from a trusted source should go in the page, but by default dynamic
	// `javascript:` URLs are filtered out since they are a frequently
	// exploited injection vector.
	URL string
)

type contentType uint8

const (
	contentTypePlain contentType = iota
	contentTypeCSS
	contentTypeHTML
	contentTypeHTMLAttr
	contentTypeJS
	contentTypeJSStr
	contentTypeURL
	// contentTypeUnsafe is used in attr.go for values that affect how
	// embedded content and network messages are formed, vetted,
	// or interpreted; or which credentials network messages carry.
	contentTypeUnsafe
)

// indirect returns the value, after dereferencing as many times
// as necessary to reach the base type (or nil).
func indirect(a interface{}) interface{} {
	if t := reflect.TypeOf(a); t.Kind() != reflect.Ptr {
		// Avoid creating a reflect.Value if it's not a pointer.
		return a
	}
	v := reflect.ValueOf(a)
	for v.Kind() == reflect.Ptr && !v.IsNil() {
		v = v.Elem()
	}
	return v.Interface()
}

var (
	errorType       = reflect.TypeOf((*error)(nil)).Elem()
	fmtStringerType = reflect.TypeOf((*fmt.Stringer)(nil)).Elem()
)

// indirectToStringerOrError returns the value, after dereferencing as many times
// as necessary to reach the base type (or nil) or an implementation of fmt.Stringer
// or error,
func indirectToStringerOrError(a interface{}) interface{} {
	v := reflect.ValueOf(a)
	for !v.Type().Implements(fmtStringerType) && !v.Type().Implements(errorType) && v.Kind() == reflect.Ptr && !v.IsNil() {
		v = v.Elem()
	}
	return v.Interface()
}

// stringify converts its arguments to a string and the type of the content.
// All pointers are dereferenced, as in the text/template package.
func stringify(args ...interface{}) (string, contentType) {
	if len(args) == 1 {
		switch s := indirect(args[0]).(type) {
		case string:
			return s, contentTypePlain
		case CSS:
			return string(s), contentTypeCSS
		case HTML:
			return string(s), contentTypeHTML
		case HTMLAttr:
			return string(s), contentTypeHTMLAttr
		case JS:
			return string(s), contentTypeJS
		case JSStr:
			return string(s), contentTypeJSStr
		case URL:
			return string(s), contentTypeURL
		}
	}
	for i, arg := range args {
		args[i] = indirectToStringerOrError(arg)
	}
	return fmt.Sprint(args...), contentTypePlain
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Netlink sockets and messages

package syscall

import (
	"unsafe"
)

// Round the length of a netlink message up to align it properly.
func nlmAlignOf(msglen int) int {
	return (msglen + NLMSG_ALIGNTO - 1) & ^(NLMSG_ALIGNTO - 1)
}

// Round the length of a netlink route attribute up to align it
// properly.
func rtaAlignOf(attrlen int) int {
	return (attrlen + RTA_ALIGNTO - 1) & ^(RTA_ALIGNTO - 1)
}

// NetlinkRouteRequest represents the request message to receive
// routing and link states from the kernel.
type NetlinkRouteRequest struct {
	Header NlMsghdr
	Data   RtGenmsg
}

func (rr *NetlinkRouteRequest) toWireFormat() []byte {
	b := make([]byte, rr.Header.Len)
	b[0] = byte(rr.Header.Len)
	b[1] = byte(rr.Header.Len >> 8)
	b[2] = byte(rr.Header.Len >> 16)
	b[3] = byte(rr.Header.Len >> 24)
	b[4] = byte(rr.Header.Type)
	b[5] = byte(rr.Header.Type >> 8)
	b[6] = byte(rr.Header.Flags)
	b[7] = byte(rr.Header.Flags >> 8)
	b[8] = byte(rr.Header.Seq)
	b[9] = byte(rr.Header.Seq >> 8)
	b[10] = byte(rr.Header.Seq >> 16)
	b[11] = byte(rr.Header.Seq >> 24)
	b[12] = byte(rr.Header.Pid)
	b[13] = byte(rr.Header.Pid >> 8)
	b[14] = byte(rr.Header.Pid >> 16)
	b[15] = byte(rr.Header.Pid >> 24)
	b[16] = byte(rr.Data.Family)
	return b
}

func newNetlinkRouteRequest(proto, seq, family int) []byte {
	rr := &NetlinkRouteRequest{}
	rr.Header.Len = NLMSG_HDRLEN + SizeofRtGenmsg
	rr.Header.Type = uint16(proto)
	rr.Header.Flags = NLM_F_DUMP | NLM_F_REQUEST
	rr.Header.Seq = uint32(seq)
	rr.Data.Family = uint8(family)
	return rr.toWireFormat()
}

// NetlinkRIB returns routing information base, as known as RIB,
// which consists of network facility information, states and
// parameters.
func NetlinkRIB(proto, family int) ([]byte, int) {
	var (
		s     int
		e     int
		lsanl SockaddrNetlink
		seq   int
		tab   []byte
	)

	s, e = Socket(AF_NETLINK, SOCK_RAW, 0)
	if e != 0 {
		return nil, e
	}
	defer Close(s)

	lsanl.Family = AF_NETLINK
	e = Bind(s, &lsanl)
	if e != 0 {
		return nil, e
	}

	seq++
	wb := newNetlinkRouteRequest(proto, seq, family)
	e = Sendto(s, wb, 0, &lsanl)
	if e != 0 {
		return nil, e
	}

	for {
		var (
			rb  []byte
			nr  int
			lsa Sockaddr
		)

		rb = make([]byte, Getpagesize())
		nr, _, e = Recvfrom(s, rb, 0)
		if e != 0 {
			return nil, e
		}
		if nr < NLMSG_HDRLEN {
			return nil, EINVAL
		}
		rb = rb[:nr]
		tab = append(tab, rb...)

		msgs, _ := ParseNetlinkMessage(rb)
		for _, m := range msgs {
			if lsa, e = Getsockname(s); e != 0 {
				return nil, e
			}
			switch v := lsa.(type) {
			case *SockaddrNetlink:
				if m.Header.Seq != uint32(seq) || m.Header.Pid != v.Pid {
					return nil, EINVAL
				}
			default:
				return nil, EINVAL
			}
			if m.Header.Type == NLMSG_DONE {
				goto done
			}
			if m.Header.Type == NLMSG_ERROR {
				return nil, EINVAL
			}
		}
	}

done:
	return tab, 0
}

// NetlinkMessage represents the netlink message.
type NetlinkMessage struct {
	Header NlMsghdr
	Data   []byte
}

// ParseNetlinkMessage parses buf as netlink messages and returns
// the slice containing the NetlinkMessage structs.
func ParseNetlinkMessage(buf []byte) ([]NetlinkMessage, int) {
	var (
		h    *NlMsghdr
		dbuf []byte
		dlen int
		e    int
		msgs []NetlinkMessage
	)

	for len(buf) >= NLMSG_HDRLEN {
		h, dbuf, dlen, e = netlinkMessageHeaderAndData(buf)
		if e != 0 {
			break
		}
		m := NetlinkMessage{}
		m.Header = *h
		m.Data = dbuf[:h.Len-NLMSG_HDRLEN]
		msgs = append(msgs, m)
		buf = buf[dlen:]
	}

	return msgs, e
}

func netlinkMessageHeaderAndData(buf []byte) (*NlMsghdr, []byte, int, int) {
	h := (*NlMsghdr)(unsafe.Pointer(&buf[0]))
	if h.Len < NLMSG_HDRLEN || int(h.Len) > len(buf) {
		return nil, nil, 0, EINVAL
	}
	return h, buf[NLMSG_HDRLEN:], nlmAlignOf(int(h.Len)), 0
}

// NetlinkRouteAttr represents the netlink route attribute.
type NetlinkRouteAttr struct {
	Attr  RtAttr
	Value []byte
}

// ParseNetlinkRouteAttr parses msg's payload as netlink route
// attributes and returns the slice containing the NetlinkRouteAttr
// structs.
func ParseNetlinkRouteAttr(msg *NetlinkMessage) ([]NetlinkRouteAttr, int) {
	var (
		buf   []byte
		a     *RtAttr
		alen  int
		vbuf  []byte
		e     int
		attrs []NetlinkRouteAttr
	)

	switch msg.Header.Type {
	case RTM_NEWLINK:
		buf = msg.Data[SizeofIfInfomsg:]
	case RTM_NEWADDR:
		buf = msg.Data[SizeofIfAddrmsg:]
	default:
		return nil, EINVAL
	}

	for len(buf) >= SizeofRtAttr {
		a, vbuf, alen, e = netlinkRouteAttrAndValue(buf)
		if e != 0 {
			break
		}
		ra := NetlinkRouteAttr{}
		ra.Attr = *a
		ra.Value = vbuf[:a.Len-SizeofRtAttr]
		attrs = append(attrs, ra)
		buf = buf[alen:]
	}

	return attrs, 0
}

func netlinkRouteAttrAndValue(buf []byte) (*RtAttr, []byte, int, int) {
	h := (*RtAttr)(unsafe.Pointer(&buf[0]))
	if h.Len < SizeofRtAttr || int(h.Len) > len(buf) {
		return nil, nil, 0, EINVAL
	}
	return h, buf[SizeofRtAttr:], rtaAlignOf(int(h.Len)), 0
}
<commit_msg>syscall: Handle RTM_NEWROUTE in ParseNetlinkRouteAttr on Linux.<commit_after>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Netlink sockets and messages

package syscall

import (
	"unsafe"
)

// Round the length of a netlink message up to align it properly.
func nlmAlignOf(msglen int) int {
	return (msglen + NLMSG_ALIGNTO - 1) & ^(NLMSG_ALIGNTO - 1)
}

// Round the length of a netlink route attribute up to align it
// properly.
func rtaAlignOf(attrlen int) int {
	return (attrlen + RTA_ALIGNTO - 1) & ^(RTA_ALIGNTO - 1)
}

// NetlinkRouteRequest represents the request message to receive
// routing and link states from the kernel.
type NetlinkRouteRequest struct {
	Header NlMsghdr
	Data   RtGenmsg
}

func (rr *NetlinkRouteRequest) toWireFormat() []byte {
	b := make([]byte, rr.Header.Len)
	b[0] = byte(rr.Header.Len)
	b[1] = byte(rr.Header.Len >> 8)
	b[2] = byte(rr.Header.Len >> 16)
	b[3] = byte(rr.Header.Len >> 24)
	b[4] = byte(rr.Header.Type)
	b[5] = byte(rr.Header.Type >> 8)
	b[6] = byte(rr.Header.Flags)
	b[7] = byte(rr.Header.Flags >> 8)
	b[8] = byte(rr.Header.Seq)
	b[9] = byte(rr.Header.Seq >> 8)
	b[10] = byte(rr.Header.Seq >> 16)
	b[11] = byte(rr.Header.Seq >> 24)
	b[12] = byte(rr.Header.Pid)
	b[13] = byte(rr.Header.Pid >> 8)
	b[14] = byte(rr.Header.Pid >> 16)
	b[15] = byte(rr.Header.Pid >> 24)
	b[16] = byte(rr.Data.Family)
	return b
}

func newNetlinkRouteRequest(proto, seq, family int) []byte {
	rr := &NetlinkRouteRequest{}
	rr.Header.Len = NLMSG_HDRLEN + SizeofRtGenmsg
	rr.Header.Type = uint16(proto)
	rr.Header.Flags = NLM_F_DUMP | NLM_F_REQUEST
	rr.Header.Seq = uint32(seq)
	rr.Data.Family = uint8(family)
	return rr.toWireFormat()
}

// NetlinkRIB returns routing information base, as known as RIB,
// which consists of network facility information, states and
// parameters.
func NetlinkRIB(proto, family int) ([]byte, int) {
	var (
		s     int
		e     int
		lsanl SockaddrNetlink
		seq   int
		tab   []byte
	)

	s, e = Socket(AF_NETLINK, SOCK_RAW, 0)
	if e != 0 {
		return nil, e
	}
	defer Close(s)

	lsanl.Family = AF_NETLINK
	e = Bind(s, &lsanl)
	if e != 0 {
		return nil, e
	}

	seq++
	wb := newNetlinkRouteRequest(proto, seq, family)
	e = Sendto(s, wb, 0, &lsanl)
	if e != 0 {
		return nil, e
	}

	for {
		var (
			rb  []byte
			nr  int
			lsa Sockaddr
		)

		rb = make([]byte, Getpagesize())
		nr, _, e = Recvfrom(s, rb, 0)
		if e != 0 {
			return nil, e
		}
		if nr < NLMSG_HDRLEN {
			return nil, EINVAL
		}
		rb = rb[:nr]
		tab = append(tab, rb...)

		msgs, _ := ParseNetlinkMessage(rb)
		for _, m := range msgs {
			if lsa, e = Getsockname(s); e != 0 {
				return nil, e
			}
			switch v := lsa.(type) {
			case *SockaddrNetlink:
				if m.Header.Seq != uint32(seq) || m.Header.Pid != v.Pid {
					return nil, EINVAL
				}
			default:
				return nil, EINVAL
			}
			if m.Header.Type == NLMSG_DONE {
				goto done
			}
			if m.Header.Type == NLMSG_ERROR {
				return nil, EINVAL
			}
		}
	}

done:
	return tab, 0
}

// NetlinkMessage represents the netlink message.
type NetlinkMessage struct {
	Header NlMsghdr
	Data   []byte
}

// ParseNetlinkMessage parses buf as netlink messages and returns
// the slice containing the NetlinkMessage structs.
func ParseNetlinkMessage(buf []byte) ([]NetlinkMessage, int) {
	var (
		h    *NlMsghdr
		dbuf []byte
		dlen int
		e    int
		msgs []NetlinkMessage
	)

	for len(buf) >= NLMSG_HDRLEN {
		h, dbuf, dlen, e = netlinkMessageHeaderAndData(buf)
		if e != 0 {
			break
		}
		m := NetlinkMessage{}
		m.Header = *h
		m.Data = dbuf[:h.Len-NLMSG_HDRLEN]
		msgs = append(msgs, m)
		buf = buf[dlen:]
	}

	return msgs, e
}

func netlinkMessageHeaderAndData(buf []byte) (*NlMsghdr, []byte, int, int) {
	h := (*NlMsghdr)(unsafe.Pointer(&buf[0]))
	if h.Len < NLMSG_HDRLEN || int(h.Len) > len(buf) {
		return nil, nil, 0, EINVAL
	}
	return h, buf[NLMSG_HDRLEN:], nlmAlignOf(int(h.Len)), 0
}

// NetlinkRouteAttr represents the netlink route attribute.
type NetlinkRouteAttr struct {
	Attr  RtAttr
	Value []byte
}

// ParseNetlinkRouteAttr parses msg's payload as netlink route
// attributes and returns the slice containing the NetlinkRouteAttr
// structs.
func ParseNetlinkRouteAttr(msg *NetlinkMessage) ([]NetlinkRouteAttr, int) {
	var (
		buf   []byte
		a     *RtAttr
		alen  int
		vbuf  []byte
		e     int
		attrs []NetlinkRouteAttr
	)

	switch msg.Header.Type {
	case RTM_NEWLINK:
		buf = msg.Data[SizeofIfInfomsg:]
	case RTM_NEWADDR:
		buf = msg.Data[SizeofIfAddrmsg:]
	case RTM_NEWROUTE:
		buf = msg.Data[SizeofRtmsg:]
	default:
		return nil, EINVAL
	}

	for len(buf) >= SizeofRtAttr {
		a, vbuf, alen, e = netlinkRouteAttrAndValue(buf)
		if e != 0 {
			break
		}
		ra := NetlinkRouteAttr{}
		ra.Attr = *a
		ra.Value = vbuf[:a.Len-SizeofRtAttr]
		attrs = append(attrs, ra)
		buf = buf[alen:]
	}

	return attrs, 0
}

func netlinkRouteAttrAndValue(buf []byte) (*RtAttr, []byte, int, int) {
	h := (*RtAttr)(unsafe.Pointer(&buf[0]))
	if h.Len < SizeofRtAttr || int(h.Len) > len(buf) {
		return nil, nil, 0, EINVAL
	}
	return h, buf[SizeofRtAttr:], rtaAlignOf(int(h.Len)), 0
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os/exec"
	"path/filepath"
	"strings"
)

var binaryPath string

func init() {
	var err error
	if binaryPath, err = exec.LookPath("go"); err != nil {
		log.Fatal(err)
	}
}

func getPackages(workspace string) ([]string, error) {
	// dir := filepath.Join(workspace, "...") // does not work with '...'
	dir := filepath.Clean(workspace) + string(filepath.Separator) + "..."
	out, err := exec.Command(binaryPath, "list", dir).Output()
	if err != nil {
		return nil, err
	}
	// TODO: improve this.
	str := strings.TrimSpace(string(out))
	return strings.Split(str, "\n"), nil
}

func getDependencies(name string) (map[string]bool, error) {
	out, err := exec.Command(binaryPath, "list", "-json", name).Output()
	if err != nil {
		return nil, err
	}

	var p struct{ Deps []string }
	if err := json.Unmarshal(out, &p); err != nil {
		return nil, err
	}

	deps := make(map[string]bool)
	for _, pkg := range p.Deps {
		deps[pkg] = true
	}
	return deps, nil
}

func getUsedPackages(deps map[string]bool, packages []string) []string {
	used := make([]string, 0)
	for _, pkg := range packages {
		if !deps[pkg] {
			continue
		}
		used = append(used, pkg)
	}
	return used
}

func unique(in []string) []string {
	seen := make(map[string]bool)
	out := make([]string, 0, len(in)) // think about in-place modification
	for _, item := range in {
		if seen[item] {
			continue
		}
		seen[item] = true
		out = append(out, item)
	}
	return out
}

func main() {
	flag.Parse()
	args := flag.Args()
	if len(args) <= 0 {
		log.Fatal("please, specify package names")
	}

	argPkgs := unique(args)

	workspacePkgs, err := getPackages(".")
	if err != nil {
		log.Fatal(err)
	}

	for _, p := range workspacePkgs {
		deps, err := getDependencies(p)
		if err != nil {
			log.Fatal(err)
		}
		used := getUsedPackages(deps, argPkgs)
		if len(used) > 0 {
			fmt.Printf("%s: [%s]\n", p, strings.Join(used, ", "))
		}
	}
}
<commit_msg>[10.4] add ability to specify relative workspace path<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os/exec"
	"path/filepath"
	"strings"
)

var binaryPath string

func init() {
	var err error
	if binaryPath, err = exec.LookPath("go"); err != nil {
		log.Fatal(err)
	}
}

func getPackages(workspace string) ([]string, error) {
	// dir := filepath.Join(workspace, "...") // does not work with '...'
	dir := filepath.Clean(workspace) + string(filepath.Separator) + "..."
	out, err := exec.Command(binaryPath, "list", dir).Output()
	if err != nil {
		return nil, err
	}
	// TODO: improve this.
	str := strings.TrimSpace(string(out))
	return strings.Split(str, "\n"), nil
}

func getDependencies(name string) (map[string]bool, error) {
	out, err := exec.Command(binaryPath, "list", "-json", name).Output()
	if err != nil {
		return nil, err
	}

	var p struct{ Deps []string }
	if err := json.Unmarshal(out, &p); err != nil {
		return nil, err
	}

	deps := make(map[string]bool)
	for _, pkg := range p.Deps {
		deps[pkg] = true
	}
	return deps, nil
}

func getUsedPackages(deps map[string]bool, packages []string) []string {
	used := make([]string, 0)
	for _, pkg := range packages {
		if !deps[pkg] {
			continue
		}
		used = append(used, pkg)
	}
	return used
}

func unique(in []string) []string {
	seen := make(map[string]bool)
	out := make([]string, 0, len(in)) // think about in-place modification
	for _, item := range in {
		if seen[item] {
			continue
		}
		seen[item] = true
		out = append(out, item)
	}
	return out
}

func main() {
	workspacePath := flag.String("workspace", ".", "workspace relative path")
	flag.Parse()
	args := flag.Args()
	if len(args) <= 0 {
		log.Fatal("please, specify package names")
	}

	argPkgs := unique(args)

	workspacePkgs, err := getPackages(*workspacePath)
	if err != nil {
		log.Fatal(err)
	}

	for _, p := range workspacePkgs {
		deps, err := getDependencies(p)
		if err != nil {
			log.Fatal(err)
		}
		used := getUsedPackages(deps, argPkgs)
		if len(used) > 0 {
			fmt.Printf("%s: [%s]\n", p, strings.Join(used, ", "))
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bolt2

import (
	"bytes"
	"encoding/binary"
	"errors"

	"github.com/boltdb/bolt"
	"github.com/cayleygraph/cayley/clog"
	"github.com/cayleygraph/cayley/internal/lru"
	"github.com/cayleygraph/cayley/quad"
	"github.com/cayleygraph/cayley/quad/pquads"

	"github.com/cayleygraph/cayley/graph"
)

func init() {
	graph.RegisterQuadStore(QuadStoreType, graph.QuadStoreRegistration{
		NewFunc:           newQuadStore,
		NewForRequestFunc: nil,
		UpgradeFunc:       nil,
		InitFunc:          createNewBolt,
		IsPersistent:      true,
	})
}

var (
	errNoBucket = errors.New("bolt2: bucket is missing")
)

const (
	QuadStoreType     = "bolt2"
	latestDataVersion = 1
	nilDataVersion    = 1
	notFound          = 0
)

type QuadStore struct {
	db            *bolt.DB
	path          string
	open          bool
	size          int64
	horizon       int64
	sameAsHorizon int64
	version       int64
	mapBucket     map[string]map[uint64][]uint64
	valueLRU      *lru.Cache
}

func createNewBolt(path string, _ graph.Options) error {
	db, err := bolt.Open(path, 0600, nil)
	if err != nil {
		clog.Errorf("Error: couldn't create Bolt database: %v", err)
		return err
	}
	defer db.Close()
	qs := &QuadStore{}
	qs.db = db
	defer qs.Close()
	err = qs.getMetadata()
	if err != errNoBucket {
		return graph.ErrDatabaseExists
	}
	err = qs.createBuckets()
	if err != nil {
		return err
	}
	err = setVersion(qs.db, latestDataVersion)
	if err != nil {
		return err
	}
	qs.Close()
	return nil
}

func newQuadStore(path string, options graph.Options) (graph.QuadStore, error) {
	var qs QuadStore
	var err error
	db, err := bolt.Open(path, 0600, nil)
	if err != nil {
		clog.Errorf("Error, couldn't open! %v", err)
		return nil, err
	}
	qs.db = db
	// BoolKey returns false on non-existence. IE, Sync by default.
	qs.db.NoSync, _, err = options.BoolKey("nosync")
	if err != nil {
		return nil, err
	}
	err = qs.getMetadata()
	if err == errNoBucket {
		return nil, errors.New("bolt: quadstore has not been initialised")
	} else if err != nil {
		return nil, err
	}
	if qs.version != latestDataVersion {
		return nil, errors.New("bolt: data version is out of date. Run cayleyupgrade for your config to update the data.")
	}
	qs.valueLRU = lru.New(500)
	return &qs, nil
}

func setVersion(db *bolt.DB, version int64) error {
	return db.Update(func(tx *bolt.Tx) error {
		buf := new(bytes.Buffer)
		err := binary.Write(buf, binary.LittleEndian, version)
		if err != nil {
			clog.Errorf("Couldn't convert version!")
			return err
		}
		b := tx.Bucket(metaBucket)
		werr := b.Put([]byte("version"), buf.Bytes())
		if werr != nil {
			clog.Errorf("Couldn't write version!")
			return werr
		}
		return nil
	})
}

func (qs *QuadStore) Size() int64 {
	return qs.size
}

func (qs *QuadStore) Close() error {
	qs.db.Update(func(tx *bolt.Tx) error {
		return qs.writeHorizonAndSize(tx)
	})
	err := qs.db.Close()
	qs.open = false
	return err
}

func (qs *QuadStore) getMetadata() error {
	err := qs.db.View(func(tx *bolt.Tx) error {
		var err error
		qs.size, err = getInt64ForMetaKey(tx, "size", 0)
		if err != nil {
			return err
		}
		qs.version, err = getInt64ForMetaKey(tx, "version", nilDataVersion)
		if err != nil {
			return err
		}
		qs.sameAsHorizon, err = getInt64ForMetaKey(tx, "sameAsHorizon", 0)
		if err != nil {
			return err
		}
		qs.horizon, err = getInt64ForMetaKey(tx, "horizon", 0)
		return err
	})
	return err
}

func getInt64ForMetaKey(tx *bolt.Tx, key string, empty int64) (int64, error) {
	return getInt64ForKey(tx, metaBucket, key, empty)
}

func getInt64ForKey(tx *bolt.Tx, bucket []byte, key string, empty int64) (int64, error) {
	var out int64
	b := tx.Bucket(bucket)
	if b == nil {
		return empty, errNoBucket
	}
	data := b.Get([]byte(key))
	if data == nil {
		return empty, nil
	}
	buf := bytes.NewBuffer(data)
	err := binary.Read(buf, binary.LittleEndian, &out)
	if err != nil {
		return 0, err
	}
	return out, nil
}

func (qs *QuadStore) Horizon() graph.PrimaryKey {
	return graph.NewSequentialKey(qs.horizon)
}

func (qs *QuadStore) NameOf(k graph.Value) quad.Value {
	if v, ok := k.(Int64Value); ok {
		if v == 0 {
			if clog.V(2) {
				clog.Infof("k was 0")
			}
			return nil
		}
		var val quad.Value
		err := qs.db.View(func(tx *bolt.Tx) error {
			var err error
			val, err = qs.getValFromLog(tx, uint64(v))
			return err
		})
		if err != nil {
			clog.Errorf("error getting NameOf %d: %s", v, err)
			return nil
		}
		return val
	}
	panic("unknown type of graph.Value; not meant for this quadstore")
}

func (qs *QuadStore) Quad(k graph.Value) quad.Quad {
	var v quad.Quad
	key := k.(Int64Value)
	err := qs.db.View(func(tx *bolt.Tx) error {
		var err error
		v, err = qs.toQuad(tx, uint64(key))
		return err
	})
	if err != nil {
		clog.Errorf("error fetching quad %d: %s", key, err)
	}
	return v
}

func (qs *QuadStore) toQuad(tx *bolt.Tx, k uint64) (quad.Quad, error) {
	q := &quad.Quad{}
	p, err := qs.getPrimitiveFromLog(tx, k)
	if err != nil {
		return *q, err
	}
	for _, dir := range quad.Directions {
		v := p.GetDirection(dir)
		val, err := qs.getValFromLog(tx, v)
		if err != nil {
			return *q, err
		}
		q.Set(dir, val)
	}
	return *q, nil
}

func (qs *QuadStore) getValFromLog(tx *bolt.Tx, k uint64) (quad.Value, error) {
	if k == 0 {
		return nil, nil
	}
	p, err := qs.getPrimitiveFromLog(tx, k)
	if err != nil {
		return nil, err
	}
	return pquads.UnmarshalValue(p.Value)
}

func (qs *QuadStore) ValueOf(s quad.Value) graph.Value {
	var out Int64Value
	qs.db.View(func(tx *bolt.Tx) error {
		out = Int64Value(qs.resolveQuadValue(tx, s))
		return nil
	})
	return out
}

func (qs *QuadStore) Type() string {
	return QuadStoreType
}

func (qs *QuadStore) QuadDirection(val graph.Value, d quad.Direction) graph.Value {
	p, ok := qs.getPrimitive(val.(Int64Value))
	if !ok {
		return Int64Value(0)
	}
	switch d {
	case quad.Subject:
		return Int64Value(p.Subject)
	case quad.Predicate:
		return Int64Value(p.Predicate)
	case quad.Object:
		return Int64Value(p.Object)
	case quad.Label:
		return Int64Value(p.Label)
	}
	return Int64Value(0)
}

func (qs *QuadStore) getPrimitive(val Int64Value) (*graph.Primitive, bool) {
	if val == 0 {
		return nil, false
	}
	var p *graph.Primitive
	err := qs.db.View(func(tx *bolt.Tx) error {
		var err error
		p, err = qs.getPrimitiveFromLog(tx, uint64(val))
		return err
	})
	if err != nil {
		clog.Errorf("error getting primitive %d: %s", val, err)
		return p, false
	}
	return p, true
}

type Int64Value uint64
<commit_msg>fix contract<commit_after>// Copyright 2016 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bolt2

import (
	"bytes"
	"encoding/binary"
	"errors"

	"github.com/boltdb/bolt"
	"github.com/cayleygraph/cayley/clog"
	"github.com/cayleygraph/cayley/internal/lru"
	"github.com/cayleygraph/cayley/quad"
	"github.com/cayleygraph/cayley/quad/pquads"

	"github.com/cayleygraph/cayley/graph"
)

func init() {
	graph.RegisterQuadStore(QuadStoreType, graph.QuadStoreRegistration{
		NewFunc:      newQuadStore,
		UpgradeFunc:  nil,
		InitFunc:     createNewBolt,
		IsPersistent: true,
	})
}

var (
	errNoBucket = errors.New("bolt2: bucket is missing")
)

const (
	QuadStoreType     = "bolt2"
	latestDataVersion = 1
	nilDataVersion    = 1
	notFound          = 0
)

type QuadStore struct {
	db            *bolt.DB
	path          string
	open          bool
	size          int64
	horizon       int64
	sameAsHorizon int64
	version       int64
	mapBucket     map[string]map[uint64][]uint64
	valueLRU      *lru.Cache
}

func createNewBolt(path string, _ graph.Options) error {
	db, err := bolt.Open(path, 0600, nil)
	if err != nil {
		clog.Errorf("Error: couldn't create Bolt database: %v", err)
		return err
	}
	defer db.Close()
	qs := &QuadStore{}
	qs.db = db
	defer qs.Close()
	err = qs.getMetadata()
	if err != errNoBucket {
		return graph.ErrDatabaseExists
	}
	err = qs.createBuckets()
	if err != nil {
		return err
	}
	err = setVersion(qs.db, latestDataVersion)
	if err != nil {
		return err
	}
	qs.Close()
	return nil
}

func newQuadStore(path string, options graph.Options) (graph.QuadStore, error) {
	var qs QuadStore
	var err error
	db, err := bolt.Open(path, 0600, nil)
	if err != nil {
		clog.Errorf("Error, couldn't open! %v", err)
		return nil, err
	}
	qs.db = db
	// BoolKey returns false on non-existence. IE, Sync by default.
	qs.db.NoSync, _, err = options.BoolKey("nosync")
	if err != nil {
		return nil, err
	}
	err = qs.getMetadata()
	if err == errNoBucket {
		return nil, errors.New("bolt: quadstore has not been initialised")
	} else if err != nil {
		return nil, err
	}
	if qs.version != latestDataVersion {
		return nil, errors.New("bolt: data version is out of date. Run cayleyupgrade for your config to update the data.")
	}
	qs.valueLRU = lru.New(500)
	return &qs, nil
}

func setVersion(db *bolt.DB, version int64) error {
	return db.Update(func(tx *bolt.Tx) error {
		buf := new(bytes.Buffer)
		err := binary.Write(buf, binary.LittleEndian, version)
		if err != nil {
			clog.Errorf("Couldn't convert version!")
			return err
		}
		b := tx.Bucket(metaBucket)
		werr := b.Put([]byte("version"), buf.Bytes())
		if werr != nil {
			clog.Errorf("Couldn't write version!")
			return werr
		}
		return nil
	})
}

func (qs *QuadStore) Size() int64 {
	return qs.size
}

func (qs *QuadStore) Close() error {
	qs.db.Update(func(tx *bolt.Tx) error {
		return qs.writeHorizonAndSize(tx)
	})
	err := qs.db.Close()
	qs.open = false
	return err
}

func (qs *QuadStore) getMetadata() error {
	err := qs.db.View(func(tx *bolt.Tx) error {
		var err error
		qs.size, err = getInt64ForMetaKey(tx, "size", 0)
		if err != nil {
			return err
		}
		qs.version, err = getInt64ForMetaKey(tx, "version", nilDataVersion)
		if err != nil {
			return err
		}
		qs.sameAsHorizon, err = getInt64ForMetaKey(tx, "sameAsHorizon", 0)
		if err != nil {
			return err
		}
		qs.horizon, err = getInt64ForMetaKey(tx, "horizon", 0)
		return err
	})
	return err
}

func getInt64ForMetaKey(tx *bolt.Tx, key string, empty int64) (int64, error) {
	return getInt64ForKey(tx, metaBucket, key, empty)
}

func getInt64ForKey(tx *bolt.Tx, bucket []byte, key string, empty int64) (int64, error) {
	var out int64
	b := tx.Bucket(bucket)
	if b == nil {
		return empty, errNoBucket
	}
	data := b.Get([]byte(key))
	if data == nil {
		return empty, nil
	}
	buf := bytes.NewBuffer(data)
	err := binary.Read(buf, binary.LittleEndian, &out)
	if err != nil {
		return 0, err
	}
	return out, nil
}

func (qs *QuadStore) Horizon() graph.PrimaryKey {
	return graph.NewSequentialKey(qs.horizon)
}

func (qs *QuadStore) NameOf(k graph.Value) quad.Value {
	if v, ok := k.(Int64Value); ok {
		if v == 0 {
			if clog.V(2) {
				clog.Infof("k was 0")
			}
			return nil
		}
		var val quad.Value
		err := qs.db.View(func(tx *bolt.Tx) error {
			var err error
			val, err = qs.getValFromLog(tx, uint64(v))
			return err
		})
		if err != nil {
			clog.Errorf("error getting NameOf %d: %s", v, err)
			return nil
		}
		return val
	}
	panic("unknown type of graph.Value; not meant for this quadstore")
}

func (qs *QuadStore) Quad(k graph.Value) quad.Quad {
	var v quad.Quad
	key := k.(Int64Value)
	err := qs.db.View(func(tx *bolt.Tx) error {
		var err error
		v, err = qs.toQuad(tx, uint64(key))
		return err
	})
	if err != nil {
		clog.Errorf("error fetching quad %d: %s", key, err)
	}
	return v
}

func (qs *QuadStore) toQuad(tx *bolt.Tx, k uint64) (quad.Quad, error) {
	q := &quad.Quad{}
	p, err := qs.getPrimitiveFromLog(tx, k)
	if err != nil {
		return *q, err
	}
	for _, dir := range quad.Directions {
		v := p.GetDirection(dir)
		val, err := qs.getValFromLog(tx, v)
		if err != nil {
			return *q, err
		}
		q.Set(dir, val)
	}
	return *q, nil
}

func (qs *QuadStore) getValFromLog(tx *bolt.Tx, k uint64) (quad.Value, error) {
	if k == 0 {
		return nil, nil
	}
	p, err := qs.getPrimitiveFromLog(tx, k)
	if err != nil {
		return nil, err
	}
	return pquads.UnmarshalValue(p.Value)
}

func (qs *QuadStore) ValueOf(s quad.Value) graph.Value {
	var out Int64Value
	qs.db.View(func(tx *bolt.Tx) error {
		out = Int64Value(qs.resolveQuadValue(tx, s))
		return nil
	})
	return out
}

func (qs *QuadStore) Type() string {
	return QuadStoreType
}

func (qs *QuadStore) QuadDirection(val graph.Value, d quad.Direction) graph.Value {
	p, ok := qs.getPrimitive(val.(Int64Value))
	if !ok {
		return Int64Value(0)
	}
	switch d {
	case quad.Subject:
		return Int64Value(p.Subject)
	case quad.Predicate:
		return Int64Value(p.Predicate)
	case quad.Object:
		return Int64Value(p.Object)
	case quad.Label:
		return Int64Value(p.Label)
	}
	return Int64Value(0)
}

func (qs *QuadStore) getPrimitive(val Int64Value) (*graph.Primitive, bool) {
	if val == 0 {
		return nil, false
	}
	var p *graph.Primitive
	err := qs.db.View(func(tx *bolt.Tx) error {
		var err error
		p, err = qs.getPrimitiveFromLog(tx, uint64(val))
		return err
	})
	if err != nil {
		clog.Errorf("error getting primitive %d: %s", val, err)
		return p, false
	}
	return p, true
}

type Int64Value uint64
<|endoftext|>


<commit_before>package main

import "strings"

type SpecialEventParam struct {
	dataList []map[string]string
}

func (evntParam *SpecialEventParam) PutString(key string, value string) {
	newField := make(map[string]string)
	newField[key] = value
	evntParam.dataList = append(evntParam.dataList, newField)
}

func (evntParam *SpecialEventParam) GetString(key string) (value string) {
	for ii := 0; ii < len(evntParam.dataList); ii++ {
		c := evntParam.dataList[ii]
		for k, v := range c {
			if k == key {
				value = v
				return
			}
		}
	}

	return
}

func (evntParam *SpecialEventParam) Extract(data string) {

	dataParts := strings.Split(data, "_")

	for ii := 0; ii < len(dataParts); ii++ {
		varParts := strings.Split(dataParts[ii], "#")
		if len(varParts) == 2 {
			newMap := make(map[string]string)
			newMap[varParts[0]] = varParts[1]
			evntParam.dataList = append(evntParam.dataList, newMap)
		}
	}
}

func (evntParam *SpecialEventParam) Compress() []byte {
	var data string
	data = ""

	for ii := 0; ii < len(evntParam.dataList); ii++ {
		p := evntParam.dataList[ii]
		for k, v := range p {
			data += k + "#" + v + "_"
		}
	}
	data = strings.TrimRight(data, "_")
	bytes := make([]byte, 1024)
	bytes = []byte(data)
	return bytes
}
<commit_msg>Put and Set<commit_after>package main

import (
	"strconv"
	"strings"
)

type SpecialEventParam struct {
	dataList []map[string]string
}

func (evntParam *SpecialEventParam) PutString(key string, value string) {
	newField := make(map[string]string)
	newField[key] = value
	evntParam.dataList = append(evntParam.dataList, newField)
}

func (evntParam *SpecialEventParam) PutFloat(key string, value float64) {
	newField := make(map[string]string)
	newField[key] = strconv.FormatFloat(value, 'f', -1, 64)
	evntParam.dataList = append(evntParam.dataList, newField)
}

func (evntParam *SpecialEventParam) PutInt(key string, value int) {
	newField := make(map[string]string)
	newField[key] = strconv.Itoa(value)
	evntParam.dataList = append(evntParam.dataList, newField)
}

func (evntParam *SpecialEventParam) GetString(key string) (value string) {
	for ii := 0; ii < len(evntParam.dataList); ii++ {
		c := evntParam.dataList[ii]
		for k, v := range c {
			if k == key {
				value = v
				return
			}
		}
	}
	return
}

func (evntParam *SpecialEventParam) GetFloat(key string) (value float64) {
	for ii := 0; ii < len(evntParam.dataList); ii++ {
		c := evntParam.dataList[ii]
		for k, v := range c {
			if k == key {
				value, _ = strconv.ParseFloat(v, 64)
				return
			}
		}
	}
	return
}

func (evntParam *SpecialEventParam) GetInt(key string) (value int) {
	for ii := 0; ii < len(evntParam.dataList); ii++ {
		c := evntParam.dataList[ii]
		for k, v := range c {
			if k == key {
				value, _ = strconv.Atoi(v)
				return
			}
		}
	}
	return
}

func (evntParam *SpecialEventParam) Extract(data string) {

	dataParts := strings.Split(data, "_")

	for ii := 0; ii < len(dataParts); ii++ {
		varParts := strings.Split(dataParts[ii], "#")
		if len(varParts) == 2 {
			newMap := make(map[string]string)
			newMap[varParts[0]] = varParts[1]
			evntParam.dataList = append(evntParam.dataList, newMap)
		}
	}
}

func (evntParam *SpecialEventParam) Compress() []byte {
	var data string
	data = ""

	for ii := 0; ii < len(evntParam.dataList); ii++ {
		p := evntParam.dataList[ii]
		for k, v := range p {
			data += k + "#" + v + "_"
		}
	}
	data = strings.TrimRight(data, "_")
	bytes := make([]byte, 1024)
	bytes = []byte(data)
	return bytes
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/tylerb/graceful"

	"github.com/stripe/sequins/backend"
)

type sequins struct {
	config  sequinsConfig
	http    *http.Server
	backend backend.Backend

	dbs     map[string]*db
	dbsLock sync.RWMutex

	peers     *peers
	zkWatcher *zkWatcher

	refreshLock    sync.Mutex
	refreshWorkers chan bool
	refreshTicker  *time.Ticker
	sighups        chan os.Signal
}

type status struct {
	DBs map[string]dbStatus `json:"dbs"`
}

func newSequins(backend backend.Backend, config sequinsConfig) *sequins {
	return &sequins{
		config:      config,
		backend:     backend,
		refreshLock: sync.Mutex{},
	}
}

func (s *sequins) init() error {
	if s.config.ZK.Servers != nil {
		err := s.initCluster()
		if err != nil {
			return err
		}
	}

	// Create local directories, and load any cached versions we have.
	err := s.initLocalStore()
	if err != nil {
		return nil
	}

	// To limit the number of parallel loads, create a full buffered channel.
	// Workers grab one from the channel when they trigger a load, and put it
	// back when they're done.
	maxLoads := s.config.MaxParallelLoads
	if maxLoads != 0 {
		s.refreshWorkers = make(chan bool, maxLoads)
		for i := 0; i < maxLoads; i++ {
			s.refreshWorkers <- true
		}
	}

	// Trigger loads before we start up.
	s.refreshAll()

	// Automatically refresh, if configured to do so.
	refresh := s.config.RefreshPeriod.Duration
	if refresh != 0 {
		s.refreshTicker = time.NewTicker(refresh)
		go func() {
			log.Println("Automatically checking for new versions every", refresh.String())
			for range s.refreshTicker.C {
				s.refreshAll()
			}
		}()
	}

	// Refresh on SIGHUP.
	sighups := make(chan os.Signal)
	signal.Notify(sighups, syscall.SIGHUP)
	go func() {
		for range sighups {
			s.refreshAll()
		}
	}()

	s.sighups = sighups
	return nil
}

func (s *sequins) initCluster() error {
	prefix := path.Join("/", s.config.ZK.ClusterName)
	zkWatcher, err := connectZookeeper(s.config.ZK.Servers, prefix)
	if err != nil {
		return err
	}

	hostname := s.config.ZK.AdvertisedHostname
	if hostname == "" {
		hostname, err = os.Hostname()
		if err != nil {
			return err
		}
	}

	_, port, err := net.SplitHostPort(s.config.Bind)
	if err != nil {
		return err
	}

	routableAddress := net.JoinHostPort(hostname, port)
	shardID := s.config.ZK.ShardID
	if shardID == "" {
		shardID = routableAddress
	}

	peers := watchPeers(zkWatcher, shardID, routableAddress)
	peers.waitToConverge(s.config.ZK.TimeToConverge.Duration)

	s.zkWatcher = zkWatcher
	s.peers = peers
	return nil
}

func (s *sequins) initLocalStore() error {
	// TODO: lock local filestore
	dataPath := filepath.Join(s.config.LocalStore, "data")
	os.MkdirAll(dataPath, 0755|os.ModeDir)

	// Attempt to load any versions we have cached locally, so that we can start
	// quickly even if there are newer versions to download.
	res, err := ioutil.ReadDir(dataPath)
	if err != nil {
		return err
	}

	localDirs := make(map[string]bool)
	for _, dir := range res {
		localDirs[dir.Name()] = true
	}

	localDBs := make(map[string]*db)
	dbs, err := s.backend.ListDBs()
	if err != nil {
		log.Println("Error listing DBs from %s: %s", s.backend.DisplayPath(""), err)
		return err
	}

	for _, dbName := range dbs {
		if !localDirs[dbName] {
			continue
		}

		db := newDB(s, dbName)
		err := db.loadLatestLocalVersion()
		if err == nil {
			localDBs[dbName] = db
		}
	}

	// TODO: clean up any local data we have that doesn't have a corresponding
	// remote DB or version

	s.dbsLock.Lock()
	s.dbs = localDBs
	s.dbsLock.Unlock()
	return nil
}

func (s *sequins) start() {
	defer s.shutdown()

	log.Println("Listening on", s.config.Bind)
	graceful.Run(s.config.Bind, time.Second, s)
}

func (s *sequins) shutdown() {
	log.Println("Shutting down...")
	signal.Stop(s.sighups)

	if s.refreshTicker != nil {
		s.refreshTicker.Stop()
	}

	zk := s.zkWatcher
	if zk != nil {
		zk.close()
	}

	s.dbsLock.Lock()
	defer s.dbsLock.Unlock()

	for _, db := range s.dbs {
		db.close()
	}
}

func (s *sequins) refreshAll() {
	s.refreshLock.Lock()
	defer s.refreshLock.Unlock()

	dbs, err := s.backend.ListDBs()
	if err != nil {
		log.Println("Error listing DBs from %s: %s", s.backend.DisplayPath(""), err)
		return
	}

	// Add any new DBS, and trigger refreshes for existing ones. This
	// Lock pattern is a bit goofy, since the lock is on the request
	// path and we want to hold it for as short a time as possible.
	s.dbsLock.RLock()

	newDBs := make(map[string]*db)
	for _, name := range dbs {
		db := s.dbs[name]
		if db == nil {
			db = newDB(s, name)
			// TODO: load the latest valid cached version
		}

		go s.refresh(db)
		newDBs[name] = db
	}

	// Now, grab the full lock to switch the new map in.
	s.dbsLock.RUnlock()
	s.dbsLock.Lock()

	oldDBs := s.dbs
	s.dbs = newDBs

	// Finally, close any dbs that we're removing.
	s.dbsLock.Unlock()
	s.dbsLock.RLock()

	for name, db := range oldDBs {
		if s.dbs[name] == nil {
			db.close()
			db.delete()
		}
	}

	s.dbsLock.RUnlock()
}

func (s *sequins) refresh(db *db) {
	if s.refreshWorkers != nil {
		<-s.refreshWorkers
		defer func() {
			s.refreshWorkers <- true
		}()
	}

	err := db.refresh()
	if err != nil {
		log.Printf("Error refreshing %s: %s", db.name, err)
	}
}

func (s *sequins) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	if r.URL.Path == "/" {
		s.serveStatus(w, r)
		return
	}

	var dbName, key string
	path := strings.TrimPrefix(r.URL.Path, "/")
	split := strings.Index(path, "/")
	if split == -1 {
		dbName = path
		key = ""
	} else {
		dbName = path[:split]
		key = path[split+1:]
	}

	if dbName == "" {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	s.dbsLock.RLock()
	db := s.dbs[dbName]
	s.dbsLock.RUnlock()

	// If this is a proxy request, we don't want to confuse "we don't have this
	// db" with "we don't have this key"; if this is a proxied request, then the
	// peer apparently does have the db, and thinks we do too (which should never
	// happen). So we use 501 Not Implemented to indicate the former. To users,
	// we present a uniform 404.
	if db == nil {
		if r.URL.Query().Get("proxy") != "" {
			w.WriteHeader(http.StatusNotImplemented)
		} else {
			w.WriteHeader(http.StatusNotFound)
		}

		return
	}

	db.serveKey(w, r, key)
}

func (s *sequins) serveStatus(w http.ResponseWriter, r *http.Request) {
	s.dbsLock.RLock()

	status := status{DBs: make(map[string]dbStatus)}
	for name, db := range s.dbs {
		status.DBs[name] = db.status()
	}

	s.dbsLock.RUnlock()

	jsonBytes, err := json.Marshal(status)
	if err != nil {
		log.Println("Error serving status:", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.Header()["Content-Type"] = []string{"application/json"}
	w.Write(jsonBytes)
}
<commit_msg>Don't try to close DBs yet; we need a way to preempt<commit_after>package main

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/tylerb/graceful"

	"github.com/stripe/sequins/backend"
)

type sequins struct {
	config  sequinsConfig
	http    *http.Server
	backend backend.Backend

	dbs     map[string]*db
	dbsLock sync.RWMutex

	peers     *peers
	zkWatcher *zkWatcher

	refreshLock    sync.Mutex
	refreshWorkers chan bool
	refreshTicker  *time.Ticker
	sighups        chan os.Signal
}

type status struct {
	DBs map[string]dbStatus `json:"dbs"`
}

func newSequins(backend backend.Backend, config sequinsConfig) *sequins {
	return &sequins{
		config:      config,
		backend:     backend,
		refreshLock: sync.Mutex{},
	}
}

func (s *sequins) init() error {
	if s.config.ZK.Servers != nil {
		err := s.initCluster()
		if err != nil {
			return err
		}
	}

	// Create local directories, and load any cached versions we have.
	err := s.initLocalStore()
	if err != nil {
		return nil
	}

	// To limit the number of parallel loads, create a full buffered channel.
	// Workers grab one from the channel when they trigger a load, and put it
	// back when they're done.
	maxLoads := s.config.MaxParallelLoads
	if maxLoads != 0 {
		s.refreshWorkers = make(chan bool, maxLoads)
		for i := 0; i < maxLoads; i++ {
			s.refreshWorkers <- true
		}
	}

	// Trigger loads before we start up.
	s.refreshAll()

	// Automatically refresh, if configured to do so.
	refresh := s.config.RefreshPeriod.Duration
	if refresh != 0 {
		s.refreshTicker = time.NewTicker(refresh)
		go func() {
			log.Println("Automatically checking for new versions every", refresh.String())
			for range s.refreshTicker.C {
				s.refreshAll()
			}
		}()
	}

	// Refresh on SIGHUP.
	sighups := make(chan os.Signal)
	signal.Notify(sighups, syscall.SIGHUP)
	go func() {
		for range sighups {
			s.refreshAll()
		}
	}()

	s.sighups = sighups
	return nil
}

func (s *sequins) initCluster() error {
	prefix := path.Join("/", s.config.ZK.ClusterName)
	zkWatcher, err := connectZookeeper(s.config.ZK.Servers, prefix)
	if err != nil {
		return err
	}

	hostname := s.config.ZK.AdvertisedHostname
	if hostname == "" {
		hostname, err = os.Hostname()
		if err != nil {
			return err
		}
	}

	_, port, err := net.SplitHostPort(s.config.Bind)
	if err != nil {
		return err
	}

	routableAddress := net.JoinHostPort(hostname, port)
	shardID := s.config.ZK.ShardID
	if shardID == "" {
		shardID = routableAddress
	}

	peers := watchPeers(zkWatcher, shardID, routableAddress)
	peers.waitToConverge(s.config.ZK.TimeToConverge.Duration)

	s.zkWatcher = zkWatcher
	s.peers = peers
	return nil
}

func (s *sequins) initLocalStore() error {
	// TODO: lock local filestore
	dataPath := filepath.Join(s.config.LocalStore, "data")
	os.MkdirAll(dataPath, 0755|os.ModeDir)

	// Attempt to load any versions we have cached locally, so that we can start
	// quickly even if there are newer versions to download.
	res, err := ioutil.ReadDir(dataPath)
	if err != nil {
		return err
	}

	localDirs := make(map[string]bool)
	for _, dir := range res {
		localDirs[dir.Name()] = true
	}

	localDBs := make(map[string]*db)
	dbs, err := s.backend.ListDBs()
	if err != nil {
		log.Println("Error listing DBs from %s: %s", s.backend.DisplayPath(""), err)
		return err
	}

	for _, dbName := range dbs {
		if !localDirs[dbName] {
			continue
		}

		db := newDB(s, dbName)
		err := db.loadLatestLocalVersion()
		if err == nil {
			localDBs[dbName] = db
		}
	}

	// TODO: clean up any local data we have that doesn't have a corresponding
	// remote DB or version

	s.dbsLock.Lock()
	s.dbs = localDBs
	s.dbsLock.Unlock()
	return nil
}

func (s *sequins) start() {
	defer s.shutdown()

	log.Println("Listening on", s.config.Bind)
	graceful.Run(s.config.Bind, time.Second, s)
}

func (s *sequins) shutdown() {
	log.Println("Shutting down...")
	signal.Stop(s.sighups)

	if s.refreshTicker != nil {
		s.refreshTicker.Stop()
	}

	zk := s.zkWatcher
	if zk != nil {
		zk.close()
	}

	// TODO: figure out how to cancel in-progress downloads
	// s.dbsLock.Lock()
	// defer s.dbsLock.Unlock()

	// for _, db := range s.dbs {
	// 	db.close()
	// }
}

func (s *sequins) refreshAll() {
	s.refreshLock.Lock()
	defer s.refreshLock.Unlock()

	dbs, err := s.backend.ListDBs()
	if err != nil {
		log.Println("Error listing DBs from %s: %s", s.backend.DisplayPath(""), err)
		return
	}

	// Add any new DBS, and trigger refreshes for existing ones. This
	// Lock pattern is a bit goofy, since the lock is on the request
	// path and we want to hold it for as short a time as possible.
	s.dbsLock.RLock()

	newDBs := make(map[string]*db)
	for _, name := range dbs {
		db := s.dbs[name]
		if db == nil {
			db = newDB(s, name)
			// TODO: load the latest valid cached version
		}

		go s.refresh(db)
		newDBs[name] = db
	}

	// Now, grab the full lock to switch the new map in.
	s.dbsLock.RUnlock()
	s.dbsLock.Lock()

	oldDBs := s.dbs
	s.dbs = newDBs

	// Finally, close any dbs that we're removing.
	s.dbsLock.Unlock()
	s.dbsLock.RLock()

	for name, db := range oldDBs {
		if s.dbs[name] == nil {
			db.close()
			db.delete()
		}
	}

	s.dbsLock.RUnlock()
}

func (s *sequins) refresh(db *db) {
	if s.refreshWorkers != nil {
		<-s.refreshWorkers
		defer func() {
			s.refreshWorkers <- true
		}()
	}

	err := db.refresh()
	if err != nil {
		log.Printf("Error refreshing %s: %s", db.name, err)
	}
}

func (s *sequins) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	if r.URL.Path == "/" {
		s.serveStatus(w, r)
		return
	}

	var dbName, key string
	path := strings.TrimPrefix(r.URL.Path, "/")
	split := strings.Index(path, "/")
	if split == -1 {
		dbName = path
		key = ""
	} else {
		dbName = path[:split]
		key = path[split+1:]
	}

	if dbName == "" {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	s.dbsLock.RLock()
	db := s.dbs[dbName]
	s.dbsLock.RUnlock()

	// If this is a proxy request, we don't want to confuse "we don't have this
	// db" with "we don't have this key"; if this is a proxied request, then the
	// peer apparently does have the db, and thinks we do too (which should never
	// happen). So we use 501 Not Implemented to indicate the former. To users,
	// we present a uniform 404.
	if db == nil {
		if r.URL.Query().Get("proxy") != "" {
			w.WriteHeader(http.StatusNotImplemented)
		} else {
			w.WriteHeader(http.StatusNotFound)
		}

		return
	}

	db.serveKey(w, r, key)
}

func (s *sequins) serveStatus(w http.ResponseWriter, r *http.Request) {
	s.dbsLock.RLock()

	status := status{DBs: make(map[string]dbStatus)}
	for name, db := range s.dbs {
		status.DBs[name] = db.status()
	}

	s.dbsLock.RUnlock()

	jsonBytes, err := json.Marshal(status)
	if err != nil {
		log.Println("Error serving status:", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.Header()["Content-Type"] = []string{"application/json"}
	w.Write(jsonBytes)
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"log"
	"net/http"
	"strings"
)

type pageResult struct {
	Error error
}

func redirectToFile(w http.ResponseWriter, r *http.Request, release Release) {
	url := ""
	if release.DownloadUrl() != "" {
		println("DownloadUrl exists")
		url = release.DownloadUrl()
	} else {
		url = "/file/" + release.Path()
	}
	log.Println("Redirecting to: " + url)
	http.Redirect(w, r, url, http.StatusFound)
	return
}

func downloadPage(url, file string) pageResult {
	log.Println("File doesn't exist yet. Downloading: " + url)
	res, err := http.Get(url)
	if err != nil {
		return pageResult{Error: err}
	}

	if file != "" {
		body, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return pageResult{Error: err}
		}
		err = ioutil.WriteFile(file, body, 0644)
		if err != nil {
			return pageResult{Error: err}
		}
	}
	return pageResult{Error: nil}
}

// If a file exists, just serve it directly. Otherwise, download, then serve.
func cachedFileHandler(w http.ResponseWriter, r *http.Request) {
	paths := strings.Split(r.URL.Path, "/")
	name := paths[len(paths)-1]
	log.Println("Request for cached file: " + name)
	release := Release{
		Name:     r.FormValue("package"),
		Version:  r.FormValue("release"),
		DataDir:  Config.DataDir,
		Filename: name,
	}

	if release.Exists() {
		log.Println(release.Filename + " already exists. Redirecting to download.")
		redirectToFile(w, r, release)
		return
	}

	original := r.FormValue("original")
	res := downloadPage(original, release.ReleaseFilePath())

	if res.Error != nil {
		http.Error(w, "Unable to download page", http.StatusInternalServerError)
		return
	}

	redirectToFile(w, r, release)
}
<commit_msg>Add some logging for download errors<commit_after>package main

import (
	"io/ioutil"
	"log"
	"net/http"
	"strings"
)

type pageResult struct {
	Error error
}

func redirectToFile(w http.ResponseWriter, r *http.Request, release Release) {
	url := ""
	if release.DownloadUrl() != "" {
		println("DownloadUrl exists")
		url = release.DownloadUrl()
	} else {
		url = "/file/" + release.Path()
	}
	log.Println("Redirecting to: " + url)
	http.Redirect(w, r, url, http.StatusFound)
	return
}

func downloadPage(url, file string) pageResult {
	log.Println("File doesn't exist yet. Downloading: " + url)
	res, err := http.Get(url)
	if err != nil {
		log.Println("Error downloading: " + url)
		return pageResult{Error: err}
	}

	if file != "" {
		body, err := ioutil.ReadAll(res.Body)
		if err != nil {
			log.Println("Error reading body: " + url)
			return pageResult{Error: err}
		}
		err = ioutil.WriteFile(file, body, 0644)
		if err != nil {
			log.Println("Error writing file: " + file)
			return pageResult{Error: err}
		}
	}
	return pageResult{Error: nil}
}

// If a file exists, just serve it directly. Otherwise, download, then serve.
func cachedFileHandler(w http.ResponseWriter, r *http.Request) {
	paths := strings.Split(r.URL.Path, "/")
	name := paths[len(paths)-1]
	log.Println("Request for cached file: " + name)
	release := Release{
		Name:     r.FormValue("package"),
		Version:  r.FormValue("release"),
		DataDir:  Config.DataDir,
		Filename: name,
	}

	if release.Exists() {
		log.Println(release.Filename + " already exists. Redirecting to download.")
		redirectToFile(w, r, release)
		return
	}

	original := r.FormValue("original")
	res := downloadPage(original, release.ReleaseFilePath())

	if res.Error != nil {
		http.Error(w, "Unable to download page", http.StatusInternalServerError)
		return
	}

	redirectToFile(w, r, release)
}
<|endoftext|>


<commit_before>package elastic_journald

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os/exec"
	"strings"
	"time"
	"unsafe"

	"github.com/mattbaird/elastigo/lib"
)

// #include <stdio.h>
// #include <string.h>
// #include <systemd/sd-journal.h>
// #cgo LDFLAGS: -lsystemd
import "C"

type Config struct {
	Hosts       elasticHostsType
	IndexPrefix string
}

type Service struct {
	Config  *Config
	Journal *C.sd_journal
	Cursor  string
	Elastic *elastigo.Conn
	Indexer *elastigo.BulkIndexer
}

func NewService() *Service {
	config := &Config{
		Hosts:       elasticHosts,
		IndexPrefix: *elasticPrefix,
	}

	elastic := elastigo.NewConn()
	indexer := elastic.NewBulkIndexerErrors(3, 10)
	indexer.Sender = func(buf *bytes.Buffer) error {
		fmt.Printf("Sending %v entries\n", indexer.PendingDocuments())
		respJson, err := elastic.DoCommand("POST", "/_bulk", nil, buf)
		if err != nil {
			// TODO
			panic(fmt.Sprintf("Bulk error: \n%v", err))
		} else {
			response := struct {
				Took   int64 `json:"took"`
				Errors bool  `json:"errors"`
				Items  []struct {
					Index struct {
						Id string `json:"_id"`
					} `json:"index"`
				} `json:"items"`
			}{}

			jsonErr := json.Unmarshal(respJson, &response)
			if jsonErr != nil {
				// TODO
				panic(jsonErr)
			}

			lastStoredCursor := response.Items[len(response.Items)-1].Index.Id
			ioutil.WriteFile(*elasticCursorFile, []byte(lastStoredCursor), 0644)
		}
		return err
	}

	service := &Service{
		Config:  config,
		Elastic: elastic,
		Indexer: indexer,
	}
	return service
}

func (s *Service) Run() {
	s.Elastic.SetHosts(s.Config.Hosts)

	s.InitJournal()
	s.ProcessStream(GetFQDN())
}

func (s *Service) ProcessStream(hostname *string) {
	s.Indexer.Start()
	defer s.Indexer.Stop()

	for {
		r := C.sd_journal_next(s.Journal)
		if r < 0 {
			panic(fmt.Sprintf("failed to iterate to next entry: %s", C.strerror(-r)))
		}
		if r == 0 {
			r = C.sd_journal_wait(s.Journal, 1000000)
			if r < 0 {
				panic(fmt.Sprintf("failed to wait for changes: %s", C.strerror(-r)))
			}
			continue
		}
		s.ProcessEntry(hostname)
	}
}

func (s *Service) ProcessEntry(hostname *string) {
	var realtime C.uint64_t
	r := C.sd_journal_get_realtime_usec(s.Journal, &realtime)
	if r < 0 {
		panic(fmt.Sprintf("failed to get realtime timestamp: %s", C.strerror(-r)))
	}

	var cursor *C.char
	r = C.sd_journal_get_cursor(s.Journal, &cursor)
	if r < 0 {
		panic(fmt.Sprintf("failed to get cursor: %s", C.strerror(-r)))
	}

	row := make(map[string]interface{})

	timestamp := time.Unix(int64(realtime/1000000), int64(realtime%1000000)).UTC()

	row["ts"] = timestamp.Format("2006-01-02T15:04:05Z")
	row["host"] = hostname
	s.ProcessEntryFields(row)

	message, _ := json.Marshal(row)
	indexName := fmt.Sprintf("%v-%v", s.Config.IndexPrefix, timestamp.Format("2006-01-02"))
	cursorId := C.GoString(cursor)

	s.Indexer.Index(
		indexName,       // index
		"journal",       // type
		cursorId,        // id
		"",              // ttl
		nil,             // date
		string(message), // content
		false)           // immediate index refresh
}

func (s *Service) ProcessEntryFields(row map[string]interface{}) {
	var length C.size_t
	var cData *C.char

	for C.sd_journal_restart_data(s.Journal); C.sd_journal_enumerate_data(s.Journal, (*unsafe.Pointer)(unsafe.Pointer(&cData)), &length) > 0; {
		data := C.GoString(cData)

		parts := strings.SplitN(data, "=", 2)

		key := strings.ToLower(parts[0])
		value := parts[1]

		switch key {
		// don't index bloat
		case "_cap_effective":
		case "_cmdline":
		case "_exe":
		case "_hostname":
		case "_systemd_cgroup":
		case "_systemd_slice":
		case "_transport":
		case "syslog_facility":
		case "syslog_identifier":
			continue
		default:
			row[strings.TrimPrefix(key, "_")] = value
		}
	}
}

func (s *Service) InitJournal() {
	r := C.sd_journal_open(&s.Journal, C.SD_JOURNAL_LOCAL_ONLY)
	if r < 0 {
		panic(fmt.Sprintf("failed to open journal: %s", C.strerror(-r)))
	}

	bytes, err := ioutil.ReadFile(*elasticCursorFile)
	if err == nil {
		s.Cursor = string(bytes)
	}

	if s.Cursor != "" {
		r = C.sd_journal_seek_cursor(s.Journal, C.CString(s.Cursor))
		if r < 0 {
			panic(fmt.Sprintf("failed to seek journal: %s", C.strerror(-r)))
		}
		r = C.sd_journal_next_skip(s.Journal, 1)
		if r < 0 {
			panic(fmt.Sprintf("failed to skip current journal entry: %s", C.strerror(-r)))
		}
	}
}

func GetFQDN() *string {
	cmd := exec.Command("hostname", "-f")
	var out bytes.Buffer
	cmd.Stdout = &out
	err := cmd.Run()
	if err != nil {
		return nil
	}
	fqdn := string(bytes.TrimSpace(out.Bytes()))
	return &fqdn
}

type elasticHostsType []string

func (e *elasticHostsType) String() string {
	return strings.Join(*e, ",")
}

func (e *elasticHostsType) Set(value string) error {
	for _, host := range strings.Split(value, ",") {
		*e = append(*e, host)
	}
	return nil
}

var elasticCursorFile = flag.String("cursor", ".elastic_journal_cursor", "The file to keep cursor state between runs")
var elasticHosts elasticHostsType
var elasticPrefix = flag.String("prefix", "journald", "The index prefix to use")

func init() {
	flag.Var(&elasticHosts, "hosts", "comma-separated list of elastic (target) hosts")
}
<commit_msg>minor fixes to sanity and output<commit_after>package elastic_journald

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os/exec"
	"strings"
	"time"
	"unsafe"

	"github.com/mattbaird/elastigo/lib"
)

// #include <stdio.h>
// #include <string.h>
// #include <systemd/sd-journal.h>
// #cgo LDFLAGS: -lsystemd
import "C"

type Config struct {
	Hosts       elasticHostsType
	IndexPrefix string
}

type Service struct {
	Config  *Config
	Journal *C.sd_journal
	Cursor  string
	Elastic *elastigo.Conn
	Indexer *elastigo.BulkIndexer
}

func NewService() *Service {
	config := &Config{
		Hosts:       elasticHosts,
		IndexPrefix: *elasticPrefix,
	}

	elastic := elastigo.NewConn()
	indexer := elastic.NewBulkIndexerErrors(3, 10)
	indexer.Sender = func(buf *bytes.Buffer) error {
		respJson, err := elastic.DoCommand("POST", "/_bulk", nil, buf)
		if err != nil {
			// TODO
			panic(fmt.Sprintf("Bulk error: \n%v", err))
		} else {
			response := struct {
				Took   int64 `json:"took"`
				Errors bool  `json:"errors"`
				Items  []struct {
					Index struct {
						Id string `json:"_id"`
					} `json:"index"`
				} `json:"items"`
			}{}

			jsonErr := json.Unmarshal(respJson, &response)
			if jsonErr != nil {
				// TODO
				panic(jsonErr)
			}
			if response.Errors {
				// TODO
				panic("elasticsearch reported errors on intake")
			}

			messagesStored := len(response.Items)
			lastStoredCursor := response.Items[messagesStored-1].Index.Id
			ioutil.WriteFile(*elasticCursorFile, []byte(lastStoredCursor), 0644)

			fmt.Printf("Sent %v entries\n", messagesStored)
		}
		return err
	}

	service := &Service{
		Config:  config,
		Elastic: elastic,
		Indexer: indexer,
	}
	return service
}

func (s *Service) Run() {
	s.Elastic.SetHosts(s.Config.Hosts)

	s.InitJournal()
	s.ProcessStream(GetFQDN())
}

func (s *Service) ProcessStream(hostname *string) {
	s.Indexer.Start()
	defer s.Indexer.Stop()

	for {
		r := C.sd_journal_next(s.Journal)
		if r < 0 {
			panic(fmt.Sprintf("failed to iterate to next entry: %s", C.strerror(-r)))
		}
		if r == 0 {
			r = C.sd_journal_wait(s.Journal, 1000000)
			if r < 0 {
				panic(fmt.Sprintf("failed to wait for changes: %s", C.strerror(-r)))
			}
			continue
		}
		s.ProcessEntry(hostname)
	}
}

func (s *Service) ProcessEntry(hostname *string) {
	var realtime C.uint64_t
	r := C.sd_journal_get_realtime_usec(s.Journal, &realtime)
	if r < 0 {
		panic(fmt.Sprintf("failed to get realtime timestamp: %s", C.strerror(-r)))
	}

	var cursor *C.char
	r = C.sd_journal_get_cursor(s.Journal, &cursor)
	if r < 0 {
		panic(fmt.Sprintf("failed to get cursor: %s", C.strerror(-r)))
	}

	row := make(map[string]interface{})

	timestamp := time.Unix(int64(realtime/1000000), int64(realtime%1000000)).UTC()

	row["ts"] = timestamp.Format("2006-01-02T15:04:05Z")
	row["host"] = hostname
	s.ProcessEntryFields(row)

	message, _ := json.Marshal(row)
	indexName := fmt.Sprintf("%v-%v", s.Config.IndexPrefix, timestamp.Format("2006-01-02"))
	cursorId := C.GoString(cursor)

	s.Indexer.Index(
		indexName,       // index
		"journal",       // type
		cursorId,        // id
		"",              // ttl
		nil,             // date
		string(message), // content
		false)           // immediate index refresh
}

func (s *Service) ProcessEntryFields(row map[string]interface{}) {
	var length C.size_t
	var cData *C.char

	for C.sd_journal_restart_data(s.Journal); C.sd_journal_enumerate_data(s.Journal, (*unsafe.Pointer)(unsafe.Pointer(&cData)), &length) > 0; {
		data := C.GoString(cData)

		parts := strings.SplitN(data, "=", 2)

		key := strings.ToLower(parts[0])
		value := parts[1]

		switch key {
		// don't index bloat
		case "_cap_effective":
		case "_cmdline":
		case "_exe":
		case "_hostname":
		case "_systemd_cgroup":
		case "_systemd_slice":
		case "_transport":
		case "syslog_facility":
		case "syslog_identifier":
			continue
		default:
			row[strings.TrimPrefix(key, "_")] = value
		}
	}
}

func (s *Service) InitJournal() {
	r := C.sd_journal_open(&s.Journal, C.SD_JOURNAL_LOCAL_ONLY)
	if r < 0 {
		panic(fmt.Sprintf("failed to open journal: %s", C.strerror(-r)))
	}

	bytes, err := ioutil.ReadFile(*elasticCursorFile)
	if err == nil {
		s.Cursor = string(bytes)
	}

	if s.Cursor != "" {
		r = C.sd_journal_seek_cursor(s.Journal, C.CString(s.Cursor))
		if r < 0 {
			panic(fmt.Sprintf("failed to seek journal: %s", C.strerror(-r)))
		}
		r = C.sd_journal_next_skip(s.Journal, 1)
		if r < 0 {
			panic(fmt.Sprintf("failed to skip current journal entry: %s", C.strerror(-r)))
		}
	}
}

func GetFQDN() *string {
	cmd := exec.Command("hostname", "-f")
	var out bytes.Buffer
	cmd.Stdout = &out
	err := cmd.Run()
	if err != nil {
		return nil
	}
	fqdn := string(bytes.TrimSpace(out.Bytes()))
	return &fqdn
}

type elasticHostsType []string

func (e *elasticHostsType) String() string {
	return strings.Join(*e, ",")
}

func (e *elasticHostsType) Set(value string) error {
	for _, host := range strings.Split(value, ",") {
		*e = append(*e, host)
	}
	return nil
}

var elasticCursorFile = flag.String("cursor", ".elastic_journal_cursor", "The file to keep cursor state between runs")
var elasticHosts elasticHostsType
var elasticPrefix = flag.String("prefix", "journald", "The index prefix to use")

func init() {
	flag.Var(&elasticHosts, "hosts", "comma-separated list of elastic (target) hosts")
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"github.com/blang/procd"
	"os"
	"path"
	"strconv"
	"strings"
)

type LaunchConfig struct {
	Name      string `json:"name"`
	Game      string `json:"game"`
	Modstring string `json:"modstring"`
	Betamod   string `json:"betamod"`
	Host      string `json:"host"`
	Port      int    `json:"port"`
	Password  string `json:"password"`
}

type Service struct {
	store   *Store
	process *procd.Process
}

func NewService(store *Store) *Service {
	return &Service{
		store: store,
	}
}

func (s *Service) Config() *Config {
	return &(s.store.Config)
}

func (s *Service) Launch(conf *LaunchConfig) error {
	if s.process != nil && s.process.State() != procd.RunStateStopped {
		return errors.New("Instance already running")
	}
	switch conf.Game {
	case "a2":
		err := s.checkArma2OA()
		if err != nil {
			return err
		}
		cwd, exe, args := s.compileParams(s.Config().Arma2OAPath, "arma2oa.exe", s.Config().Arma2Params, s.Config().Arma2Profile, conf)
		s.process = procd.NewProcess(cwd, exe, args)
		s.process.Start()
	case "a3":
		err := s.checkArma3()
		if err != nil {
			return err
		}
		cwd, exe, args := s.compileParams(s.Config().Arma3Path, "arma3.exe", s.Config().Arma3Params, s.Config().Arma3Profile, conf)
		s.process = procd.NewProcess(cwd, exe, args)
		s.process.Start()
		return errors.New("Not yet implemented")
	default:
		return errors.New("Unknown gametype")
	}
	return nil
}

func (s *Service) Shutdown() {
	if s.process != nil && s.process.State() != procd.RunStateStopped {
		s.process.Stop()
		s.process.Wait()
	}
}

func (s *Service) checkArma2OA() error {
	if s.Config().Arma2OAPath == "" {
		return errors.New("Arma2OAPath not configured")
	}
	fi, err := os.Stat(s.Config().Arma2OAPath)
	if err != nil {
		return fmt.Errorf("Arma2OA Path not exists: %q", s.Config().Arma2OAPath)
	}
	if !fi.IsDir() {
		return fmt.Errorf("Arma2OA Path is not a directory: %q", s.Config().Arma2OAPath)
	}
	exePath := path.Join(s.Config().Arma2OAPath, "arma2oa.exe")
	fi, err = os.Stat(exePath)
	if err != nil {
		return fmt.Errorf("Arma2OA Executable path not exists: %q", exePath)
	}
	if !(fi.Size() > 0) {
		return fmt.Errorf("Arma2OA Executable not exists: ", exePath)
	}

	return nil
}

func (s *Service) checkArma3() error {
	if s.Config().Arma3Path == "" {
		return errors.New("Arma3Path not configured")
	}
	fi, err := os.Stat(s.Config().Arma3Path)
	if err != nil {
		return fmt.Errorf("Arma3 Path not exists: %q", s.Config().Arma3Path)
	}
	if !fi.IsDir() {
		return fmt.Errorf("Arma3 Path is not a directory: %q", s.Config().Arma3Path)
	}
	exePath := path.Join(s.Config().Arma3Path, "arma3.exe")
	fi, err = os.Stat(exePath)
	if err != nil {
		return fmt.Errorf("Arma3 Executable path not exists: %q", exePath)
	}
	if !(fi.Size() > 0) {
		return fmt.Errorf("Arma3 Executable not exists: ", exePath)
	}

	return nil
}

func (s *Service) compileParams(cwd, exe string, addParams string, profile string, conf *LaunchConfig) (string, string, []string) {
	startPath := path.Join(cwd, exe)
	if conf.Betamod != "" {
		startPath = path.Join(cwd, conf.Betamod, exe)
	}

	args := []string{
		"-nosplash",
		"-world=empty",
		"-pause",
		"-connect=" + conf.Host,
		"-port=" + strconv.Itoa(conf.Port),
		"-password=" + conf.Password,
		"-name=" + profile,
		"-mod=" + conf.Modstring,
	}
	args = append(args, strings.Split(addParams, "|")...)
	return cwd, startPath, args
}
<commit_msg>Fix a3 start error msg<commit_after>package main

import (
	"errors"
	"fmt"
	"github.com/blang/procd"
	"os"
	"path"
	"strconv"
	"strings"
)

type LaunchConfig struct {
	Name      string `json:"name"`
	Game      string `json:"game"`
	Modstring string `json:"modstring"`
	Betamod   string `json:"betamod"`
	Host      string `json:"host"`
	Port      int    `json:"port"`
	Password  string `json:"password"`
}

type Service struct {
	store   *Store
	process *procd.Process
}

func NewService(store *Store) *Service {
	return &Service{
		store: store,
	}
}

func (s *Service) Config() *Config {
	return &(s.store.Config)
}

func (s *Service) Launch(conf *LaunchConfig) error {
	if s.process != nil && s.process.State() != procd.RunStateStopped {
		return errors.New("Instance already running")
	}
	switch conf.Game {
	case "a2":
		err := s.checkArma2OA()
		if err != nil {
			return err
		}
		cwd, exe, args := s.compileParams(s.Config().Arma2OAPath, "arma2oa.exe", s.Config().Arma2Params, s.Config().Arma2Profile, conf)
		s.process = procd.NewProcess(cwd, exe, args)
		s.process.Start()
	case "a3":
		err := s.checkArma3()
		if err != nil {
			return err
		}
		cwd, exe, args := s.compileParams(s.Config().Arma3Path, "arma3.exe", s.Config().Arma3Params, s.Config().Arma3Profile, conf)
		s.process = procd.NewProcess(cwd, exe, args)
		s.process.Start()
	default:
		return errors.New("Unknown gametype")
	}
	return nil
}

func (s *Service) Shutdown() {
	if s.process != nil && s.process.State() != procd.RunStateStopped {
		s.process.Stop()
		s.process.Wait()
	}
}

func (s *Service) checkArma2OA() error {
	if s.Config().Arma2OAPath == "" {
		return errors.New("Arma2OAPath not configured")
	}
	fi, err := os.Stat(s.Config().Arma2OAPath)
	if err != nil {
		return fmt.Errorf("Arma2OA Path not exists: %q", s.Config().Arma2OAPath)
	}
	if !fi.IsDir() {
		return fmt.Errorf("Arma2OA Path is not a directory: %q", s.Config().Arma2OAPath)
	}
	exePath := path.Join(s.Config().Arma2OAPath, "arma2oa.exe")
	fi, err = os.Stat(exePath)
	if err != nil {
		return fmt.Errorf("Arma2OA Executable path not exists: %q", exePath)
	}
	if !(fi.Size() > 0) {
		return fmt.Errorf("Arma2OA Executable not exists: ", exePath)
	}

	return nil
}

func (s *Service) checkArma3() error {
	if s.Config().Arma3Path == "" {
		return errors.New("Arma3Path not configured")
	}
	fi, err := os.Stat(s.Config().Arma3Path)
	if err != nil {
		return fmt.Errorf("Arma3 Path not exists: %q", s.Config().Arma3Path)
	}
	if !fi.IsDir() {
		return fmt.Errorf("Arma3 Path is not a directory: %q", s.Config().Arma3Path)
	}
	exePath := path.Join(s.Config().Arma3Path, "arma3.exe")
	fi, err = os.Stat(exePath)
	if err != nil {
		return fmt.Errorf("Arma3 Executable path not exists: %q", exePath)
	}
	if !(fi.Size() > 0) {
		return fmt.Errorf("Arma3 Executable not exists: ", exePath)
	}

	return nil
}

func (s *Service) compileParams(cwd, exe string, addParams string, profile string, conf *LaunchConfig) (string, string, []string) {
	startPath := path.Join(cwd, exe)
	if conf.Betamod != "" {
		startPath = path.Join(cwd, conf.Betamod, exe)
	}

	args := []string{
		"-nosplash",
		"-world=empty",
		"-pause",
		"-connect=" + conf.Host,
		"-port=" + strconv.Itoa(conf.Port),
		"-password=" + conf.Password,
		"-name=" + profile,
		"-mod=" + conf.Modstring,
	}
	args = append(args, strings.Split(addParams, "|")...)
	return cwd, startPath, args
}
<|endoftext|>


<commit_before>package pluto

import (
	"context"
	"errors"
	"fmt"
	"os"
	"os/signal"
	"sync"
	"syscall"
	"time"

	"github.com/aukbit/fibonacci"
	"github.com/aukbit/pluto/client"
	"github.com/aukbit/pluto/common"
	"github.com/aukbit/pluto/server"
	"github.com/aukbit/pluto/server/router"
	"github.com/rs/zerolog"
	"google.golang.org/grpc/health"
	healthpb "google.golang.org/grpc/health/grpc_health_v1"
)

const (
	defaultName       = "pluto"
	defaultHealthAddr = ":9090"
)

var (
	ErrDatastoreNotInitialized = errors.New("datastore not initialized")
)

// Service representacion of a pluto service
type Service struct {
	close  chan bool
	cfg    Config
	wg     *sync.WaitGroup
	health *health.Server
	logger zerolog.Logger
}

func init() {
	zerolog.TimestampFieldName = "timestamp"
	zerolog.TimeFieldFormat = time.RFC3339Nano
	zerolog.LevelFieldName = "severity"
	zerolog.MessageFieldName = "message"
	switch os.Getenv("LOGGER_LEVEL") {
	case "debug":
		zerolog.SetGlobalLevel(zerolog.DebugLevel)
	default:
		zerolog.SetGlobalLevel(zerolog.InfoLevel)
	}

}

// New returns a new pluto service with Options passed in
func New(opts ...Option) *Service {
	return newService(opts...)
}

func newService(opts ...Option) *Service {
	s := &Service{
		cfg:    newConfig(),
		close:  make(chan bool),
		wg:     &sync.WaitGroup{},
		health: health.NewServer(),
	}
	s.logger = zerolog.New(os.Stderr).With().Timestamp().Logger()
	if len(opts) > 0 {
		s = s.WithOptions(opts...)
	}
	return s
}

// WithOptions clones the current Service, applies the supplied Options, and
// returns the resulting Service. It's safe to use concurrently.
func (s *Service) WithOptions(opts ...Option) *Service {
	for _, opt := range opts {
		opt.apply(s)
	}
	return s
}

// Run starts service
func (s *Service) Run() error {
	s.logger = s.logger.With().Str("id", s.cfg.ID).Str("name", s.cfg.Name).Logger()
	// set health server
	s.setHealthServer()
	// start service
	if err := s.start(); err != nil {
		return err
	}
	// hook run after start
	if err := s.hookAfterStart(); err != nil {
		return err
	}
	// wait for all go routines to finish
	s.wg.Wait()
	s.logger.Warn().Msg(fmt.Sprintf("%s has just exited", s.Name()))
	return nil
}

// Stop stops service
func (s *Service) Stop() {
	s.logger.Info().Msg(fmt.Sprintf("shutting down %s", s.Name()))
	s.close <- true
}

// Push allows to start additional options while service is running
func (s *Service) Push(opts ...Option) {
	for _, opt := range opts {
		opt.apply(s)
	}
}

// Server returns a server instance by name if initialized in service
func (s *Service) Server(name string) (srv *server.Server, ok bool) {
	name = common.SafeName(name, server.DefaultName)
	if srv, ok = s.cfg.Servers[name]; !ok {
		return
	}
	return srv, true
}

// Client returns a client instance by name if initialized in service
func (s *Service) Client(name string) (clt *client.Client, ok bool) {
	name = common.SafeName(name, client.DefaultName)
	if clt, ok = s.cfg.Clients[name]; !ok {
		return
	}
	return clt, true
}

// Health ...
func (s *Service) Health() *healthpb.HealthCheckResponse {
	hcr, err := s.health.Check(
		context.Background(), &healthpb.HealthCheckRequest{Service: s.cfg.ID})
	if err != nil {
		s.logger.Error().Msg(err.Error())
	}
	return hcr
}

// Name returns service name
func (s *Service) Name() string {
	return s.cfg.Name
}

// Logger returns service logger
func (s *Service) Logger() zerolog.Logger {
	return s.logger
}

func (s *Service) setHealthServer() {
	s.health.SetServingStatus(s.cfg.ID, 1)
	// Define Router
	mux := router.New()
	mux.GET("/_health/:module/:name", healthHandler)
	// Define server
	srv := server.New(
		server.Name(s.cfg.Name+"_health"),
		server.Addr(s.cfg.HealthAddr),
		server.Mux(mux),
		// server.Logger(s.logger),
	)
	s.cfg.Servers[srv.Name()] = srv
}

func (s *Service) start() error {
	s.logger.Info().Str("ip4", common.IPaddress()).
		Int("servers", len(s.cfg.Servers)).
		Int("clients", len(s.cfg.Clients)).
		Msg(fmt.Sprintf("starting %s, servers: %d clients: %d", s.Name(), len(s.cfg.Servers), len(s.cfg.Clients)))
	// run servers
	s.startServers()
	// dial clients
	s.startClients()
	// add go routine to WaitGroup
	s.wg.Add(1)
	go s.waitUntilStopOrSig()
	return nil
}

func (s *Service) hookAfterStart() error {
	hooks, ok := s.cfg.Hooks["after_start"]
	if !ok {
		return nil
	}
	ctx := context.Background()
	// make service available in hooks context
	ctx = s.WithContext(ctx)
	// make logger available in hooks context
	sublogger := s.logger.With().Str("hook", "after_start").Logger()
	ctx = sublogger.WithContext(ctx)
	for _, h := range hooks {
		if err := h(ctx); err != nil {
			return err
		}
	}
	return nil
}

func (s *Service) startServers() {
	for _, srv := range s.cfg.Servers {
		// add go routine to WaitGroup
		s.wg.Add(1)
		go func(s *Service, srv *server.Server) {
			defer s.wg.Done()
			f := fibonacci.F()
			for {
				err := srv.Run(
					server.Middlewares(
						serviceContextMiddleware(s),
					),
					server.UnaryServerInterceptors(
						serviceContextUnaryServerInterceptor(s),
					),
					server.StreamServerInterceptors(
						serviceContextStreamServerInterceptore(s),
					),
					server.Logger(s.logger),
				)
				if err == nil {
					return
				}
				l := srv.Logger()
				l.Error().Msg(fmt.Sprintf("%v failed to start, error: %v", srv.Name(), err.Error()))
				time.Sleep(time.Duration(f()) * time.Second)
			}
		}(s, srv)
	}
}

// startClients listen to the clientsCh
func (s *Service) startClients() {
	go func() {
		for {
			select {
			case clt, ok := <-s.cfg.clientsCh:
				if !ok {
					break
				}
				s.startClient(clt)
			default:
				time.Sleep(500 * time.Millisecond)
				continue
			}
		}
	}()
}

func (s *Service) startClient(clt *client.Client) {
	go func(clt *client.Client) {
		clt.Init(client.Logger(s.logger))
	}(clt)
}

// waitUntilStopOrSig waits for close channel or syscall Signal
func (s *Service) waitUntilStopOrSig() {
	defer s.wg.Done()
	//  Stop also in case of any host signal
	sigch := make(chan os.Signal, 1)
	signal.Notify(sigch, syscall.SIGTERM, syscall.SIGINT)

outer:
	for {
		select {
		case <-s.close:
			// Waits for call to stop
			s.health.SetServingStatus(s.cfg.ID, 2)
			s.unregister()
			s.closeClients()
			s.stopServers()
			break outer
		case sig := <-sigch:
			// Waits for signal to stop
			s.logger.Info().Msg(fmt.Sprintf("shutting down, got signal: %v", sig))
			s.health.SetServingStatus(s.cfg.ID, 2)
			s.unregister()
			s.closeClients()
			s.stopServers()
			break outer
		default:
			// keep on looping, non-blocking channel operations
			time.Sleep(50 * time.Millisecond)
			continue
		}
	}
}

func (s *Service) closeClients() {
	close(s.cfg.clientsCh)
	// for _, clt := range s.cfg.Clients {
	// 	// add go routine to WaitGroup
	// 	s.wg.Add(1)
	// 	go func(clt *client.Client) {
	// 		defer s.wg.Done()
	// 		clt.Close()
	// 	}(clt)
	// }
}

func (s *Service) stopServers() {
	for _, srv := range s.cfg.Servers {
		// add go routine to WaitGroup
		s.wg.Add(1)
		go func(s *Service, srv *server.Server) {
			defer s.wg.Done()
			srv.Stop()
		}(s, srv)
	}
}
<commit_msg>allow access to service ID<commit_after>package pluto

import (
	"context"
	"errors"
	"fmt"
	"os"
	"os/signal"
	"sync"
	"syscall"
	"time"

	"github.com/aukbit/fibonacci"
	"github.com/aukbit/pluto/client"
	"github.com/aukbit/pluto/common"
	"github.com/aukbit/pluto/server"
	"github.com/aukbit/pluto/server/router"
	"github.com/rs/zerolog"
	"google.golang.org/grpc/health"
	healthpb "google.golang.org/grpc/health/grpc_health_v1"
)

const (
	defaultName       = "pluto"
	defaultHealthAddr = ":9090"
)

var (
	ErrDatastoreNotInitialized = errors.New("datastore not initialized")
)

// Service representacion of a pluto service
type Service struct {
	close  chan bool
	cfg    Config
	wg     *sync.WaitGroup
	health *health.Server
	logger zerolog.Logger
}

func init() {
	zerolog.TimestampFieldName = "timestamp"
	zerolog.TimeFieldFormat = time.RFC3339Nano
	zerolog.LevelFieldName = "severity"
	zerolog.MessageFieldName = "message"
	switch os.Getenv("LOGGER_LEVEL") {
	case "debug":
		zerolog.SetGlobalLevel(zerolog.DebugLevel)
	default:
		zerolog.SetGlobalLevel(zerolog.InfoLevel)
	}

}

// New returns a new pluto service with Options passed in
func New(opts ...Option) *Service {
	return newService(opts...)
}

func newService(opts ...Option) *Service {
	s := &Service{
		cfg:    newConfig(),
		close:  make(chan bool),
		wg:     &sync.WaitGroup{},
		health: health.NewServer(),
	}
	s.logger = zerolog.New(os.Stderr).With().Timestamp().Logger()
	if len(opts) > 0 {
		s = s.WithOptions(opts...)
	}
	return s
}

// WithOptions clones the current Service, applies the supplied Options, and
// returns the resulting Service. It's safe to use concurrently.
func (s *Service) WithOptions(opts ...Option) *Service {
	for _, opt := range opts {
		opt.apply(s)
	}
	return s
}

// Run starts service
func (s *Service) Run() error {
	s.logger = s.logger.With().Str("id", s.cfg.ID).Str("name", s.cfg.Name).Logger()
	// set health server
	s.setHealthServer()
	// start service
	if err := s.start(); err != nil {
		return err
	}
	// hook run after start
	if err := s.hookAfterStart(); err != nil {
		return err
	}
	// wait for all go routines to finish
	s.wg.Wait()
	s.logger.Warn().Msg(fmt.Sprintf("%s has just exited", s.Name()))
	return nil
}

// Stop stops service
func (s *Service) Stop() {
	s.logger.Info().Msg(fmt.Sprintf("shutting down %s", s.Name()))
	s.close <- true
}

// Push allows to start additional options while service is running
func (s *Service) Push(opts ...Option) {
	for _, opt := range opts {
		opt.apply(s)
	}
}

// Server returns a server instance by name if initialized in service
func (s *Service) Server(name string) (srv *server.Server, ok bool) {
	name = common.SafeName(name, server.DefaultName)
	if srv, ok = s.cfg.Servers[name]; !ok {
		return
	}
	return srv, true
}

// Client returns a client instance by name if initialized in service
func (s *Service) Client(name string) (clt *client.Client, ok bool) {
	name = common.SafeName(name, client.DefaultName)
	if clt, ok = s.cfg.Clients[name]; !ok {
		return
	}
	return clt, true
}

// Health ...
func (s *Service) Health() *healthpb.HealthCheckResponse {
	hcr, err := s.health.Check(
		context.Background(), &healthpb.HealthCheckRequest{Service: s.cfg.ID})
	if err != nil {
		s.logger.Error().Msg(err.Error())
	}
	return hcr
}

// ID returns service id
func (s *Service) ID() string {
	return s.cfg.ID
}

// Name returns service name
func (s *Service) Name() string {
	return s.cfg.Name
}

// Logger returns service logger
func (s *Service) Logger() zerolog.Logger {
	return s.logger
}

func (s *Service) setHealthServer() {
	s.health.SetServingStatus(s.cfg.ID, 1)
	// Define Router
	mux := router.New()
	mux.GET("/_health/:module/:name", healthHandler)
	// Define server
	srv := server.New(
		server.Name(s.cfg.Name+"_health"),
		server.Addr(s.cfg.HealthAddr),
		server.Mux(mux),
		// server.Logger(s.logger),
	)
	s.cfg.Servers[srv.Name()] = srv
}

func (s *Service) start() error {
	s.logger.Info().Str("ip4", common.IPaddress()).
		Int("servers", len(s.cfg.Servers)).
		Int("clients", len(s.cfg.Clients)).
		Msg(fmt.Sprintf("starting %s, servers: %d clients: %d", s.Name(), len(s.cfg.Servers), len(s.cfg.Clients)))
	// run servers
	s.startServers()
	// dial clients
	s.startClients()
	// add go routine to WaitGroup
	s.wg.Add(1)
	go s.waitUntilStopOrSig()
	return nil
}

func (s *Service) hookAfterStart() error {
	hooks, ok := s.cfg.Hooks["after_start"]
	if !ok {
		return nil
	}
	ctx := context.Background()
	// make service available in hooks context
	ctx = s.WithContext(ctx)
	// make logger available in hooks context
	sublogger := s.logger.With().Str("hook", "after_start").Logger()
	ctx = sublogger.WithContext(ctx)
	for _, h := range hooks {
		if err := h(ctx); err != nil {
			return err
		}
	}
	return nil
}

func (s *Service) startServers() {
	for _, srv := range s.cfg.Servers {
		// add go routine to WaitGroup
		s.wg.Add(1)
		go func(s *Service, srv *server.Server) {
			defer s.wg.Done()
			f := fibonacci.F()
			for {
				err := srv.Run(
					server.Middlewares(
						serviceContextMiddleware(s),
					),
					server.UnaryServerInterceptors(
						serviceContextUnaryServerInterceptor(s),
					),
					server.StreamServerInterceptors(
						serviceContextStreamServerInterceptore(s),
					),
					server.Logger(s.logger),
				)
				if err == nil {
					return
				}
				l := srv.Logger()
				l.Error().Msg(fmt.Sprintf("%v failed to start, error: %v", srv.Name(), err.Error()))
				time.Sleep(time.Duration(f()) * time.Second)
			}
		}(s, srv)
	}
}

// startClients listen to the clientsCh
func (s *Service) startClients() {
	go func() {
		for {
			select {
			case clt, ok := <-s.cfg.clientsCh:
				if !ok {
					break
				}
				s.startClient(clt)
			default:
				time.Sleep(500 * time.Millisecond)
				continue
			}
		}
	}()
}

func (s *Service) startClient(clt *client.Client) {
	go func(clt *client.Client) {
		clt.Init(client.Logger(s.logger))
	}(clt)
}

// waitUntilStopOrSig waits for close channel or syscall Signal
func (s *Service) waitUntilStopOrSig() {
	defer s.wg.Done()
	//  Stop also in case of any host signal
	sigch := make(chan os.Signal, 1)
	signal.Notify(sigch, syscall.SIGTERM, syscall.SIGINT)

outer:
	for {
		select {
		case <-s.close:
			// Waits for call to stop
			s.health.SetServingStatus(s.cfg.ID, 2)
			s.unregister()
			s.closeClients()
			s.stopServers()
			break outer
		case sig := <-sigch:
			// Waits for signal to stop
			s.logger.Info().Msg(fmt.Sprintf("shutting down, got signal: %v", sig))
			s.health.SetServingStatus(s.cfg.ID, 2)
			s.unregister()
			s.closeClients()
			s.stopServers()
			break outer
		default:
			// keep on looping, non-blocking channel operations
			time.Sleep(50 * time.Millisecond)
			continue
		}
	}
}

func (s *Service) closeClients() {
	close(s.cfg.clientsCh)
	// for _, clt := range s.cfg.Clients {
	// 	// add go routine to WaitGroup
	// 	s.wg.Add(1)
	// 	go func(clt *client.Client) {
	// 		defer s.wg.Done()
	// 		clt.Close()
	// 	}(clt)
	// }
}

func (s *Service) stopServers() {
	for _, srv := range s.cfg.Servers {
		// add go routine to WaitGroup
		s.wg.Add(1)
		go func(s *Service, srv *server.Server) {
			defer s.wg.Done()
			srv.Stop()
		}(s, srv)
	}
}
<|endoftext|>