<commit_before>package scanner // import "github.com/amalog/go/scanner"

import (
	"fmt"
	"io"
)

const eof rune = -1

type Scanner struct {
	r io.RuneScanner

	err *SyntaxError

	line       int
	column     int
	prevLine   int // before calling next()
	prevColumn int // before calling next()

	prevToken *Token
}

func New(r io.RuneScanner) *Scanner {
	s := &Scanner{
		r:      r,
		line:   1,
		column: 0,
	}
	return s
}

// Scan returns the next token in the input.  Returns an infinite stream of Eof
// tokens once reaching the end of input.
func (s *Scanner) Scan() (*Token, error) {
	t, err := s.scan()
	if err != nil {
		s.prevToken = nil
		return nil, err
	}

	// handle comma insertion
	s.insertComma(t)
	if t.Class == nl { // newlines are for internal use only. skip them
		return s.Scan()
	}

	s.prevToken = t
	return t, err
}

func (s *Scanner) scan() (*Token, error) {
	var ch rune
	for {
		ch = s.next()
		if ch == eof {
			if s.err == nil {
				return &Token{Class: Eof, Position: s.Pos()}, nil
			}
			return nil, s.err
		}

		if ch >= 'a' && ch <= 'z' { // atom
			return s.scanAtom(ch)
		} else if ch >= 'A' && ch <= 'Z' { // variable
			return s.scanVariable(ch)
		} else if ch >= '0' && ch <= '9' { // number
			return s.scanNumber(ch)
		} else if ch == '"' { // string
			return s.scanString(ch)
		} else if ch == ' ' { // space
			s.skipSpace()
			continue
		} else if ch == '#' { // end of line comment
			return s.scanEolComment(ch)
		} else if ch == '\n' { // newline
			return &Token{Class: nl, Position: s.Pos()}, nil
		} else {
			break
		}
	}

	t := &Token{
		Class:    Punct,
		Position: s.Pos(),
		Text:     string([]rune{ch}),
	}
	return t, nil
}

func (s *Scanner) insertComma(t *Token) {
	// don't insert comma as first token
	if s.prevToken == nil {
		return
	}

	// certain tokens prohibit comma insertion
	switch s.prevToken.Class {
	case Punct:
		switch s.prevToken.Text {
		case ",", "(", "{":
			return
		}
	case Comment:
		return
	}

	switch t.Class {
	case Eof:
		if s.prevToken.Class != Eof { // before first EOF token
			t.Class = Punct
			t.Text = ","
		}
	case nl:
		t.Text = ","
		t.Class = Punct
	case Punct:
		if t.Text == ")" || t.Text == "}" { // before closing a seq or db
			t.Text = ","
			s.back()
		}
	}
}

func (s *Scanner) back() {
	err := s.r.UnreadRune()
	if err != nil {
		panic(err)
	}

	s.line = s.prevLine
	s.column = s.prevColumn
}

// consumes the next character in the stream. stores errors in s.err
func (s *Scanner) next() rune {
	s.prevLine = s.line
	s.prevColumn = s.column

	ch, _, err := s.r.ReadRune()
	if err == io.EOF {
		return eof
	}
	if err != nil {
		panic(err)
	}

	// handle prohibited characters
	if ch == '\t' || ch == '\r' {
		s.err = s.prohibitedCharacter(ch)
		return eof
	}

	// update position information
	if ch == '\n' {
		s.line++
		s.column = 0
	} else {
		s.column++
	}

	return ch
}

func (s *Scanner) prohibitedCharacter(ch rune) *SyntaxError {
	var name string
	switch ch {
	case '\t':
		name = "tab"
	case '\r':
		name = "carriage return"
	}
	return &SyntaxError{
		Position: s.Pos(),
		Message:  fmt.Sprintf("The %s character is prohibited", name),
	}
}

func (s *Scanner) Pos() Position {
	return Position{
		Line:   s.line,
		Column: s.column,
	}
}

func (s *Scanner) skipSpace() {
	for {
		ch := s.next()
		if ch == ' ' {
			continue
		}
		if ch != eof {
			s.back()
		}
		break
	}
}

func (s *Scanner) scanAtom(ch rune) (*Token, error) {
	chars := make([]rune, 0)

	pos := s.Pos()
CH:
	for {
		switch {
		case ch >= 'a' && ch <= 'z', ch == '_':
			chars = append(chars, ch)
		case ch == eof:
			break CH
		default:
			s.back()
			break CH
		}

		ch = s.next()
	}

	t := &Token{
		Class:    Atom,
		Position: pos,
		Text:     string(chars),
	}
	return t, nil
}

func (s *Scanner) scanVariable(ch rune) (*Token, error) {
	chars := make([]rune, 0)

	pos := s.Pos()
CH:
	for {
		switch {
		case ch >= 'A' && ch <= 'Z', ch >= 'a' && ch <= 'z':
			chars = append(chars, ch)
		case ch == eof:
			break CH
		default:
			s.back()
			break CH
		}

		ch = s.next()
	}

	t := &Token{
		Class:    Var,
		Position: pos,
		Text:     string(chars),
	}
	return t, nil
}

func (s *Scanner) scanNumber(ch rune) (*Token, error) {
	chars := make([]rune, 0)

	pos := s.Pos()
CH:
	for {
		switch {
		case ch >= '0' && ch <= '9', ch == '_', ch == '.':
			chars = append(chars, ch)
		case ch == eof:
			break CH
		default:
			s.back()
			break CH
		}

		ch = s.next()
	}

	t := &Token{
		Class:    Num,
		Position: pos,
		Text:     string(chars),
	}
	return t, nil
}

func (s *Scanner) scanString(ch rune) (*Token, error) {
	chars := []rune{ch}

	pos := s.Pos()
	for {
		ch = s.next()
		chars = append(chars, ch)
		if ch == '"' {
			break
		}
		if ch == eof {
			return nil, &SyntaxError{
				Position: pos,
				Message:  "Runaway string",
			}
		}
	}

	t := &Token{
		Class:    String,
		Position: pos,
		Text:     string(chars),
	}
	return t, nil
}

func (s *Scanner) scanEolComment(ch rune) (*Token, error) {
	chars := []rune{ch}

	pos := s.Pos()
	for {
		ch = s.next()
		if ch == '\n' || ch == eof {
			break
		}
		chars = append(chars, ch)
	}

	t := &Token{
		Class:    Comment,
		Position: pos,
		Text:     string(chars),
	}
	return t, nil
}
<commit_msg>Factor out scanToken()<commit_after>package scanner // import "github.com/amalog/go/scanner"

import (
	"fmt"
	"io"
)

const eof rune = -1

type Scanner struct {
	r io.RuneScanner

	err *SyntaxError

	line       int
	column     int
	prevLine   int // before calling next()
	prevColumn int // before calling next()

	prevToken *Token
}

func New(r io.RuneScanner) *Scanner {
	s := &Scanner{
		r:      r,
		line:   1,
		column: 0,
	}
	return s
}

// Scan returns the next token in the input.  Returns an infinite stream of Eof
// tokens once reaching the end of input.
func (s *Scanner) Scan() (*Token, error) {
	t, err := s.scan()
	if err != nil {
		s.prevToken = nil
		return nil, err
	}

	// handle comma insertion
	s.insertComma(t)
	if t.Class == nl { // newlines are for internal use only. skip them
		return s.Scan()
	}

	s.prevToken = t
	return t, err
}

func (s *Scanner) scan() (*Token, error) {
	var ch rune
	for {
		ch = s.next()
		if ch == eof {
			if s.err == nil {
				return &Token{Class: Eof, Position: s.Pos()}, nil
			}
			return nil, s.err
		}

		if ch >= 'a' && ch <= 'z' { // atom
			return s.scanToken(Atom, isAtomRune, ch)
		} else if ch >= 'A' && ch <= 'Z' { // variable
			return s.scanToken(Var, isVarRune, ch)
		} else if ch >= '0' && ch <= '9' { // number
			return s.scanToken(Num, isNumRune, ch)
		} else if ch == '"' { // string
			return s.scanString(ch)
		} else if ch == ' ' { // space
			s.skipSpace()
			continue
		} else if ch == '#' { // end of line comment
			return s.scanEolComment(ch)
		} else if ch == '\n' { // newline
			return &Token{Class: nl, Position: s.Pos()}, nil
		} else {
			break
		}
	}

	t := &Token{
		Class:    Punct,
		Position: s.Pos(),
		Text:     string([]rune{ch}),
	}
	return t, nil
}

func (s *Scanner) insertComma(t *Token) {
	// don't insert comma as first token
	if s.prevToken == nil {
		return
	}

	// certain tokens prohibit comma insertion
	switch s.prevToken.Class {
	case Punct:
		switch s.prevToken.Text {
		case ",", "(", "{":
			return
		}
	case Comment:
		return
	}

	switch t.Class {
	case Eof:
		if s.prevToken.Class != Eof { // before first EOF token
			t.Class = Punct
			t.Text = ","
		}
	case nl:
		t.Text = ","
		t.Class = Punct
	case Punct:
		if t.Text == ")" || t.Text == "}" { // before closing a seq or db
			t.Text = ","
			s.back()
		}
	}
}

func (s *Scanner) back() {
	err := s.r.UnreadRune()
	if err != nil {
		panic(err)
	}

	s.line = s.prevLine
	s.column = s.prevColumn
}

// consumes the next character in the stream. stores errors in s.err
func (s *Scanner) next() rune {
	s.prevLine = s.line
	s.prevColumn = s.column

	ch, _, err := s.r.ReadRune()
	if err == io.EOF {
		return eof
	}
	if err != nil {
		panic(err)
	}

	// handle prohibited characters
	if ch == '\t' || ch == '\r' {
		s.err = s.prohibitedCharacter(ch)
		return eof
	}

	// update position information
	if ch == '\n' {
		s.line++
		s.column = 0
	} else {
		s.column++
	}

	return ch
}

func (s *Scanner) prohibitedCharacter(ch rune) *SyntaxError {
	var name string
	switch ch {
	case '\t':
		name = "tab"
	case '\r':
		name = "carriage return"
	}
	return &SyntaxError{
		Position: s.Pos(),
		Message:  fmt.Sprintf("The %s character is prohibited", name),
	}
}

func (s *Scanner) Pos() Position {
	return Position{
		Line:   s.line,
		Column: s.column,
	}
}

func (s *Scanner) skipSpace() {
	for {
		ch := s.next()
		if ch == ' ' {
			continue
		}
		if ch != eof {
			s.back()
		}
		break
	}
}

func isAtomRune(ch rune) bool {
	return (ch >= 'a' && ch <= 'z') || ch == '_'
}

func isVarRune(ch rune) bool {
	return (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z')
}

func isNumRune(ch rune) bool {
	return (ch >= '0' && ch <= '9') || ch == '_' || ch == '.'
}

func (s *Scanner) scanToken(class Class, isAllowed func(rune) bool, ch rune) (*Token, error) {
	chars := make([]rune, 0)
	pos := s.Pos()
CH:
	for {
		switch {
		case isAllowed(ch):
			chars = append(chars, ch)
		case ch == eof:
			break CH
		default:
			s.back()
			break CH
		}

		ch = s.next()
	}

	t := &Token{
		Class:    class,
		Position: pos,
		Text:     string(chars),
	}
	return t, nil
}

func (s *Scanner) scanString(ch rune) (*Token, error) {
	chars := []rune{ch}

	pos := s.Pos()
	for {
		ch = s.next()
		chars = append(chars, ch)
		if ch == '"' {
			break
		}
		if ch == eof {
			return nil, &SyntaxError{
				Position: pos,
				Message:  "Runaway string",
			}
		}
	}

	t := &Token{
		Class:    String,
		Position: pos,
		Text:     string(chars),
	}
	return t, nil
}

func (s *Scanner) scanEolComment(ch rune) (*Token, error) {
	chars := []rune{ch}

	pos := s.Pos()
	for {
		ch = s.next()
		if ch == '\n' || ch == eof {
			break
		}
		chars = append(chars, ch)
	}

	t := &Token{
		Class:    Comment,
		Position: pos,
		Text:     string(chars),
	}
	return t, nil
}
<|endoftext|>


<commit_before>// Copyright 2018 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package v1alpha3

import (
	"fmt"
	"time"

	"github.com/envoyproxy/go-control-plane/envoy/api/v2/listener"
	fileaccesslog "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v2"
	accesslog "github.com/envoyproxy/go-control-plane/envoy/config/filter/accesslog/v2"
	mongo_proxy "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/mongo_proxy/v2"
	mysql_proxy "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/mysql_proxy/v1alpha1"
	redis_proxy "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/redis_proxy/v2"
	tcp_proxy "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/tcp_proxy/v2"
	xdsutil "github.com/envoyproxy/go-control-plane/pkg/util"

	networking "istio.io/api/networking/v1alpha3"
	"istio.io/istio/pilot/pkg/model"
	istio_route "istio.io/istio/pilot/pkg/networking/core/v1alpha3/route"
	"istio.io/istio/pilot/pkg/networking/util"
)

// redisOpTimeout is the default operation timeout for the Redis proxy filter.
var redisOpTimeout = 5 * time.Second

// buildInboundNetworkFilters generates a TCP proxy network filter on the inbound path
func buildInboundNetworkFilters(env *model.Environment, node *model.Proxy, instance *model.ServiceInstance) []listener.Filter {
	clusterName := model.BuildSubsetKey(model.TrafficDirectionInbound, instance.Endpoint.ServicePort.Name,
		instance.Service.Hostname, instance.Endpoint.ServicePort.Port)
	config := &tcp_proxy.TcpProxy{
		StatPrefix:       clusterName,
		ClusterSpecifier: &tcp_proxy.TcpProxy_Cluster{Cluster: clusterName},
	}
	tcpFilter := setAccessLogAndBuildTCPFilter(env, node, config)
	return buildNetworkFiltersStack(node, instance.Endpoint.ServicePort, tcpFilter, clusterName, clusterName)
}

// setAccessLogAndBuildTCPFilter sets the AccessLog configuration in the given
// TcpProxy instance and builds a TCP filter out of it.
func setAccessLogAndBuildTCPFilter(env *model.Environment, node *model.Proxy, config *tcp_proxy.TcpProxy) *listener.Filter {
	if env.Mesh.AccessLogFile != "" {
		fl := &fileaccesslog.FileAccessLog{
			Path: env.Mesh.AccessLogFile,
		}

		acc := &accesslog.AccessLog{
			Name: xdsutil.FileAccessLog,
		}
		if util.IsProxyVersionGE11(node) {
			buildAccessLog(fl, env)
			acc.ConfigType = &accesslog.AccessLog_TypedConfig{TypedConfig: util.MessageToAny(fl)}
		} else {
			acc.ConfigType = &accesslog.AccessLog_Config{Config: util.MessageToStruct(fl)}
		}

		config.AccessLog = []*accesslog.AccessLog{acc}

	}

	tcpFilter := &listener.Filter{
		Name: xdsutil.TCPProxy,
	}
	if util.IsProxyVersionGE11(node) {
		tcpFilter.ConfigType = &listener.Filter_TypedConfig{TypedConfig: util.MessageToAny(config)}
	} else {
		tcpFilter.ConfigType = &listener.Filter_Config{Config: util.MessageToStruct(config)}
	}
	return tcpFilter
}

// buildOutboundNetworkFiltersWithSingleDestination takes a single cluster name
// and builds a stack of network filters.
func buildOutboundNetworkFiltersWithSingleDestination(env *model.Environment, node *model.Proxy,
	clusterName string, port *model.Port) []listener.Filter {

	config := &tcp_proxy.TcpProxy{
		StatPrefix:       clusterName,
		ClusterSpecifier: &tcp_proxy.TcpProxy_Cluster{Cluster: clusterName},
		// TODO: Need to set other fields such as Idle timeouts
	}
	tcpFilter := setAccessLogAndBuildTCPFilter(env, node, config)
	return buildNetworkFiltersStack(node, port, tcpFilter, clusterName, clusterName)
}

// buildOutboundNetworkFiltersWithWeightedClusters takes a set of weighted
// destination routes and builds a stack of network filters.
func buildOutboundNetworkFiltersWithWeightedClusters(env *model.Environment, node *model.Proxy, routes []*networking.RouteDestination,
	push *model.PushContext, port *model.Port, config model.ConfigMeta) []listener.Filter {

	statPrefix := fmt.Sprintf("%s.%s", config.Name, config.Namespace)
	clusterSpecifier := &tcp_proxy.TcpProxy_WeightedClusters{
		WeightedClusters: &tcp_proxy.TcpProxy_WeightedCluster{},
	}
	proxyConfig := &tcp_proxy.TcpProxy{
		StatPrefix:       statPrefix,
		ClusterSpecifier: clusterSpecifier,
		// TODO: Need to set other fields such as Idle timeouts
	}

	for _, route := range routes {
		if route.Weight > 0 {
			clusterName := istio_route.GetDestinationCluster(route.Destination, push.ServiceByHostname[model.Hostname(route.Destination.Host)], port.Port)
			clusterSpecifier.WeightedClusters.Clusters = append(clusterSpecifier.WeightedClusters.Clusters, &tcp_proxy.TcpProxy_WeightedCluster_ClusterWeight{
				Name:   clusterName,
				Weight: uint32(route.Weight),
			})
		}
	}

	// TODO: Need to handle multiple cluster names for Redis
	clusterName := clusterSpecifier.WeightedClusters.Clusters[0].Name
	tcpFilter := setAccessLogAndBuildTCPFilter(env, node, proxyConfig)
	return buildNetworkFiltersStack(node, port, tcpFilter, statPrefix, clusterName)
}

// buildNetworkFiltersStack builds a slice of network filters based on
// the protocol in use and the given TCP filter instance.
func buildNetworkFiltersStack(node *model.Proxy, port *model.Port, tcpFilter *listener.Filter, statPrefix,
	clusterName string) []listener.Filter {

	filterstack := make([]listener.Filter, 0)
	switch port.Protocol {
	case model.ProtocolMongo:
		filterstack = append(filterstack, buildMongoFilter(statPrefix, util.IsProxyVersionGE11(node)))
		filterstack = append(filterstack, *tcpFilter)
	case model.ProtocolRedis:
		if util.IsProxyVersionGE11(node) {
			filterstack = append(filterstack, buildRedisFilter(statPrefix, clusterName, util.IsProxyVersionGE11(node)))
		} else {
			filterstack = append(filterstack, *tcpFilter)
		}
	case model.ProtocolMySQL:
		filterstack = append(filterstack, buildMySQLFilter(statPrefix, util.IsProxyVersionGE11(node)))
		filterstack = append(filterstack, *tcpFilter)
	default:
		filterstack = append(filterstack, *tcpFilter)
	}

	return filterstack
}

// buildOutboundNetworkFilters generates a TCP proxy network filter for outbound
// connections. In addition, it generates protocol specific filters (e.g., Mongo
// filter).
func buildOutboundNetworkFilters(env *model.Environment, node *model.Proxy,
	routes []*networking.RouteDestination, push *model.PushContext,
	port *model.Port, config model.ConfigMeta) []listener.Filter {

	if !util.IsProxyVersionGE11(node) || len(routes) == 1 {
		service := push.ServiceByHostname[model.Hostname(routes[0].Destination.Host)]
		clusterName := istio_route.GetDestinationCluster(routes[0].Destination, service, port.Port)
		return buildOutboundNetworkFiltersWithSingleDestination(env, node, clusterName, port)
	}
	return buildOutboundNetworkFiltersWithWeightedClusters(env, node, routes, push, port, config)
}

// buildMongoFilter builds an outbound Envoy MongoProxy filter.
func buildMongoFilter(statPrefix string, is11 bool) listener.Filter {
	// TODO: add a watcher for /var/lib/istio/mongo/certs
	// if certs are found use, TLS or mTLS clusters for talking to MongoDB.
	// User is responsible for mounting those certs in the pod.
	config := &mongo_proxy.MongoProxy{
		StatPrefix: statPrefix, // mongo stats are prefixed with mongo.<statPrefix> by Envoy
		// TODO enable faults in mongo
	}

	out := listener.Filter{
		Name: xdsutil.MongoProxy,
	}
	if is11 {
		out.ConfigType = &listener.Filter_TypedConfig{TypedConfig: util.MessageToAny(config)}
	} else {
		out.ConfigType = &listener.Filter_Config{Config: util.MessageToStruct(config)}
	}

	return out
}

// buildOutboundAutoPassthroughFilterStack builds a filter stack with sni_cluster and tcp_proxy
// used by auto_passthrough gateway servers
func buildOutboundAutoPassthroughFilterStack(env *model.Environment, node *model.Proxy, port *model.Port) []listener.Filter {
	// First build tcp_proxy with access logs
	// then add sni_cluster to the front
	tcpProxy := buildOutboundNetworkFiltersWithSingleDestination(env, node, util.BlackHoleCluster, port)
	filterstack := make([]listener.Filter, 0)
	filterstack = append(filterstack, listener.Filter{
		Name: util.SniClusterFilter,
	})
	filterstack = append(filterstack, tcpProxy...)

	return filterstack
}

// buildRedisFilter builds an outbound Envoy RedisProxy filter.
// Currently, if multiple clusters are defined, one of them will be picked for
// configuring the Redis proxy.
func buildRedisFilter(statPrefix, clusterName string, is11 bool) listener.Filter {
	config := &redis_proxy.RedisProxy{
		StatPrefix: statPrefix, // redis stats are prefixed with redis.<statPrefix> by Envoy
		Cluster:    clusterName,
		Settings: &redis_proxy.RedisProxy_ConnPoolSettings{
			OpTimeout: &redisOpTimeout, // TODO: Make this user configurable
		},
	}

	out := listener.Filter{
		Name: xdsutil.RedisProxy,
	}
	if is11 {
		out.ConfigType = &listener.Filter_TypedConfig{TypedConfig: util.MessageToAny(config)}
	} else {
		out.ConfigType = &listener.Filter_Config{Config: util.MessageToStruct(config)}
	}

	return out
}

// buildMySQLFilter builds an outbound Envoy MySQLProxy filter.
func buildMySQLFilter(statPrefix string, is11 bool) listener.Filter {
	config := &mysql_proxy.MySQLProxy{
		StatPrefix: statPrefix, // MySQL stats are prefixed with mysql.<statPrefix> by Envoy.
	}

	out := listener.Filter{
		Name: xdsutil.MySQLProxy,
	}

	if is11 {
		out.ConfigType = &listener.Filter_TypedConfig{TypedConfig: util.MessageToAny(config)}
	} else {
		out.ConfigType = &listener.Filter_Config{Config: util.MessageToStruct(config)}
	}

	return out
}
<commit_msg>guard mysql proxy with version check (#12225)<commit_after>// Copyright 2018 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package v1alpha3

import (
	"fmt"
	"time"

	"github.com/envoyproxy/go-control-plane/envoy/api/v2/listener"
	fileaccesslog "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v2"
	accesslog "github.com/envoyproxy/go-control-plane/envoy/config/filter/accesslog/v2"
	mongo_proxy "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/mongo_proxy/v2"
	mysql_proxy "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/mysql_proxy/v1alpha1"
	redis_proxy "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/redis_proxy/v2"
	tcp_proxy "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/tcp_proxy/v2"
	xdsutil "github.com/envoyproxy/go-control-plane/pkg/util"

	networking "istio.io/api/networking/v1alpha3"
	"istio.io/istio/pilot/pkg/model"
	istio_route "istio.io/istio/pilot/pkg/networking/core/v1alpha3/route"
	"istio.io/istio/pilot/pkg/networking/util"
)

// redisOpTimeout is the default operation timeout for the Redis proxy filter.
var redisOpTimeout = 5 * time.Second

// buildInboundNetworkFilters generates a TCP proxy network filter on the inbound path
func buildInboundNetworkFilters(env *model.Environment, node *model.Proxy, instance *model.ServiceInstance) []listener.Filter {
	clusterName := model.BuildSubsetKey(model.TrafficDirectionInbound, instance.Endpoint.ServicePort.Name,
		instance.Service.Hostname, instance.Endpoint.ServicePort.Port)
	config := &tcp_proxy.TcpProxy{
		StatPrefix:       clusterName,
		ClusterSpecifier: &tcp_proxy.TcpProxy_Cluster{Cluster: clusterName},
	}
	tcpFilter := setAccessLogAndBuildTCPFilter(env, node, config)
	return buildNetworkFiltersStack(node, instance.Endpoint.ServicePort, tcpFilter, clusterName, clusterName)
}

// setAccessLogAndBuildTCPFilter sets the AccessLog configuration in the given
// TcpProxy instance and builds a TCP filter out of it.
func setAccessLogAndBuildTCPFilter(env *model.Environment, node *model.Proxy, config *tcp_proxy.TcpProxy) *listener.Filter {
	if env.Mesh.AccessLogFile != "" {
		fl := &fileaccesslog.FileAccessLog{
			Path: env.Mesh.AccessLogFile,
		}

		acc := &accesslog.AccessLog{
			Name: xdsutil.FileAccessLog,
		}
		if util.IsProxyVersionGE11(node) {
			buildAccessLog(fl, env)
			acc.ConfigType = &accesslog.AccessLog_TypedConfig{TypedConfig: util.MessageToAny(fl)}
		} else {
			acc.ConfigType = &accesslog.AccessLog_Config{Config: util.MessageToStruct(fl)}
		}

		config.AccessLog = []*accesslog.AccessLog{acc}

	}

	tcpFilter := &listener.Filter{
		Name: xdsutil.TCPProxy,
	}
	if util.IsProxyVersionGE11(node) {
		tcpFilter.ConfigType = &listener.Filter_TypedConfig{TypedConfig: util.MessageToAny(config)}
	} else {
		tcpFilter.ConfigType = &listener.Filter_Config{Config: util.MessageToStruct(config)}
	}
	return tcpFilter
}

// buildOutboundNetworkFiltersWithSingleDestination takes a single cluster name
// and builds a stack of network filters.
func buildOutboundNetworkFiltersWithSingleDestination(env *model.Environment, node *model.Proxy,
	clusterName string, port *model.Port) []listener.Filter {

	config := &tcp_proxy.TcpProxy{
		StatPrefix:       clusterName,
		ClusterSpecifier: &tcp_proxy.TcpProxy_Cluster{Cluster: clusterName},
		// TODO: Need to set other fields such as Idle timeouts
	}
	tcpFilter := setAccessLogAndBuildTCPFilter(env, node, config)
	return buildNetworkFiltersStack(node, port, tcpFilter, clusterName, clusterName)
}

// buildOutboundNetworkFiltersWithWeightedClusters takes a set of weighted
// destination routes and builds a stack of network filters.
func buildOutboundNetworkFiltersWithWeightedClusters(env *model.Environment, node *model.Proxy, routes []*networking.RouteDestination,
	push *model.PushContext, port *model.Port, config model.ConfigMeta) []listener.Filter {

	statPrefix := fmt.Sprintf("%s.%s", config.Name, config.Namespace)
	clusterSpecifier := &tcp_proxy.TcpProxy_WeightedClusters{
		WeightedClusters: &tcp_proxy.TcpProxy_WeightedCluster{},
	}
	proxyConfig := &tcp_proxy.TcpProxy{
		StatPrefix:       statPrefix,
		ClusterSpecifier: clusterSpecifier,
		// TODO: Need to set other fields such as Idle timeouts
	}

	for _, route := range routes {
		if route.Weight > 0 {
			clusterName := istio_route.GetDestinationCluster(route.Destination, push.ServiceByHostname[model.Hostname(route.Destination.Host)], port.Port)
			clusterSpecifier.WeightedClusters.Clusters = append(clusterSpecifier.WeightedClusters.Clusters, &tcp_proxy.TcpProxy_WeightedCluster_ClusterWeight{
				Name:   clusterName,
				Weight: uint32(route.Weight),
			})
		}
	}

	// TODO: Need to handle multiple cluster names for Redis
	clusterName := clusterSpecifier.WeightedClusters.Clusters[0].Name
	tcpFilter := setAccessLogAndBuildTCPFilter(env, node, proxyConfig)
	return buildNetworkFiltersStack(node, port, tcpFilter, statPrefix, clusterName)
}

// buildNetworkFiltersStack builds a slice of network filters based on
// the protocol in use and the given TCP filter instance.
func buildNetworkFiltersStack(node *model.Proxy, port *model.Port, tcpFilter *listener.Filter, statPrefix,
	clusterName string) []listener.Filter {

	filterstack := make([]listener.Filter, 0)
	switch port.Protocol {
	case model.ProtocolMongo:
		filterstack = append(filterstack, buildMongoFilter(statPrefix, util.IsProxyVersionGE11(node)))
		filterstack = append(filterstack, *tcpFilter)
	case model.ProtocolRedis:
		if util.IsProxyVersionGE11(node) {
			filterstack = append(filterstack, buildRedisFilter(statPrefix, clusterName, util.IsProxyVersionGE11(node)))
		} else {
			filterstack = append(filterstack, *tcpFilter)
		}
	case model.ProtocolMySQL:
		if util.IsProxyVersionGE11(node) {
			filterstack = append(filterstack, buildMySQLFilter(statPrefix, util.IsProxyVersionGE11(node)))
		}
		filterstack = append(filterstack, *tcpFilter)
	default:
		filterstack = append(filterstack, *tcpFilter)
	}

	return filterstack
}

// buildOutboundNetworkFilters generates a TCP proxy network filter for outbound
// connections. In addition, it generates protocol specific filters (e.g., Mongo
// filter).
func buildOutboundNetworkFilters(env *model.Environment, node *model.Proxy,
	routes []*networking.RouteDestination, push *model.PushContext,
	port *model.Port, config model.ConfigMeta) []listener.Filter {

	if !util.IsProxyVersionGE11(node) || len(routes) == 1 {
		service := push.ServiceByHostname[model.Hostname(routes[0].Destination.Host)]
		clusterName := istio_route.GetDestinationCluster(routes[0].Destination, service, port.Port)
		return buildOutboundNetworkFiltersWithSingleDestination(env, node, clusterName, port)
	}
	return buildOutboundNetworkFiltersWithWeightedClusters(env, node, routes, push, port, config)
}

// buildMongoFilter builds an outbound Envoy MongoProxy filter.
func buildMongoFilter(statPrefix string, is11 bool) listener.Filter {
	// TODO: add a watcher for /var/lib/istio/mongo/certs
	// if certs are found use, TLS or mTLS clusters for talking to MongoDB.
	// User is responsible for mounting those certs in the pod.
	config := &mongo_proxy.MongoProxy{
		StatPrefix: statPrefix, // mongo stats are prefixed with mongo.<statPrefix> by Envoy
		// TODO enable faults in mongo
	}

	out := listener.Filter{
		Name: xdsutil.MongoProxy,
	}
	if is11 {
		out.ConfigType = &listener.Filter_TypedConfig{TypedConfig: util.MessageToAny(config)}
	} else {
		out.ConfigType = &listener.Filter_Config{Config: util.MessageToStruct(config)}
	}

	return out
}

// buildOutboundAutoPassthroughFilterStack builds a filter stack with sni_cluster and tcp_proxy
// used by auto_passthrough gateway servers
func buildOutboundAutoPassthroughFilterStack(env *model.Environment, node *model.Proxy, port *model.Port) []listener.Filter {
	// First build tcp_proxy with access logs
	// then add sni_cluster to the front
	tcpProxy := buildOutboundNetworkFiltersWithSingleDestination(env, node, util.BlackHoleCluster, port)
	filterstack := make([]listener.Filter, 0)
	filterstack = append(filterstack, listener.Filter{
		Name: util.SniClusterFilter,
	})
	filterstack = append(filterstack, tcpProxy...)

	return filterstack
}

// buildRedisFilter builds an outbound Envoy RedisProxy filter.
// Currently, if multiple clusters are defined, one of them will be picked for
// configuring the Redis proxy.
func buildRedisFilter(statPrefix, clusterName string, is11 bool) listener.Filter {
	config := &redis_proxy.RedisProxy{
		StatPrefix: statPrefix, // redis stats are prefixed with redis.<statPrefix> by Envoy
		Cluster:    clusterName,
		Settings: &redis_proxy.RedisProxy_ConnPoolSettings{
			OpTimeout: &redisOpTimeout, // TODO: Make this user configurable
		},
	}

	out := listener.Filter{
		Name: xdsutil.RedisProxy,
	}
	if is11 {
		out.ConfigType = &listener.Filter_TypedConfig{TypedConfig: util.MessageToAny(config)}
	} else {
		out.ConfigType = &listener.Filter_Config{Config: util.MessageToStruct(config)}
	}

	return out
}

// buildMySQLFilter builds an outbound Envoy MySQLProxy filter.
func buildMySQLFilter(statPrefix string, is11 bool) listener.Filter {
	config := &mysql_proxy.MySQLProxy{
		StatPrefix: statPrefix, // MySQL stats are prefixed with mysql.<statPrefix> by Envoy.
	}

	out := listener.Filter{
		Name: xdsutil.MySQLProxy,
	}

	if is11 {
		out.ConfigType = &listener.Filter_TypedConfig{TypedConfig: util.MessageToAny(config)}
	} else {
		out.ConfigType = &listener.Filter_Config{Config: util.MessageToStruct(config)}
	}

	return out
}
<|endoftext|>


<commit_before>// +build go1.13

// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

package azcore

import (
	"context"
	"errors"
	"io"
	"net/http"
	"reflect"
)

// Policy represents an extensibility point for the Pipeline that can mutate the specified
// Request and react to the received Response.
type Policy interface {
	// Do applies the policy to the specified Request.  When implementing a Policy, mutate the
	// request before calling req.Next() to move on to the next policy, and respond to the result
	// before returning to the caller.
	Do(req *Request) (*Response, error)
}

// PolicyFunc is a type that implements the Policy interface.
// Use this type when implementing a stateless policy as a first-class function.
type PolicyFunc func(*Request) (*Response, error)

// Do implements the Policy interface on PolicyFunc.
func (pf PolicyFunc) Do(req *Request) (*Response, error) {
	return pf(req)
}

// Transport represents an HTTP pipeline transport used to send HTTP requests and receive responses.
type Transport interface {
	// Do sends the HTTP request and returns the HTTP response or error.
	Do(req *http.Request) (*http.Response, error)
}

// TransportFunc is a type that implements the Transport interface.
// Use this type when implementing a stateless transport as a first-class function.
type TransportFunc func(*http.Request) (*http.Response, error)

// Do implements the Transport interface on TransportFunc.
func (tf TransportFunc) Do(req *http.Request) (*http.Response, error) {
	return tf(req)
}

// used to adapt a TransportPolicy to a Policy
type transportPolicy struct {
	trans Transport
}

func (tp transportPolicy) Do(req *Request) (*Response, error) {
	resp, err := tp.trans.Do(req.Request)
	if err != nil {
		return nil, err
	} else if resp == nil {
		// there was no response and no error (rare but can happen)
		// this ensures the retry policy will retry the request
		return nil, errors.New("received nil response")
	}
	return &Response{Response: resp}, nil
}

// Pipeline represents a primitive for sending HTTP requests and receiving responses.
// Its behavior can be extended by specifying policies during construction.
type Pipeline struct {
	policies []Policy
}

// NewPipeline creates a new Pipeline object from the specified Transport and Policies.
// If no transport is provided then the default *http.Client transport will be used.
func NewPipeline(transport Transport, policies ...Policy) Pipeline {
	if transport == nil {
		transport = defaultHTTPClient
	}
	// transport policy must always be the last in the slice
	policies = append(policies, PolicyFunc(httpHeaderPolicy), PolicyFunc(bodyDownloadPolicy), transportPolicy{trans: transport})
	return Pipeline{
		policies: policies,
	}
}

// Do is called for each and every HTTP request. It passes the request through all
// the Policy objects (which can transform the Request's URL/query parameters/headers)
// and ultimately sends the transformed HTTP request over the network.
func (p Pipeline) Do(req *Request) (*Response, error) {
	if err := req.valid(); err != nil {
		return nil, err
	}
	req.policies = p.policies
	return req.Next()
}

// ReadSeekCloser is the interface that groups the io.ReadCloser and io.Seeker interfaces.
type ReadSeekCloser interface {
	io.ReadCloser
	io.Seeker
}

type nopCloser struct {
	io.ReadSeeker
}

func (n nopCloser) Close() error {
	return nil
}

// NopCloser returns a ReadSeekCloser with a no-op close method wrapping the provided io.ReadSeeker.
func NopCloser(rs io.ReadSeeker) ReadSeekCloser {
	return nopCloser{rs}
}

// Poller provides operations for checking the state of a long-running operation.
// An LRO can be in either a non-terminal or terminal state.  A non-terminal state
// indicates the LRO is still in progress.  A terminal state indicates the LRO has
// completed successfully, failed, or was cancelled.
type Poller interface {
	// Done returns true if the LRO has reached a terminal state.
	Done() bool

	// Poll fetches the latest state of the LRO.  It returns an HTTP response or error.
	// If the LRO has completed successfully, the poller's state is update and the HTTP
	// response is returned.
	// If the LRO has completed with failure or was cancelled, the poller's state is
	// updated and the error is returned.
	// If the LRO has not reached a terminal state, the poller's state is updated and
	// the latest HTTP response is returned.
	// If Poll fails, the poller's state is unmodified and the error is returned.
	// Calling Poll on an LRO that has reached a terminal state will return the final
	// HTTP response or error.
	Poll(context.Context) (*http.Response, error)

	// ResumeToken returns a value representing the poller that can be used to resume
	// the LRO at a later time. ResumeTokens are unique per service operation.
	ResumeToken() (string, error)
}

// Pager provides operations for iterating over paged responses.
type Pager interface {
	// NextPage returns true if the pager advanced to the next page.
	// Returns false if there are no more pages or an error occurred.
	NextPage(context.Context) bool

	// Err returns the last error encountered while paging.
	Err() error
}

// holds sentinel values used to send nulls
var nullables map[reflect.Type]interface{} = map[reflect.Type]interface{}{}

// NullValue is used to send an explicit 'null' within a request.
// This is typically used in JSON-MERGE-PATCH operations to delete a value.
func NullValue(v interface{}) interface{} {
	t := reflect.TypeOf(v)
	if k := t.Kind(); k != reflect.Ptr && k != reflect.Slice && k != reflect.Map {
		// t is not of pointer type, make it be of pointer type
		t = reflect.PtrTo(t)
	}
	v, found := nullables[t]
	if !found {
		var o reflect.Value
		if k := t.Kind(); k == reflect.Slice || k == reflect.Map {
			o = reflect.New(t) // *[]type
			o = o.Elem()       // []type
		} else {
			o = reflect.New(t.Elem())
		}
		v = o.Interface()
		nullables[t] = v
	}
	// return the sentinel object
	return v
}

// IsNullValue returns true if the field contains a null sentinel value.
// This is used by custom marshallers to properly encode a null value.
func IsNullValue(v interface{}) bool {
	// see if our map has a sentinel object for this *T
	t := reflect.TypeOf(v)
	if k := t.Kind(); k != reflect.Ptr && k != reflect.Slice && k != reflect.Map {
		// v isn't a pointer type so it can never be a null
		return false
	}
	if o, found := nullables[t]; found {
		o1 := reflect.ValueOf(o)
		v1 := reflect.ValueOf(v)
		// we found it; return true if v points to the sentinel object.
		// NOTE: maps and slices can only be compared to nil, else you get
		// a runtime panic.  so we compare addresses instead.
		return o1.Pointer() == v1.Pointer()
	}
	// no sentinel object for this *t
	return false
}
<commit_msg>adjust comment for completeness (#14681)<commit_after>// +build go1.13

// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

package azcore

import (
	"context"
	"errors"
	"io"
	"net/http"
	"reflect"
)

// Policy represents an extensibility point for the Pipeline that can mutate the specified
// Request and react to the received Response.
type Policy interface {
	// Do applies the policy to the specified Request.  When implementing a Policy, mutate the
	// request before calling req.Next() to move on to the next policy, and respond to the result
	// before returning to the caller.
	Do(req *Request) (*Response, error)
}

// PolicyFunc is a type that implements the Policy interface.
// Use this type when implementing a stateless policy as a first-class function.
type PolicyFunc func(*Request) (*Response, error)

// Do implements the Policy interface on PolicyFunc.
func (pf PolicyFunc) Do(req *Request) (*Response, error) {
	return pf(req)
}

// Transport represents an HTTP pipeline transport used to send HTTP requests and receive responses.
type Transport interface {
	// Do sends the HTTP request and returns the HTTP response or error.
	Do(req *http.Request) (*http.Response, error)
}

// TransportFunc is a type that implements the Transport interface.
// Use this type when implementing a stateless transport as a first-class function.
type TransportFunc func(*http.Request) (*http.Response, error)

// Do implements the Transport interface on TransportFunc.
func (tf TransportFunc) Do(req *http.Request) (*http.Response, error) {
	return tf(req)
}

// used to adapt a TransportPolicy to a Policy
type transportPolicy struct {
	trans Transport
}

func (tp transportPolicy) Do(req *Request) (*Response, error) {
	resp, err := tp.trans.Do(req.Request)
	if err != nil {
		return nil, err
	} else if resp == nil {
		// there was no response and no error (rare but can happen)
		// this ensures the retry policy will retry the request
		return nil, errors.New("received nil response")
	}
	return &Response{Response: resp}, nil
}

// Pipeline represents a primitive for sending HTTP requests and receiving responses.
// Its behavior can be extended by specifying policies during construction.
type Pipeline struct {
	policies []Policy
}

// NewPipeline creates a new Pipeline object from the specified Transport and Policies.
// If no transport is provided then the default *http.Client transport will be used.
func NewPipeline(transport Transport, policies ...Policy) Pipeline {
	if transport == nil {
		transport = defaultHTTPClient
	}
	// transport policy must always be the last in the slice
	policies = append(policies, PolicyFunc(httpHeaderPolicy), PolicyFunc(bodyDownloadPolicy), transportPolicy{trans: transport})
	return Pipeline{
		policies: policies,
	}
}

// Do is called for each and every HTTP request. It passes the request through all
// the Policy objects (which can transform the Request's URL/query parameters/headers)
// and ultimately sends the transformed HTTP request over the network.
func (p Pipeline) Do(req *Request) (*Response, error) {
	if err := req.valid(); err != nil {
		return nil, err
	}
	req.policies = p.policies
	return req.Next()
}

// ReadSeekCloser is the interface that groups the io.ReadCloser and io.Seeker interfaces.
type ReadSeekCloser interface {
	io.ReadCloser
	io.Seeker
}

type nopCloser struct {
	io.ReadSeeker
}

func (n nopCloser) Close() error {
	return nil
}

// NopCloser returns a ReadSeekCloser with a no-op close method wrapping the provided io.ReadSeeker.
func NopCloser(rs io.ReadSeeker) ReadSeekCloser {
	return nopCloser{rs}
}

// Poller provides operations for checking the state of a long-running operation.
// An LRO can be in either a non-terminal or terminal state.  A non-terminal state
// indicates the LRO is still in progress.  A terminal state indicates the LRO has
// completed successfully, failed, or was cancelled.
type Poller interface {
	// Done returns true if the LRO has reached a terminal state.
	Done() bool

	// Poll fetches the latest state of the LRO.  It returns an HTTP response or error.
	// If the LRO has completed successfully, the poller's state is update and the HTTP
	// response is returned.
	// If the LRO has completed with failure or was cancelled, the poller's state is
	// updated and the error is returned.
	// If the LRO has not reached a terminal state, the poller's state is updated and
	// the latest HTTP response is returned.
	// If Poll fails, the poller's state is unmodified and the error is returned.
	// Calling Poll on an LRO that has reached a terminal state will return the final
	// HTTP response or error.
	Poll(context.Context) (*http.Response, error)

	// ResumeToken returns a value representing the poller that can be used to resume
	// the LRO at a later time. ResumeTokens are unique per service operation.
	ResumeToken() (string, error)
}

// Pager provides operations for iterating over paged responses.
type Pager interface {
	// NextPage returns true if the pager advanced to the next page.
	// Returns false if there are no more pages or an error occurred.
	NextPage(context.Context) bool

	// Err returns the last error encountered while paging.
	Err() error
}

// holds sentinel values used to send nulls
var nullables map[reflect.Type]interface{} = map[reflect.Type]interface{}{}

// NullValue is used to send an explicit 'null' within a request.
// This is typically used in JSON-MERGE-PATCH operations to delete a value.
func NullValue(v interface{}) interface{} {
	t := reflect.TypeOf(v)
	if k := t.Kind(); k != reflect.Ptr && k != reflect.Slice && k != reflect.Map {
		// t is not of pointer type, make it be of pointer type
		t = reflect.PtrTo(t)
	}
	v, found := nullables[t]
	if !found {
		var o reflect.Value
		if k := t.Kind(); k == reflect.Slice || k == reflect.Map {
			o = reflect.New(t) // *[]type / *map[]
			o = o.Elem()       // []type / map[]
		} else {
			o = reflect.New(t.Elem())
		}
		v = o.Interface()
		nullables[t] = v
	}
	// return the sentinel object
	return v
}

// IsNullValue returns true if the field contains a null sentinel value.
// This is used by custom marshallers to properly encode a null value.
func IsNullValue(v interface{}) bool {
	// see if our map has a sentinel object for this *T
	t := reflect.TypeOf(v)
	if k := t.Kind(); k != reflect.Ptr && k != reflect.Slice && k != reflect.Map {
		// v isn't a pointer type so it can never be a null
		return false
	}
	if o, found := nullables[t]; found {
		o1 := reflect.ValueOf(o)
		v1 := reflect.ValueOf(v)
		// we found it; return true if v points to the sentinel object.
		// NOTE: maps and slices can only be compared to nil, else you get
		// a runtime panic.  so we compare addresses instead.
		return o1.Pointer() == v1.Pointer()
	}
	// no sentinel object for this *t
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"testing"
	"net/http"
	"net/http/httptest"
	"net/url"
	"github.com/stretchr/testify/assert"
	"encoding/json"
)

func TestRoot(t *testing.T) {
	record := httptest.NewRecorder()
	req := &http.Request{
		Method: "GET",
		URL: &url.URL{Path: "/"},
	}

	handleRoot(record, req)
	assert.Equal(t, record.Code, http.StatusOK)

	type Status struct {
		Status string
	}

	var status Status
	err := json.Unmarshal(record.Body.Bytes(), &status)
	assert.Nil(t, err)
	assert.Equal(t, status.Status, "ok")
}
<commit_msg>Refactor tests<commit_after>package main

import (
	"testing"
	"net/http"
	"net/http/httptest"
	"net/url"
	"github.com/stretchr/testify/assert"
	"encoding/json"
)

func request(handler func(http.ResponseWriter, *http.Request), method string, path string) (record *httptest.ResponseRecorder) {
	record = httptest.NewRecorder()
	req := &http.Request{
		Method: method,
		URL: &url.URL{Path: path},
	}
	handler(record, req)
	return
}

func assertOK(t *testing.T, record *httptest.ResponseRecorder) {
	assert.Equal(t, record.Code, http.StatusOK)

	type Status struct {
		Status string
	}

	var status Status
	err := json.Unmarshal(record.Body.Bytes(), &status)
	assert.Nil(t, err)
	assert.Equal(t, status.Status, "ok")
}

func TestRoot(t *testing.T) {
	record := request(handleRoot, "GET", "/")
	assertOK(t, record)
}
<|endoftext|>


<commit_before>package conf

import (
	"strings"
	"time"
)

const (
	AppName = "jiacrontab"
	Version = "v1.4.4"
)

var (
	AppService *App
	starTime   = time.Now()
)

type App struct {
	Debug           bool      `json:"debug"`
	HttpListenAddr  string    `json:"http_listen_addr"`
	StaticDir       string    `json:"static_dir"`
	DataFile        string    `json:"data_file"`
	RpcListenAddr   string    `json:"rpc_listen_addr"`
	ServerStartTime time.Time `json:"-"`
	TplDir          string    `json:"tpl_dir"`
	TplExt          string    `json:"tpl_ext"`

	User          string   `json:"user"`
	Passwd        string   `json:"-"`
	AllowCommands []string `json:"allow_commands"`
}

func LoadAppService() {
	app := cf.Section("app")
	AppService = &App{
		Debug:           app.Key("debug").MustBool(false),
		HttpListenAddr:  app.Key("http_listen_addr").MustString("0.0.0.0:20000"),
		StaticDir:       app.Key("static_dir").MustString("/static"),
		TplExt:          ".html",
		TplDir:          "template",
		DataFile:        app.Key("data_file").MustString("data.json"),
		RpcListenAddr:   app.Key("rpc_listen_addr").MustString(":20003"),
		User:            app.Key("app_user").MustString("admin"),
		Passwd:          app.Key("app_passwd").MustString("123456"),
		ServerStartTime: starTime,
		AllowCommands:   strings.Split(app.Key("allow_commands").MustString("php,python,node,curl,wget,lua"), ","),
	}
}
<commit_msg>v1.4.5<commit_after>package conf

import (
	"strings"
	"time"
)

const (
	AppName = "jiacrontab"
	Version = "v1.4.5"
)

var (
	AppService *App
	starTime   = time.Now()
)

type App struct {
	Debug           bool      `json:"debug"`
	HttpListenAddr  string    `json:"http_listen_addr"`
	StaticDir       string    `json:"static_dir"`
	DataFile        string    `json:"data_file"`
	RpcListenAddr   string    `json:"rpc_listen_addr"`
	ServerStartTime time.Time `json:"-"`
	TplDir          string    `json:"tpl_dir"`
	TplExt          string    `json:"tpl_ext"`

	User          string   `json:"user"`
	Passwd        string   `json:"-"`
	AllowCommands []string `json:"allow_commands"`
}

func LoadAppService() {
	app := cf.Section("app")
	AppService = &App{
		Debug:           app.Key("debug").MustBool(false),
		HttpListenAddr:  app.Key("http_listen_addr").MustString("0.0.0.0:20000"),
		StaticDir:       app.Key("static_dir").MustString("/static"),
		TplExt:          ".html",
		TplDir:          "template",
		DataFile:        app.Key("data_file").MustString("data.json"),
		RpcListenAddr:   app.Key("rpc_listen_addr").MustString(":20003"),
		User:            app.Key("app_user").MustString("admin"),
		Passwd:          app.Key("app_passwd").MustString("123456"),
		ServerStartTime: starTime,
		AllowCommands:   strings.Split(app.Key("allow_commands").MustString("php,python,node,curl,wget,lua"), ","),
	}
}
<|endoftext|>


<commit_before>package token

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/docker/swarm/discovery"
)

// DiscoveryUrl is exported
const DiscoveryURL = "https://discovery.hub.docker.com/v1"

// Discovery is exported
type Discovery struct {
	heartbeat time.Duration
	ttl       time.Duration
	url       string
	token     string
}

func init() {
	Init()
}

// Init is exported
func Init() {
	discovery.Register("token", &Discovery{})
}

// Initialize is exported
func (s *Discovery) Initialize(urltoken string, heartbeat time.Duration, ttl time.Duration) error {
	if i := strings.LastIndex(urltoken, "/"); i != -1 {
		s.url = "https://" + urltoken[:i]
		s.token = urltoken[i+1:]
	} else {
		s.url = DiscoveryURL
		s.token = urltoken
	}

	if s.token == "" {
		return errors.New("token is empty")
	}
	s.heartbeat = heartbeat
	s.ttl = ttl

	return nil
}

// Fetch returns the list of entries for the discovery service at the specified endpoint
func (s *Discovery) fetch() (discovery.Entries, error) {
	resp, err := http.Get(fmt.Sprintf("%s/%s/%s", s.url, "clusters", s.token))
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	var addrs []string
	if resp.StatusCode == http.StatusOK {
		if err := json.NewDecoder(resp.Body).Decode(&addrs); err != nil {
			return nil, fmt.Errorf("Failed to decode response: %v", err)
		}
	} else {
		return nil, fmt.Errorf("Failed to fetch entries, Discovery service returned %d HTTP status code", resp.StatusCode)
	}

	return discovery.CreateEntries(addrs)
}

// Watch is exported
func (s *Discovery) Watch(stopCh <-chan struct{}) (<-chan discovery.Entries, <-chan error) {
	ch := make(chan discovery.Entries)
	ticker := time.NewTicker(s.heartbeat)
	errCh := make(chan error)

	go func() {
		defer close(ch)
		defer close(errCh)

		// Send the initial entries if available.
		currentEntries, err := s.fetch()
		if err != nil {
			errCh <- err
		} else {
			ch <- currentEntries
		}

		// Periodically send updates.
		for {
			select {
			case <-ticker.C:
				newEntries, err := s.fetch()
				if err != nil {
					errCh <- err
					continue
				}

				// Check if the file has really changed.
				if !newEntries.Equals(currentEntries) {
					ch <- newEntries
				}
				currentEntries = newEntries
			case <-stopCh:
				ticker.Stop()
				return
			}
		}
	}()

	return ch, nil
}

// Register adds a new entry identified by the into the discovery service
func (s *Discovery) Register(addr string) error {
	buf := strings.NewReader(addr)

	resp, err := http.Post(fmt.Sprintf("%s/%s/%s?ttl=%d", s.url,
		"clusters", s.token, uint64(s.ttl.Seconds())), "application/json", buf)

	if err != nil {
		return err
	}

	resp.Body.Close()
	return nil
}

// CreateCluster returns a unique cluster token
func (s *Discovery) CreateCluster() (string, error) {
	resp, err := http.Post(fmt.Sprintf("%s/%s", s.url, "clusters"), "", nil)
	if err != nil {
		return "", err
	}

	defer resp.Body.Close()
	token, err := ioutil.ReadAll(resp.Body)
	return string(token), err
}
<commit_msg>Fix token discovery, error chan for Watch was not passed back to the caller<commit_after>package token

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/docker/swarm/discovery"
)

// DiscoveryUrl is exported
const DiscoveryURL = "https://discovery.hub.docker.com/v1"

// Discovery is exported
type Discovery struct {
	heartbeat time.Duration
	ttl       time.Duration
	url       string
	token     string
}

func init() {
	Init()
}

// Init is exported
func Init() {
	discovery.Register("token", &Discovery{})
}

// Initialize is exported
func (s *Discovery) Initialize(urltoken string, heartbeat time.Duration, ttl time.Duration) error {
	if i := strings.LastIndex(urltoken, "/"); i != -1 {
		s.url = "https://" + urltoken[:i]
		s.token = urltoken[i+1:]
	} else {
		s.url = DiscoveryURL
		s.token = urltoken
	}

	if s.token == "" {
		return errors.New("token is empty")
	}
	s.heartbeat = heartbeat
	s.ttl = ttl

	return nil
}

// Fetch returns the list of entries for the discovery service at the specified endpoint
func (s *Discovery) fetch() (discovery.Entries, error) {
	resp, err := http.Get(fmt.Sprintf("%s/%s/%s", s.url, "clusters", s.token))
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	var addrs []string
	if resp.StatusCode == http.StatusOK {
		if err := json.NewDecoder(resp.Body).Decode(&addrs); err != nil {
			return nil, fmt.Errorf("Failed to decode response: %v", err)
		}
	} else {
		return nil, fmt.Errorf("Failed to fetch entries, Discovery service returned %d HTTP status code", resp.StatusCode)
	}

	return discovery.CreateEntries(addrs)
}

// Watch is exported
func (s *Discovery) Watch(stopCh <-chan struct{}) (<-chan discovery.Entries, <-chan error) {
	ch := make(chan discovery.Entries)
	ticker := time.NewTicker(s.heartbeat)
	errCh := make(chan error)

	go func() {
		defer close(ch)
		defer close(errCh)

		// Send the initial entries if available.
		currentEntries, err := s.fetch()
		if err != nil {
			errCh <- err
		} else {
			ch <- currentEntries
		}

		// Periodically send updates.
		for {
			select {
			case <-ticker.C:
				newEntries, err := s.fetch()
				if err != nil {
					errCh <- err
					continue
				}

				// Check if the file has really changed.
				if !newEntries.Equals(currentEntries) {
					ch <- newEntries
				}
				currentEntries = newEntries
			case <-stopCh:
				ticker.Stop()
				return
			}
		}
	}()

	return ch, errCh
}

// Register adds a new entry identified by the into the discovery service
func (s *Discovery) Register(addr string) error {
	buf := strings.NewReader(addr)

	resp, err := http.Post(fmt.Sprintf("%s/%s/%s?ttl=%d", s.url,
		"clusters", s.token, uint64(s.ttl.Seconds())), "application/json", buf)

	if err != nil {
		return err
	}

	resp.Body.Close()
	return nil
}

// CreateCluster returns a unique cluster token
func (s *Discovery) CreateCluster() (string, error) {
	resp, err := http.Post(fmt.Sprintf("%s/%s", s.url, "clusters"), "", nil)
	if err != nil {
		return "", err
	}

	defer resp.Body.Close()
	token, err := ioutil.ReadAll(resp.Body)
	return string(token), err
}
<|endoftext|>


<commit_before>package db

import (
	"fmt"
	"strings"

	"github.com/NetSys/di/util"
)

// A Container row is created for each container specified by the policy.  Each row will
// eventually be instantiated within its corresponding cluster.
// Used only by the minion.
type Container struct {
	ID int

	Pid     int
	IP      string
	Mac     string
	SchedID string
	Image   string
	Command []string
	Labels  []string

	Placement
}

// Placement represents scheduler placement constraints.
type Placement struct {
	Exclusive map[[2]string]struct{}
}

// InsertContainer creates a new container row and inserts it into the database.
func (db Database) InsertContainer() Container {
	result := Container{ID: db.nextID()}
	db.insert(result)
	return result
}

// SelectFromContainer gets all containers in the database that satisfy 'check'.
func (db Database) SelectFromContainer(check func(Container) bool) []Container {
	var result []Container
	for _, row := range db.tables[ContainerTable].rows {
		if check == nil || check(row.(Container)) {
			result = append(result, row.(Container))
		}
	}

	return result
}

// SelectFromContainer gets all containers in the database that satisfy the 'check'.
func (conn Conn) SelectFromContainer(check func(Container) bool) []Container {
	var containers []Container
	conn.Transact(func(view Database) error {
		containers = view.SelectFromContainer(check)
		return nil
	})
	return containers
}

func (c Container) String() string {
	cmdStr := strings.Join(append([]string{"run", c.Image}, c.Command...), " ")
	tags := []string{cmdStr}

	if c.SchedID != "" {
		id := util.ShortUUID(c.SchedID)
		tags = append(tags, fmt.Sprintf("SchedID: %s", id))
	}

	if len(c.Labels) > 0 {
		tags = append(tags, fmt.Sprintf("Labels: %s", c.Labels))
	}

	if len(c.Placement.Exclusive) > 0 {
		tags = append(tags, fmt.Sprintf("Env: %s", c.Placement.Exclusive))
	}

	if c.IP != "" {
		tags = append(tags, c.IP)
	}

	return fmt.Sprintf("Container-%d{%s}", c.ID, strings.Join(tags, ", "))
}

func (c Container) less(r row) bool {
	return c.ID < r.(Container).ID
}
<commit_msg>db: Print all relevant fields in container table<commit_after>package db

import (
	"fmt"
	"strings"

	"github.com/NetSys/di/util"
)

// A Container row is created for each container specified by the policy.  Each row will
// eventually be instantiated within its corresponding cluster.
// Used only by the minion.
type Container struct {
	ID int

	Pid     int
	IP      string
	Mac     string
	SchedID string
	Image   string
	Command []string
	Labels  []string

	Placement
}

// Placement represents scheduler placement constraints.
type Placement struct {
	Exclusive map[[2]string]struct{}
}

// InsertContainer creates a new container row and inserts it into the database.
func (db Database) InsertContainer() Container {
	result := Container{ID: db.nextID()}
	db.insert(result)
	return result
}

// SelectFromContainer gets all containers in the database that satisfy 'check'.
func (db Database) SelectFromContainer(check func(Container) bool) []Container {
	var result []Container
	for _, row := range db.tables[ContainerTable].rows {
		if check == nil || check(row.(Container)) {
			result = append(result, row.(Container))
		}
	}

	return result
}

// SelectFromContainer gets all containers in the database that satisfy the 'check'.
func (conn Conn) SelectFromContainer(check func(Container) bool) []Container {
	var containers []Container
	conn.Transact(func(view Database) error {
		containers = view.SelectFromContainer(check)
		return nil
	})
	return containers
}

func (c Container) String() string {
	cmdStr := strings.Join(append([]string{"run", c.Image}, c.Command...), " ")
	tags := []string{cmdStr}

	if c.SchedID != "" {
		id := util.ShortUUID(c.SchedID)
		tags = append(tags, fmt.Sprintf("SchedID: %s", id))
	}

	if c.Pid != 0 {
		tags = append(tags, fmt.Sprintf("Pid: %d", c.Pid))
	}

	if c.IP != "" {
		tags = append(tags, fmt.Sprintf("IP: %s", c.IP))
	}

	if c.Mac != "" {
		tags = append(tags, fmt.Sprintf("Mac: %s", c.Mac))
	}

	if len(c.Labels) > 0 {
		tags = append(tags, fmt.Sprintf("Labels: %s", c.Labels))
	}

	if len(c.Placement.Exclusive) > 0 {
		tags = append(tags, fmt.Sprintf("Env: %s", c.Placement.Exclusive))
	}

	return fmt.Sprintf("Container-%d{%s}", c.ID, strings.Join(tags, ", "))
}

func (c Container) less(r row) bool {
	return c.ID < r.(Container).ID
}
<|endoftext|>


<commit_before>// Copyright 2017 Tomas Machalek <tomas.machalek@gmail.com>
// Copyright 2017 Charles University, Faculty of Arts,
//                Institute of the Czech National Corpus
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package db

import (
	"database/sql"
	"fmt"
	"log"

	"github.com/czcorpus/vert-tagextract/db/colgen"
	_ "github.com/mattn/go-sqlite3" // sqlite3 driver load
	"github.com/tomachalek/vertigo"
)

// prepareInsert creates a prepared statement for the INSERT
// operation.
func prepareInsert(database *sql.Tx, cols []string) *sql.Stmt {
	valReplac := make([]string, len(cols))
	for i := range cols {
		valReplac[i] = "?"
	}
	ans, err := database.Prepare(fmt.Sprintf("INSERT INTO item (%s) VALUES (%s)", joinArgs(cols), joinArgs(valReplac)))
	if err != nil {
		panic(err)
	}
	return ans
}

// TTExtractor handles writing parsed data
// to a sqlite3 database. Parsed values are
// received pasivelly by implementing vertigo.LineProcessor
type TTExtractor struct {
	lineCounter        int
	atomCounter        int
	tokenInAtomCounter int
	corpusID           string
	database           *sql.DB
	transaction        *sql.Tx
	insertStatement    *sql.Stmt
	stack              *structStack
	atomStruct         string
	structures         map[string][]string
	attrNames          []string
	colgenFn           colgen.AlignedColGenFn
}

// ProcToken is a part of vertigo.LineProcessor implementation.
// It is called by Vertigo parser when a token line is encountered.
func (tte *TTExtractor) ProcToken(tk *vertigo.Token) {
	tte.lineCounter++
	tte.tokenInAtomCounter++
}

// ProcStructClose is a part of vertigo.LineProcessor implementation.
// It is called by Vertigo parser when a closing structure tag is
// encountered.
func (tte *TTExtractor) ProcStructClose(st *vertigo.StructureClose) {
	tte.stack.Pop()
	tte.lineCounter++
}

// acceptAttr tests whether a structural attribute
// [structName].[attrName] is configured (see _example/*.json) to be imported
func (tte *TTExtractor) acceptAttr(structName string, attrName string) bool {
	tmp := tte.structures[structName]
	for _, v := range tmp {
		if v == attrName {
			return true
		}
	}
	return false
}

// ProcStruct is a part of vertigo.LineProcessor implementation.
// It si called by Vertigo parser when an opening structure tag
// is encountered.
func (tte *TTExtractor) ProcStruct(st *vertigo.Structure) {
	tte.stack.Push(st)
	if st.Name == tte.atomStruct {
		attrs := make(map[string]interface{})
		tte.stack.forEachAttr(func(s string, k string, v string) {
			if tte.acceptAttr(s, k) {
				attrs[fmt.Sprintf("%s_%s", s, k)] = v
			}
		})
		if tte.atomCounter == 0 {
			tte.attrNames = make([]string, len(attrs)+4)
			i := 0
			for k := range attrs {
				tte.attrNames[i] = k
				i++
			}
			tte.attrNames[i] = "wordcount"
			tte.attrNames[i+1] = "poscount"
			tte.attrNames[i+2] = "corpus_id"
			if tte.colgenFn != nil {
				tte.attrNames[i+3] = "item_id"

			} else {
				tte.attrNames = tte.attrNames[:i+3]
			}
			tte.insertStatement = prepareInsert(tte.transaction, tte.attrNames)
		}
		attrs["wordcount"] = 0
		attrs["poscount"] = tte.tokenInAtomCounter
		attrs["corpus_id"] = tte.corpusID
		if tte.colgenFn != nil {
			attrs["item_id"] = tte.colgenFn(attrs)
		}
		values := make([]interface{}, len(tte.attrNames))
		for i, n := range tte.attrNames {
			if attrs[n] != nil {
				values[i] = attrs[n]

			} else {
				values[i] = "" // liveattrs plug-in does not like NULLs
			}
		}
		_, err := tte.insertStatement.Exec(values...)
		if err != nil {
			log.Fatalf("Failed to insert data: %s", err)
		}
		tte.atomCounter++
		tte.tokenInAtomCounter = 0
	}
	tte.lineCounter++
}

// Run starts the parsing and metadata extraction
// process. The method expects a proper database
// schema to be ready (see database.go for details).
// The whole process runs within a transaction which
// makes sqlite3 inserts a few orders of magnitude
// faster.
func (tte *TTExtractor) Run(conf *vertigo.ParserConf) {
	log.Print("Starting to process the vertical file...")
	tte.database.Exec("PRAGMA synchronous = OFF")
	tte.database.Exec("PRAGMA journal_mode = MEMORY")
	var err error
	tte.transaction, err = tte.database.Begin()
	if err != nil {
		log.Fatalf("Failed to start a database transaction: %s", err)
	}
	parserErr := vertigo.ParseVerticalFile(conf, tte)
	if parserErr != nil {
		tte.transaction.Rollback()
		log.Fatalf("Failed to parse vertical file: %s", parserErr)

	} else {
		tte.transaction.Commit()
		log.Print("...DONE")
	}
}

// NewTTExtractor is a factory function to
// instantiate proper TTExtractor.
func NewTTExtractor(database *sql.DB, corpusID string, atomStruct string, structures map[string][]string,
	colgenFn colgen.AlignedColGenFn) *TTExtractor {
	return &TTExtractor{
		database:   database,
		corpusID:   corpusID,
		atomStruct: atomStruct,
		structures: structures,
		stack:      &structStack{},
		colgenFn:   colgenFn,
	}
}
<commit_msg>Fix calculation of poscount<commit_after>// Copyright 2017 Tomas Machalek <tomas.machalek@gmail.com>
// Copyright 2017 Charles University, Faculty of Arts,
//                Institute of the Czech National Corpus
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package db

import (
	"database/sql"
	"fmt"
	"log"

	"github.com/czcorpus/vert-tagextract/db/colgen"
	_ "github.com/mattn/go-sqlite3" // sqlite3 driver load
	"github.com/tomachalek/vertigo"
)

// prepareInsert creates a prepared statement for the INSERT
// operation.
func prepareInsert(database *sql.Tx, cols []string) *sql.Stmt {
	valReplac := make([]string, len(cols))
	for i := range cols {
		valReplac[i] = "?"
	}
	ans, err := database.Prepare(fmt.Sprintf("INSERT INTO item (%s) VALUES (%s)", joinArgs(cols), joinArgs(valReplac)))
	if err != nil {
		panic(err)
	}
	return ans
}

// TTExtractor handles writing parsed data
// to a sqlite3 database. Parsed values are
// received pasivelly by implementing vertigo.LineProcessor
type TTExtractor struct {
	lineCounter        int
	atomCounter        int
	tokenInAtomCounter int
	corpusID           string
	database           *sql.DB
	transaction        *sql.Tx
	insertStatement    *sql.Stmt
	stack              *structStack
	atomStruct         string
	structures         map[string][]string
	attrNames          []string
	colgenFn           colgen.AlignedColGenFn
	currAtomAttrs      map[string]interface{}
}

// ProcToken is a part of vertigo.LineProcessor implementation.
// It is called by Vertigo parser when a token line is encountered.
func (tte *TTExtractor) ProcToken(tk *vertigo.Token) {
	tte.lineCounter++
	tte.tokenInAtomCounter++
}

// ProcStructClose is a part of vertigo.LineProcessor implementation.
// It is called by Vertigo parser when a closing structure tag is
// encountered.
func (tte *TTExtractor) ProcStructClose(st *vertigo.StructureClose) {
	tte.stack.Pop()
	tte.lineCounter++

	if st.Name == tte.atomStruct {
		tte.currAtomAttrs["poscount"] = tte.tokenInAtomCounter

		values := make([]interface{}, len(tte.attrNames))
		for i, n := range tte.attrNames {
			if tte.currAtomAttrs[n] != nil {
				values[i] = tte.currAtomAttrs[n]

			} else {
				values[i] = "" // liveattrs plug-in does not like NULLs
			}
		}
		_, err := tte.insertStatement.Exec(values...)
		if err != nil {
			log.Fatalf("Failed to insert data: %s", err)
		}
		tte.currAtomAttrs = make(map[string]interface{})
	}
}

// acceptAttr tests whether a structural attribute
// [structName].[attrName] is configured (see _example/*.json) to be imported
func (tte *TTExtractor) acceptAttr(structName string, attrName string) bool {
	tmp := tte.structures[structName]
	for _, v := range tmp {
		if v == attrName {
			return true
		}
	}
	return false
}

// ProcStruct is a part of vertigo.LineProcessor implementation.
// It si called by Vertigo parser when an opening structure tag
// is encountered.
func (tte *TTExtractor) ProcStruct(st *vertigo.Structure) {
	tte.stack.Push(st)
	if st.Name == tte.atomStruct {
		tte.tokenInAtomCounter = 0
		attrs := make(map[string]interface{})
		tte.stack.forEachAttr(func(s string, k string, v string) {
			if tte.acceptAttr(s, k) {
				attrs[fmt.Sprintf("%s_%s", s, k)] = v
			}
		})
		if tte.atomCounter == 0 {
			tte.attrNames = make([]string, len(attrs)+4)
			i := 0
			for k := range attrs {
				tte.attrNames[i] = k
				i++
			}
			tte.attrNames[i] = "wordcount"
			tte.attrNames[i+1] = "poscount"
			tte.attrNames[i+2] = "corpus_id"
			if tte.colgenFn != nil {
				tte.attrNames[i+3] = "item_id"

			} else {
				tte.attrNames = tte.attrNames[:i+3]
			}
			tte.insertStatement = prepareInsert(tte.transaction, tte.attrNames)
		}
		attrs["wordcount"] = 0 // This value is currently unused
		attrs["poscount"] = 0  // This value is updated once we hit the closing tag
		attrs["corpus_id"] = tte.corpusID
		if tte.colgenFn != nil {
			attrs["item_id"] = tte.colgenFn(attrs)
		}
		tte.currAtomAttrs = attrs
		tte.atomCounter++
	}
	tte.lineCounter++
}

// Run starts the parsing and metadata extraction
// process. The method expects a proper database
// schema to be ready (see database.go for details).
// The whole process runs within a transaction which
// makes sqlite3 inserts a few orders of magnitude
// faster.
func (tte *TTExtractor) Run(conf *vertigo.ParserConf) {
	log.Print("Starting to process the vertical file...")
	tte.database.Exec("PRAGMA synchronous = OFF")
	tte.database.Exec("PRAGMA journal_mode = MEMORY")
	var err error
	tte.transaction, err = tte.database.Begin()
	if err != nil {
		log.Fatalf("Failed to start a database transaction: %s", err)
	}
	parserErr := vertigo.ParseVerticalFile(conf, tte)
	if parserErr != nil {
		tte.transaction.Rollback()
		log.Fatalf("Failed to parse vertical file: %s", parserErr)

	} else {
		tte.transaction.Commit()
		log.Print("...DONE")
	}
}

// NewTTExtractor is a factory function to
// instantiate proper TTExtractor.
func NewTTExtractor(database *sql.DB, corpusID string, atomStruct string, structures map[string][]string,
	colgenFn colgen.AlignedColGenFn) *TTExtractor {
	return &TTExtractor{
		database:   database,
		corpusID:   corpusID,
		atomStruct: atomStruct,
		structures: structures,
		stack:      &structStack{},
		colgenFn:   colgenFn,
	}
}
<|endoftext|>


<commit_before>package oak

import (
	"bufio"
	"fmt"
	"io"
	"reflect"
	"strconv"
	"strings"

	"github.com/oakmound/oak/oakerr"

	"github.com/davecgh/go-spew/spew"
	"github.com/oakmound/oak/collision"
	"github.com/oakmound/oak/dlog"
	"github.com/oakmound/oak/event"
	"github.com/oakmound/oak/mouse"
	"github.com/oakmound/oak/render"
	"github.com/oakmound/oak/render/mod"
)

var (
	viewportLocked = true
	commands       = make(map[string]func([]string))
)

// AddCommand adds a console command to call fn when
// '<s> <args>' is input to the console. fn will be called
// with args split on whitespace.
func AddCommand(s string, fn func([]string)) error {
	if _, ok := commands[s]; ok {
		return oakerr.ExistingElement{
			InputName:   "s",
			InputType:   "string",
			Overwritten: false,
		}
	}
	commands[s] = fn
	return nil
}

// ResetCommands will throw out all existing debug commands from the
// debug console.
func ResetCommands() {
	commands = map[string]func([]string){}
}

func debugConsole(resetCh, skipScene chan bool, input io.Reader) {
	scanner := bufio.NewScanner(input)
	spew.Config.DisableMethods = true
	spew.Config.MaxDepth = 2

	// built in commands
	if conf.LoadBuiltinCommands {
		dlog.ErrorCheck(AddCommand("viewport", viewportCommands))
		dlog.ErrorCheck(AddCommand("fade", fadeCommands))
		dlog.ErrorCheck(AddCommand("skip", skipCommands(skipScene)))
		dlog.ErrorCheck(AddCommand("print", printCommands))
		dlog.ErrorCheck(AddCommand("mouse", mouseCommands))
		dlog.ErrorCheck(AddCommand("move", moveWindow))
		dlog.ErrorCheck(AddCommand("fullscreen", fullScreen))
		dlog.ErrorCheck(AddCommand("quit", func([]string) { Quit() }))
	}

	for {
		select {
		case <-resetCh: //reset all vars in debug console that save state
			viewportLocked = true
		default:
		}
		for scanner.Scan() {
			tokenString := strings.Fields(scanner.Text())
			if len(tokenString) == 0 {
				continue
			}
			if fn, ok := commands[tokenString[0]]; ok {
				fn(tokenString[1:])
			} else {
				fmt.Println("Unknown command", tokenString[0])
			}
		}
	}
}

func parseTokenAsInt(tokenString []string, arrIndex int, defaultVal int) int {
	if len(tokenString) > arrIndex {
		tmp, err := strconv.Atoi(tokenString[arrIndex])
		if err == nil {
			return tmp
		}
	}
	return defaultVal
}

func mouseDetails(nothing int, mevent interface{}) int {
	me := mevent.(mouse.Event)
	x := int(me.X()) + ViewPos.X
	y := int(me.Y()) + ViewPos.Y
	loc := collision.NewUnassignedSpace(float64(x), float64(y), 16, 16)
	results := collision.Hits(loc)
	fmt.Println("Mouse at:", x, y, "rel:", me.X(), me.Y())
	if len(results) == 0 {
		results = mouse.Hits(loc)
	}
	if len(results) > 0 {
		i := int(results[0].CID)
		if i > 0 && event.HasEntity(i) {
			e := event.GetEntity(i)
			spew.Dump(e)
		} else {
			fmt.Println("No entity ", i)
		}
	}

	return 0
}

func viewportCommands(tokenString []string) {
	switch tokenString[0] {
	case "unlock":
		if viewportLocked {
			speed := parseTokenAsInt(tokenString, 1, 5)
			viewportLocked = false
			event.GlobalBind(moveViewportBinding(speed), event.Enter)
		} else {
			fmt.Println("Viewport is already unbound")
		}
	case "lock":
		if viewportLocked {
			fmt.Println("Viewport is already locked")
		} else {
			viewportLocked = true
		}
	default:
		fmt.Println("Unrecognized command for viewport")
	}
}

func fadeCommands(tokenString []string) {
	toFade, ok := render.GetDebugRenderable(tokenString[0])
	fadeVal := parseTokenAsInt(tokenString, 1, 255)
	if ok {
		toFade.(render.Modifiable).Filter(mod.Fade(fadeVal))
	} else {
		fmt.Println("Could not fade input")
	}
}

func skipCommands(skipScene chan bool) func(tokenString []string) {
	return func(tokenString []string) {
		switch tokenString[0] {
		case "scene":
			skipScene <- true
		default:
			fmt.Println("Bad Skip Input")
		}
	}
}

func printCommands(tokenString []string) {
	if i, err := strconv.Atoi(tokenString[0]); err == nil {
		if i > 0 && event.HasEntity(i) {
			e := event.GetEntity(i)
			fmt.Println(reflect.TypeOf(e), e)
		} else {
			fmt.Println("No entity ", i)
		}
	} else {
		fmt.Println("Unable to parse", tokenString[0])
	}
}

func mouseCommands(tokenString []string) {
	switch tokenString[0] {
	case "details":
		event.GlobalBind(mouseDetails, "MouseRelease")
	default:
		fmt.Println("Bad Mouse Input")
	}
}

func moveWindow(in []string) {
	if len(in) < 4 {
		dlog.Error("Insufficient integer arguments for moving window")
		return
	}
	ints := make([]int, 4)
	var err error
	for i := range ints {
		ints[i], err = strconv.Atoi(in[i])
		if err != nil {
			dlog.Error(err)
			return
		}
	}
	err = MoveWindow(ints[0], ints[1], ints[2], ints[3])
	dlog.ErrorCheck(err)
}

func fullScreen(sub []string) {
	on := true
	if len(sub) > 0 {
		if sub[0] == "off" {
			on = false
		}
	}
	err := SetFullScreen(on)
	dlog.ErrorCheck(err)
}
<commit_msg>Add a log when commands are added to the debug console<commit_after>package oak

import (
	"bufio"
	"fmt"
	"io"
	"reflect"
	"strconv"
	"strings"

	"github.com/oakmound/oak/oakerr"

	"github.com/davecgh/go-spew/spew"
	"github.com/oakmound/oak/collision"
	"github.com/oakmound/oak/dlog"
	"github.com/oakmound/oak/event"
	"github.com/oakmound/oak/mouse"
	"github.com/oakmound/oak/render"
	"github.com/oakmound/oak/render/mod"
)

var (
	viewportLocked = true
	commands       = make(map[string]func([]string))
)

// AddCommand adds a console command to call fn when
// '<s> <args>' is input to the console. fn will be called
// with args split on whitespace.
func AddCommand(s string, fn func([]string)) error {
	if _, ok := commands[s]; ok {
		return oakerr.ExistingElement{
			InputName:   "s",
			InputType:   "string",
			Overwritten: false,
		}
	}
	dlog.Info("Adding command", s)
	commands[s] = fn
	return nil
}

// ResetCommands will throw out all existing debug commands from the
// debug console.
func ResetCommands() {
	commands = map[string]func([]string){}
}

func debugConsole(resetCh, skipScene chan bool, input io.Reader) {
	scanner := bufio.NewScanner(input)
	spew.Config.DisableMethods = true
	spew.Config.MaxDepth = 2

	// built in commands
	if conf.LoadBuiltinCommands {
		dlog.ErrorCheck(AddCommand("viewport", viewportCommands))
		dlog.ErrorCheck(AddCommand("fade", fadeCommands))
		dlog.ErrorCheck(AddCommand("skip", skipCommands(skipScene)))
		dlog.ErrorCheck(AddCommand("print", printCommands))
		dlog.ErrorCheck(AddCommand("mouse", mouseCommands))
		dlog.ErrorCheck(AddCommand("move", moveWindow))
		dlog.ErrorCheck(AddCommand("fullscreen", fullScreen))
		dlog.ErrorCheck(AddCommand("quit", func([]string) { Quit() }))
	}

	for {
		select {
		case <-resetCh: //reset all vars in debug console that save state
			viewportLocked = true
		default:
		}
		for scanner.Scan() {
			tokenString := strings.Fields(scanner.Text())
			if len(tokenString) == 0 {
				continue
			}
			if fn, ok := commands[tokenString[0]]; ok {
				fn(tokenString[1:])
			} else {
				fmt.Println("Unknown command", tokenString[0])
			}
		}
	}
}

func parseTokenAsInt(tokenString []string, arrIndex int, defaultVal int) int {
	if len(tokenString) > arrIndex {
		tmp, err := strconv.Atoi(tokenString[arrIndex])
		if err == nil {
			return tmp
		}
	}
	return defaultVal
}

func mouseDetails(nothing int, mevent interface{}) int {
	me := mevent.(mouse.Event)
	x := int(me.X()) + ViewPos.X
	y := int(me.Y()) + ViewPos.Y
	loc := collision.NewUnassignedSpace(float64(x), float64(y), 16, 16)
	results := collision.Hits(loc)
	fmt.Println("Mouse at:", x, y, "rel:", me.X(), me.Y())
	if len(results) == 0 {
		results = mouse.Hits(loc)
	}
	if len(results) > 0 {
		i := int(results[0].CID)
		if i > 0 && event.HasEntity(i) {
			e := event.GetEntity(i)
			spew.Dump(e)
		} else {
			fmt.Println("No entity ", i)
		}
	}

	return 0
}

func viewportCommands(tokenString []string) {
	switch tokenString[0] {
	case "unlock":
		if viewportLocked {
			speed := parseTokenAsInt(tokenString, 1, 5)
			viewportLocked = false
			event.GlobalBind(moveViewportBinding(speed), event.Enter)
		} else {
			fmt.Println("Viewport is already unbound")
		}
	case "lock":
		if viewportLocked {
			fmt.Println("Viewport is already locked")
		} else {
			viewportLocked = true
		}
	default:
		fmt.Println("Unrecognized command for viewport")
	}
}

func fadeCommands(tokenString []string) {
	toFade, ok := render.GetDebugRenderable(tokenString[0])
	fadeVal := parseTokenAsInt(tokenString, 1, 255)
	if ok {
		toFade.(render.Modifiable).Filter(mod.Fade(fadeVal))
	} else {
		fmt.Println("Could not fade input")
	}
}

func skipCommands(skipScene chan bool) func(tokenString []string) {
	return func(tokenString []string) {
		switch tokenString[0] {
		case "scene":
			skipScene <- true
		default:
			fmt.Println("Bad Skip Input")
		}
	}
}

func printCommands(tokenString []string) {
	if i, err := strconv.Atoi(tokenString[0]); err == nil {
		if i > 0 && event.HasEntity(i) {
			e := event.GetEntity(i)
			fmt.Println(reflect.TypeOf(e), e)
		} else {
			fmt.Println("No entity ", i)
		}
	} else {
		fmt.Println("Unable to parse", tokenString[0])
	}
}

func mouseCommands(tokenString []string) {
	switch tokenString[0] {
	case "details":
		event.GlobalBind(mouseDetails, "MouseRelease")
	default:
		fmt.Println("Bad Mouse Input")
	}
}

func moveWindow(in []string) {
	if len(in) < 4 {
		dlog.Error("Insufficient integer arguments for moving window")
		return
	}
	ints := make([]int, 4)
	var err error
	for i := range ints {
		ints[i], err = strconv.Atoi(in[i])
		if err != nil {
			dlog.Error(err)
			return
		}
	}
	err = MoveWindow(ints[0], ints[1], ints[2], ints[3])
	dlog.ErrorCheck(err)
}

func fullScreen(sub []string) {
	on := true
	if len(sub) > 0 {
		if sub[0] == "off" {
			on = false
		}
	}
	err := SetFullScreen(on)
	dlog.ErrorCheck(err)
}
<|endoftext|>


<commit_before>package rmarsh_test

import (
	"bytes"
	"reflect"
	"testing"

	"github.com/samcday/rmarsh"
)

func testDecoder(t *testing.T, expr string, v interface{}) {
	b := rbEncode(t, expr)
	if err := rmarsh.ReadValue(bytes.NewReader(b), v); err != nil {
		t.Fatal(err)
	}
}

func TestDecoderNilPtr(t *testing.T) {
	b := true
	ptr := &b
	testDecoder(t, "nil", &ptr)

	if ptr != nil {
		t.Fatalf("ptr %+v is not nil", ptr)
	}
}

func TestDecoderBool(t *testing.T) {
	var v bool
	testDecoder(t, "true", &v)

	if v != true {
		t.Errorf("%v != true", v)
	}

	var ptr *bool
	testDecoder(t, "true", &ptr)

	if *ptr != true {
		t.Errorf("%v != true", ptr)
	}

	var silly *****bool
	testDecoder(t, "true", &silly)

	if *****silly != true {
		t.Errorf("%v != true", silly)
	}
}

func BenchmarkMapperReadTrue(b *testing.B) {
	r := newCyclicReader(rbEncode(b, "true"))
	p := rmarsh.NewParser(r)
	dec := rmarsh.NewDecoder(p)

	var v bool

	for i := 0; i < b.N; i++ {
		v = false
		p.Reset(nil)

		if err := dec.Decode(&v); err != nil {
			b.Fatal(err)
		} else if v != true {
			b.Fatalf("%v != true", v)
		}
	}
}

func TestDecoderInt(t *testing.T) {
	var n uint8
	testDecoder(t, "254", &n)
	if n != 254 {
		t.Errorf("%v != 254", n)
	}

	var un uint16
	testDecoder(t, "666", &un)
	if un != 666 {
		t.Errorf("%v != 666", un)
	}
}

func BenchmarkMapperReadUint(b *testing.B) {
	r := newCyclicReader(rbEncode(b, "0xDEAD"))
	p := rmarsh.NewParser(r)
	dec := rmarsh.NewDecoder(p)

	var n int32

	for i := 0; i < b.N; i++ {
		n = 0
		p.Reset(nil)

		if err := dec.Decode(&n); err != nil {
			b.Fatal(err)
		} else if n != 0xDEAD {
			b.Fatalf("%X != 0xDEAD", n)
		}
	}
}

func TestDecoderFloat(t *testing.T) {
	var n float32
	testDecoder(t, "123.321", &n)
	if n != 123.321 {
		t.Errorf("%v != 123.321", n)
	}
}

func TestDecoderString(t *testing.T) {
	var s string
	testDecoder(t, `"test".force_encoding("ASCII-8BIT")`, &s)
	if s != "test" {
		t.Errorf(`%v != "test"`, s)
	}
}

func TestDecoderFixnumArray(t *testing.T) {
	var arr []int
	testDecoder(t, `[123,321]`, &arr)
	if !reflect.DeepEqual(arr, []int{123, 321}) {
		t.Fatalf("%+v != [123,321]", arr)
	}
}

func TestDecoderStringArray(t *testing.T) {
	var s []string
	testDecoder(t, `["test".force_encoding("ASCII-8BIT"),"test".force_encoding("ASCII-8BIT")]`, &s)
	t.Log(s)
	if !reflect.DeepEqual(s, []string{"test", "test"}) {
		t.Errorf(`%+v != ["test", "test"]`, s)
	}
}

func TestDecoderStringLink(t *testing.T) {
	var s []*string
	testDecoder(t, `s = "test".force_encoding("ASCII-8BIT"); [s, s]`, &s)

	if *s[0] != "test" {
		t.Errorf(`%+v != "test"`, s[0])
	}

	if s[0] != s[1] {
		t.Errorf("ptrs do not match: %+v vs %+v", s[0], s[1])
	}
}

func TestDecoderArrayLink(t *testing.T) {
	var arr []*[]int
	testDecoder(t, `a = [123]; [a,a]`, &arr)

	if arr[0] != arr[1] {
		t.Errorf("ptrs do not match: %+v vs %+v", arr[0], arr[1])
	}
}

func TestDecoderStringLinkSafety(t *testing.T) {
	r := bytes.NewBuffer(rbEncode(t, `s = "test"; s2 = "foo"; [s, s, s2, s]`))
	p := rmarsh.NewParser(r)

	expectToken(t, p, rmarsh.TokenStartArray)

	dec := rmarsh.NewDecoder(p)

	var s string
	if err := dec.Decode(&s); err != nil {
		t.Fatal(err)
	}
	if s != "test" {
		t.Fatalf(`%s != "test"`, s)
	}

	if err := dec.Decode(&s); err != nil {
		t.Fatal(err)
	}
	if s != "test" {
		t.Fatalf(`%s != "test"`, s)
	}

	if err := dec.Decode(&s); err != nil {
		t.Fatal(err)
	}
	if s != "foo" {
		t.Fatalf(`%s != "foo"`, s)
	}

	if err := dec.Decode(&s); err != nil {
		t.Fatal(err)
	}
	if s != "test" {
		t.Fatalf(`%s != "test"`, s)
	}
}

func TestDecoderPtrLinkSafety(t *testing.T) {
	r := bytes.NewBuffer(rbEncode(t, `s = "test"; s2 = "foo"; [s, s, s2, s]`))
	p := rmarsh.NewParser(r)

	expectToken(t, p, rmarsh.TokenStartArray)

	dec := rmarsh.NewDecoder(p)

	var s *string
	if err := dec.Decode(&s); err != nil {
		t.Fatal(err)
	}
	if *s != "test" {
		t.Fatalf(`%s != "test"`, *s)
	}

	if err := dec.Decode(&s); err != nil {
		t.Fatal(err)
	}
	if *s != "test" {
		t.Fatalf(`%s != "test"`, *s)
	}

	if err := dec.Decode(&s); err != nil {
		t.Fatal(err)
	}
	if *s != "foo" {
		t.Fatalf(`%s != "foo"`, *s)
	}

	if err := dec.Decode(&s); err != nil {
		t.Fatal(err)
	}
	if *s != "test" {
		t.Fatalf(`%s != "test"`, *s)
	}
}
<commit_msg>decoder: rename benchmarks<commit_after>package rmarsh_test

import (
	"bytes"
	"reflect"
	"testing"

	"github.com/samcday/rmarsh"
)

func testDecoder(t *testing.T, expr string, v interface{}) {
	b := rbEncode(t, expr)
	if err := rmarsh.ReadValue(bytes.NewReader(b), v); err != nil {
		t.Fatal(err)
	}
}

func TestDecoderNilPtr(t *testing.T) {
	b := true
	ptr := &b
	testDecoder(t, "nil", &ptr)

	if ptr != nil {
		t.Fatalf("ptr %+v is not nil", ptr)
	}
}

func TestDecoderBool(t *testing.T) {
	var v bool
	testDecoder(t, "true", &v)

	if v != true {
		t.Errorf("%v != true", v)
	}

	var ptr *bool
	testDecoder(t, "true", &ptr)

	if *ptr != true {
		t.Errorf("%v != true", ptr)
	}

	var silly *****bool
	testDecoder(t, "true", &silly)

	if *****silly != true {
		t.Errorf("%v != true", silly)
	}
}

func BenchmarkDecoderReadTrue(b *testing.B) {
	r := newCyclicReader(rbEncode(b, "true"))
	p := rmarsh.NewParser(r)
	dec := rmarsh.NewDecoder(p)

	var v bool

	for i := 0; i < b.N; i++ {
		v = false
		p.Reset(nil)

		if err := dec.Decode(&v); err != nil {
			b.Fatal(err)
		} else if v != true {
			b.Fatalf("%v != true", v)
		}
	}
}

func TestDecoderInt(t *testing.T) {
	var n uint8
	testDecoder(t, "254", &n)
	if n != 254 {
		t.Errorf("%v != 254", n)
	}

	var un uint16
	testDecoder(t, "666", &un)
	if un != 666 {
		t.Errorf("%v != 666", un)
	}
}

func BenchmarkDecoderReadUint(b *testing.B) {
	r := newCyclicReader(rbEncode(b, "0xDEAD"))
	p := rmarsh.NewParser(r)
	dec := rmarsh.NewDecoder(p)

	var n int32

	for i := 0; i < b.N; i++ {
		n = 0
		p.Reset(nil)

		if err := dec.Decode(&n); err != nil {
			b.Fatal(err)
		} else if n != 0xDEAD {
			b.Fatalf("%X != 0xDEAD", n)
		}
	}
}

func TestDecoderFloat(t *testing.T) {
	var n float32
	testDecoder(t, "123.321", &n)
	if n != 123.321 {
		t.Errorf("%v != 123.321", n)
	}
}

func TestDecoderString(t *testing.T) {
	var s string
	testDecoder(t, `"test".force_encoding("ASCII-8BIT")`, &s)
	if s != "test" {
		t.Errorf(`%v != "test"`, s)
	}
}

func TestDecoderFixnumArray(t *testing.T) {
	var arr []int
	testDecoder(t, `[123,321]`, &arr)
	if !reflect.DeepEqual(arr, []int{123, 321}) {
		t.Fatalf("%+v != [123,321]", arr)
	}
}

func TestDecoderStringArray(t *testing.T) {
	var s []string
	testDecoder(t, `["test".force_encoding("ASCII-8BIT"),"test".force_encoding("ASCII-8BIT")]`, &s)
	t.Log(s)
	if !reflect.DeepEqual(s, []string{"test", "test"}) {
		t.Errorf(`%+v != ["test", "test"]`, s)
	}
}

func TestDecoderStringLink(t *testing.T) {
	var s []*string
	testDecoder(t, `s = "test".force_encoding("ASCII-8BIT"); [s, s]`, &s)

	if *s[0] != "test" {
		t.Errorf(`%+v != "test"`, s[0])
	}

	if s[0] != s[1] {
		t.Errorf("ptrs do not match: %+v vs %+v", s[0], s[1])
	}
}

func TestDecoderArrayLink(t *testing.T) {
	var arr []*[]int
	testDecoder(t, `a = [123]; [a,a]`, &arr)

	if arr[0] != arr[1] {
		t.Errorf("ptrs do not match: %+v vs %+v", arr[0], arr[1])
	}
}

func TestDecoderStringLinkSafety(t *testing.T) {
	r := bytes.NewBuffer(rbEncode(t, `s = "test"; s2 = "foo"; [s, s, s2, s]`))
	p := rmarsh.NewParser(r)

	expectToken(t, p, rmarsh.TokenStartArray)

	dec := rmarsh.NewDecoder(p)

	var s string
	if err := dec.Decode(&s); err != nil {
		t.Fatal(err)
	}
	if s != "test" {
		t.Fatalf(`%s != "test"`, s)
	}

	if err := dec.Decode(&s); err != nil {
		t.Fatal(err)
	}
	if s != "test" {
		t.Fatalf(`%s != "test"`, s)
	}

	if err := dec.Decode(&s); err != nil {
		t.Fatal(err)
	}
	if s != "foo" {
		t.Fatalf(`%s != "foo"`, s)
	}

	if err := dec.Decode(&s); err != nil {
		t.Fatal(err)
	}
	if s != "test" {
		t.Fatalf(`%s != "test"`, s)
	}
}

func TestDecoderPtrLinkSafety(t *testing.T) {
	r := bytes.NewBuffer(rbEncode(t, `s = "test"; s2 = "foo"; [s, s, s2, s]`))
	p := rmarsh.NewParser(r)

	expectToken(t, p, rmarsh.TokenStartArray)

	dec := rmarsh.NewDecoder(p)

	var s *string
	if err := dec.Decode(&s); err != nil {
		t.Fatal(err)
	}
	if *s != "test" {
		t.Fatalf(`%s != "test"`, *s)
	}

	if err := dec.Decode(&s); err != nil {
		t.Fatal(err)
	}
	if *s != "test" {
		t.Fatalf(`%s != "test"`, *s)
	}

	if err := dec.Decode(&s); err != nil {
		t.Fatal(err)
	}
	if *s != "foo" {
		t.Fatalf(`%s != "foo"`, *s)
	}

	if err := dec.Decode(&s); err != nil {
		t.Fatal(err)
	}
	if *s != "test" {
		t.Fatalf(`%s != "test"`, *s)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"runtime"
	"strings"
	"time"

	"cloud.google.com/go/logging"
	"github.com/cloudfoundry-community/firehose-to-syslog/caching"
	"github.com/cloudfoundry-community/gcp-tools-release/src/stackdriver-nozzle/config"
	"github.com/cloudfoundry-community/gcp-tools-release/src/stackdriver-nozzle/filter"
	"github.com/cloudfoundry-community/gcp-tools-release/src/stackdriver-nozzle/firehose"
	"github.com/cloudfoundry-community/gcp-tools-release/src/stackdriver-nozzle/heartbeat"
	"github.com/cloudfoundry-community/gcp-tools-release/src/stackdriver-nozzle/nozzle"
	"github.com/cloudfoundry-community/gcp-tools-release/src/stackdriver-nozzle/stackdriver"
	"github.com/cloudfoundry-community/gcp-tools-release/src/stackdriver-nozzle/version"
	"github.com/cloudfoundry-community/go-cfclient"
	"github.com/cloudfoundry/lager"
)

func main() {
	a := newApp()

	defer handleFatalError(a)

	producer := a.newProducer()
	consumer := a.newConsumer()

	errs, fhErrs := consumer.Start(producer)
	defer consumer.Stop()

	go func() {
		for err := range errs {
			a.logger.Error("nozzle", err)
		}
	}()

	fatalErr := <-fhErrs
	if fatalErr != nil {
		a.logger.Fatal("firehose", fatalErr)
	}
}

func handleFatalError(a *app) {
	if !a.c.DebugNozzle {
		return
	}

	if e := recover(); e != nil {
		stack := make([]byte, 1<<16)
		stackSize := runtime.Stack(stack, true)
		stackTrace := string(stack[:stackSize])

		payload := map[string]interface{}{
			"serviceContext": map[string]interface{}{
				"service": version.Name,
				"version": version.Release,
			},
			"message": stackTrace,
		}

		log := &stackdriver.Log{
			Payload:  payload,
			Labels:   map[string]string{},
			Severity: logging.Error,
		}

		logAdapter, _ := a.newLogAdapter()
		logAdapter.PostLog(log)
		logAdapter.Flush()

		// Re-throw the error, we want to ensure it's logged directly to
		// stackdriver but we are not in a recoverable state.
		panic(e)
	}
}

func newApp() *app {
	logger := lager.NewLogger("stackdriver-nozzle")
	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.DEBUG))

	c, err := config.NewConfig()
	if err != nil {
		logger.Fatal("config", err)
	}

	logger.Info("arguments", c.ToData())

	trigger := time.NewTicker(time.Duration(c.HeartbeatRate) * time.Second).C
	heartbeater := heartbeat.NewHeartbeater(logger, trigger)

	cfConfig := &cfclient.Config{
		ApiAddress:        c.APIEndpoint,
		Username:          c.Username,
		Password:          c.Password,
		SkipSslValidation: c.SkipSSL}
	cfClient := cfclient.NewClient(cfConfig)

	var cachingClient caching.Caching
	if c.ResolveAppMetadata {
		cachingClient = caching.NewCachingBolt(cfClient, c.BoltDBPath)
	} else {
		cachingClient = caching.NewCachingEmpty()
	}
	cachingClient.CreateBucket()
	labelMaker := nozzle.NewLabelMaker(cachingClient)

	return &app{
		logger:      logger,
		c:           c,
		cfConfig:    cfConfig,
		cfClient:    cfClient,
		labelMaker:  labelMaker,
		heartbeater: heartbeater,
	}
}

type app struct {
	logger      lager.Logger
	c           *config.Config
	cfConfig    *cfclient.Config
	cfClient    *cfclient.Client
	labelMaker  nozzle.LabelMaker
	heartbeater heartbeat.Heartbeater
}

func (a *app) newProducer() firehose.Client {
	fhClient := firehose.NewClient(a.cfConfig, a.cfClient, a.c.SubscriptionID)

	producer, err := filter.New(fhClient, strings.Split(a.c.Events, ","), a.heartbeater)
	if err != nil {
		a.logger.Fatal("filter", err)
	}

	return producer
}

func (a *app) newConsumer() *nozzle.Nozzle {
	return &nozzle.Nozzle{
		LogSink:     a.newLogSink(),
		MetricSink:  a.newMetricSink(),
		Heartbeater: a.heartbeater,
	}
}

func (a *app) newLogSink() nozzle.Sink {
	logAdapter, logErrs := a.newLogAdapter()
	go func() {
		err := <-logErrs
		a.logger.Error("logAdapter", err)
	}()

	return nozzle.NewLogSink(a.labelMaker, logAdapter)
}

func (a *app) newLogAdapter() (stackdriver.LogAdapter, <-chan error) {
	return stackdriver.NewLogAdapter(
		a.c.ProjectID,
		a.c.BatchCount,
		time.Duration(a.c.BatchDuration)*time.Second,
		a.heartbeater,
	)
}

func (a *app) newMetricSink() nozzle.Sink {
	metricClient, err := stackdriver.NewMetricClient()
	if err != nil {
		a.logger.Fatal("metricClient", err)
	}

	metricAdapter, err := stackdriver.NewMetricAdapter(a.c.ProjectID, metricClient, a.heartbeater)
	if err != nil {
		a.logger.Error("metricAdapter", err)
	}

	metricBuffer, errs := stackdriver.NewMetricsBuffer(a.c.BatchCount, metricAdapter)
	go func() {
		for err = range errs {
			a.logger.Error("metricsBuffer", err)
		}
	}()

	return nozzle.NewMetricSink(a.labelMaker, metricBuffer, nozzle.NewUnitParser())
}
<commit_msg>add comment explaining why we get a new logAdapter for logging nozzle panics<commit_after>package main

import (
	"os"
	"runtime"
	"strings"
	"time"

	"cloud.google.com/go/logging"
	"github.com/cloudfoundry-community/firehose-to-syslog/caching"
	"github.com/cloudfoundry-community/gcp-tools-release/src/stackdriver-nozzle/config"
	"github.com/cloudfoundry-community/gcp-tools-release/src/stackdriver-nozzle/filter"
	"github.com/cloudfoundry-community/gcp-tools-release/src/stackdriver-nozzle/firehose"
	"github.com/cloudfoundry-community/gcp-tools-release/src/stackdriver-nozzle/heartbeat"
	"github.com/cloudfoundry-community/gcp-tools-release/src/stackdriver-nozzle/nozzle"
	"github.com/cloudfoundry-community/gcp-tools-release/src/stackdriver-nozzle/stackdriver"
	"github.com/cloudfoundry-community/gcp-tools-release/src/stackdriver-nozzle/version"
	"github.com/cloudfoundry-community/go-cfclient"
	"github.com/cloudfoundry/lager"
)

func main() {
	a := newApp()

	defer handleFatalError(a)

	producer := a.newProducer()
	consumer := a.newConsumer()

	errs, fhErrs := consumer.Start(producer)
	defer consumer.Stop()

	go func() {
		for err := range errs {
			a.logger.Error("nozzle", err)
		}
	}()

	fatalErr := <-fhErrs
	if fatalErr != nil {
		a.logger.Fatal("firehose", fatalErr)
	}
}

func handleFatalError(a *app) {
	if !a.c.DebugNozzle {
		return
	}

	if e := recover(); e != nil {
		stack := make([]byte, 1<<16)
		stackSize := runtime.Stack(stack, true)
		stackTrace := string(stack[:stackSize])

		payload := map[string]interface{}{
			"serviceContext": map[string]interface{}{
				"service": version.Name,
				"version": version.Release,
			},
			"message": stackTrace,
		}

		log := &stackdriver.Log{
			Payload:  payload,
			Labels:   map[string]string{},
			Severity: logging.Error,
		}

		// Purposefully get a new log adapter here since there
		// were issues re-using the one that the nozzle uses.
		logAdapter, _ := a.newLogAdapter()
		logAdapter.PostLog(log)
		logAdapter.Flush()

		// Re-throw the error, we want to ensure it's logged directly to
		// stackdriver but we are not in a recoverable state.
		panic(e)
	}
}

func newApp() *app {
	logger := lager.NewLogger("stackdriver-nozzle")
	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.DEBUG))

	c, err := config.NewConfig()
	if err != nil {
		logger.Fatal("config", err)
	}

	logger.Info("arguments", c.ToData())

	trigger := time.NewTicker(time.Duration(c.HeartbeatRate) * time.Second).C
	heartbeater := heartbeat.NewHeartbeater(logger, trigger)

	cfConfig := &cfclient.Config{
		ApiAddress:        c.APIEndpoint,
		Username:          c.Username,
		Password:          c.Password,
		SkipSslValidation: c.SkipSSL}
	cfClient := cfclient.NewClient(cfConfig)

	var cachingClient caching.Caching
	if c.ResolveAppMetadata {
		cachingClient = caching.NewCachingBolt(cfClient, c.BoltDBPath)
	} else {
		cachingClient = caching.NewCachingEmpty()
	}
	cachingClient.CreateBucket()
	labelMaker := nozzle.NewLabelMaker(cachingClient)

	return &app{
		logger:      logger,
		c:           c,
		cfConfig:    cfConfig,
		cfClient:    cfClient,
		labelMaker:  labelMaker,
		heartbeater: heartbeater,
	}
}

type app struct {
	logger      lager.Logger
	c           *config.Config
	cfConfig    *cfclient.Config
	cfClient    *cfclient.Client
	labelMaker  nozzle.LabelMaker
	heartbeater heartbeat.Heartbeater
}

func (a *app) newProducer() firehose.Client {
	fhClient := firehose.NewClient(a.cfConfig, a.cfClient, a.c.SubscriptionID)

	producer, err := filter.New(fhClient, strings.Split(a.c.Events, ","), a.heartbeater)
	if err != nil {
		a.logger.Fatal("filter", err)
	}

	return producer
}

func (a *app) newConsumer() *nozzle.Nozzle {
	return &nozzle.Nozzle{
		LogSink:     a.newLogSink(),
		MetricSink:  a.newMetricSink(),
		Heartbeater: a.heartbeater,
	}
}

func (a *app) newLogSink() nozzle.Sink {
	logAdapter, logErrs := a.newLogAdapter()
	go func() {
		err := <-logErrs
		a.logger.Error("logAdapter", err)
	}()

	return nozzle.NewLogSink(a.labelMaker, logAdapter)
}

func (a *app) newLogAdapter() (stackdriver.LogAdapter, <-chan error) {
	return stackdriver.NewLogAdapter(
		a.c.ProjectID,
		a.c.BatchCount,
		time.Duration(a.c.BatchDuration)*time.Second,
		a.heartbeater,
	)
}

func (a *app) newMetricSink() nozzle.Sink {
	metricClient, err := stackdriver.NewMetricClient()
	if err != nil {
		a.logger.Fatal("metricClient", err)
	}

	metricAdapter, err := stackdriver.NewMetricAdapter(a.c.ProjectID, metricClient, a.heartbeater)
	if err != nil {
		a.logger.Error("metricAdapter", err)
	}

	metricBuffer, errs := stackdriver.NewMetricsBuffer(a.c.BatchCount, metricAdapter)
	go func() {
		for err = range errs {
			a.logger.Error("metricsBuffer", err)
		}
	}()

	return nozzle.NewMetricSink(a.labelMaker, metricBuffer, nozzle.NewUnitParser())
}
<|endoftext|>