<commit_before>package yang

import (
	"fmt"
	"os"
	"strings"

	"github.com/openconfig/goyang/pkg/yang"
	"github.com/oshothebig/pbast"
)

var builtinMap = map[yang.TypeKind]pbast.Type{
	yang.Yint8:   pbast.Int32,
	yang.Yint16:  pbast.Int32,
	yang.Yint32:  pbast.Int32,
	yang.Yint64:  pbast.Int64,
	yang.Yuint8:  pbast.UInt32,
	yang.Yuint16: pbast.UInt32,
	yang.Yuint32: pbast.UInt32,
	yang.Yuint64: pbast.UInt64,
	yang.Ystring: pbast.String,
	yang.Ybool:   pbast.Bool,
	yang.Ybinary: pbast.Bytes,
}

var builtinTypes = stringSet{
	"int8":                struct{}{},
	"int16":               struct{}{},
	"int32":               struct{}{},
	"int64":               struct{}{},
	"uint8":               struct{}{},
	"uint16":              struct{}{},
	"unit32":              struct{}{},
	"uint64":              struct{}{},
	"string":              struct{}{},
	"boolean":             struct{}{},
	"enumeration":         struct{}{},
	"bits":                struct{}{},
	"binary":              struct{}{},
	"leafref":             struct{}{},
	"identityref":         struct{}{},
	"empty":               struct{}{},
	"union":               struct{}{},
	"instance-identifier": struct{}{},
}

type transformer struct {
	topScope           *scope
	decimal64          *pbast.Message
	leafRef            *pbast.Message
	identityRef        *pbast.Message
	instanceIdentifier *pbast.Message
	emptyNeeded        bool
}

// e must be YANG module
func Transform(e *yang.Entry) *pbast.File {
	if _, ok := e.Node.(*yang.Module); !ok {
		return nil
	}

	t := &transformer{
		topScope: newScope(),
	}

	return t.module(entry{e})
}

func (t *transformer) declare(m *pbast.Message) {
	if err := t.topScope.addType(m); err != nil {
		fmt.Fprintln(os.Stderr, err)
	}
}

func (t *transformer) reflectTo(f *pbast.File) {
	t.topScope.reflectTo(f)
	f.AddMessage(t.decimal64)
	f.AddMessage(t.leafRef)
	f.AddMessage(t.identityRef)
	f.AddMessage(t.instanceIdentifier)
	if t.emptyNeeded {
		f.AddImport(pbast.NewImport("google/protobuf/empty.proto"))
	}
}

func (t *transformer) module(e entry) *pbast.File {
	namespace := e.Namespace().Name
	f := pbast.NewFile(pbast.NewPackageWithElements(guessElements(namespace)))

	f.Comment = t.moduleComment(e)

	root := t.buildMessage("Root", e)
	// Clear Root messgage comment because it overlaps with
	// the file level comment being generated from module description too
	root.Comment = nil
	// Child nodes are enclosed with Root message
	f.AddMessage(root)

	// RPCs
	s := t.rpcs(e)
	f.AddService(s)

	// Notifications
	n := t.notifications(e)
	f.AddService(n)

	t.reflectTo(f)

	return f
}

func (t *transformer) moduleComment(e entry) pbast.Comment {
	description := t.description(e)
	namespace := t.namespace(e)
	revisions := t.revisions(e)
	reference := t.reference(e)

	var comment []string
	comment = append(comment, description...)
	comment = append(comment, namespace...)
	comment = append(comment, revisions...)
	comment = append(comment, reference...)

	return comment
}

func (t *transformer) genericComments(e entry) pbast.Comment {
	description := t.description(e)
	reference := t.reference(e)

	comments := append(description, reference...)
	return comments
}

func (t *transformer) description(e entry) pbast.Comment {
	description := e.Description
	if e.Description == "" {
		return nil
	}

	lines := strings.Split(strings.TrimRight(description, "\n "), "\n")

	ret := make([]string, 0, len(lines)+1)
	ret = append(ret, "Description:")
	ret = append(ret, lines...)
	return ret
}

func (t *transformer) revisions(e entry) pbast.Comment {
	var lines []string
	if v := e.Extra["revision"]; len(v) > 0 {
		for _, rev := range v[0].([]*yang.Revision) {
			lines = append(lines, "Revision: "+rev.Name)
		}
	}

	return lines
}

func (t *transformer) namespace(e entry) pbast.Comment {
	namespace := e.Namespace().Name
	if namespace == "" {
		return nil
	}

	return []string{"Namespace: " + namespace}
}

func (t *transformer) reference(e entry) pbast.Comment {
	v := e.Extra["reference"]
	if len(v) == 0 {
		return nil
	}

	ref := v[0].(*yang.Value)
	if ref == nil {
		return nil
	}
	if ref.Name == "" {
		return nil
	}

	lines := strings.Split(strings.TrimRight(ref.Name, "\n "), "\n")

	ret := make([]string, 0, len(lines)+1)
	ret = append(ret, "Reference:")
	ret = append(ret, lines...)
	return ret
}

func (t *transformer) rpcs(e entry) *pbast.Service {
	rpcs := e.rpcs()
	if len(rpcs) == 0 {
		return nil
	}

	s := pbast.NewService(CamelCase(e.Name))
	for _, rpc := range rpcs {
		r := t.rpc(rpc)
		s.AddRPC(r)
	}

	return s
}

func (t *transformer) rpc(e entry) *pbast.RPC {
	method := CamelCase(e.Name)
	in := method + "Request"
	out := method + "Response"

	rpc := pbast.NewRPC(
		method,
		pbast.NewReturnType(in),
		pbast.NewReturnType(out),
	)
	rpc.Comment = t.genericComments(e)

	t.declare(t.buildMessage(in, entry{e.RPC.Input}))
	t.declare(t.buildMessage(out, entry{e.RPC.Output}))

	return rpc
}

func (t *transformer) notifications(e entry) *pbast.Service {
	notifications := e.notifications()
	if len(notifications) == 0 {
		return nil
	}

	s := pbast.NewService(CamelCase(e.Name + "Notification"))
	for _, notification := range notifications {
		n := t.notification(notification)
		n.Comment = t.genericComments(notification)
		s.AddRPC(n)
	}

	return s
}

func (t *transformer) notification(e entry) *pbast.RPC {
	const common = "Notification"
	method := CamelCase(e.Name)
	in := buildName(method, common, "Request")
	out := buildName(method, common, "Response")

	rpc := pbast.NewRPC(method, pbast.NewReturnType(in), pbast.NewReturnType(out))

	// notification statement doesn't have an input parameter equivalent,
	// then empty message is used for input as RPC
	t.declare(pbast.NewMessage(in))
	t.declare(t.buildMessage(out, e))

	return rpc
}

func (t *transformer) buildMessage(name string, e entry) *pbast.Message {
	if e.Entry == nil {
		return nil
	}

	msg := pbast.NewMessage(name)
	scope := newScope()
	msg.Comment = t.genericComments(e)
	for index, child := range e.children() {
		fieldNum := index + 1
		var field *pbast.MessageField
		switch {
		case child.Type != nil:
			typ := t.leaf(scope, child.Type, child.Name)

			if !t.useType(typ) {
				if err := scope.addType(typ); err != nil {
					fmt.Fprintln(os.Stderr, err)
				}
			}

			field = &pbast.MessageField{
				Repeated: child.ListAttr != nil,
				Type:     typ.TypeName(),
				Name:     underscoreCase(child.Name),
				Index:    fieldNum,
				Comment:  t.genericComments(child),
			}
		// list case
		case child.ListAttr != nil:
			field = t.directory(scope, child, fieldNum, true)
		// others might be container case
		default:
			field = t.directory(scope, child, fieldNum, false)
		}
		msg.AddField(field)
	}

	scope.reflectTo(msg)

	return msg
}

func (t *transformer) leaf(scope *scope, typ *yang.YangType, name string) pbast.Type {
	if isBuiltinType(typ) {
		return convertType(typ, CamelCase(name))
	}
	switch typ.Kind {
	case yang.Yenum, yang.Ybits, yang.Yunion:
		return convertType(typ, CamelCase(typ.Name))
	default:
		inner := convertType(typ, "Value")
		msg := pbast.NewMessage(CamelCase(typ.Name)).
			AddField(pbast.NewMessageField(inner, "value", 1))
		if !t.useType(inner) {
			msg.AddType(inner)
		}
		return msg
	}
}

func (t *transformer) useType(typ pbast.Type) bool {
	switch typ {
	case decimal64:
		t.decimal64 = decimal64
	case leafRef:
		t.leafRef = leafRef
	case identityRef:
		t.identityRef = identityRef
	case instanceIdentifier:
		t.instanceIdentifier = instanceIdentifier
	case pbast.Empty:
		t.emptyNeeded = true
	default:
		return false
	}

	return true
}

func isBuiltinType(typ *yang.YangType) bool {
	if _, ok := yang.TypeKindFromName[typ.Name]; !ok {
		return false
	}
	return true
}

func convertType(typ *yang.YangType, name string) pbast.Type {
	switch typ.Kind {
	case yang.Yint8, yang.Yint16, yang.Yint32:
		return pbast.Int32
	case yang.Yint64:
		return pbast.Int64
	case yang.Yuint8, yang.Yuint16, yang.Yuint32:
		return pbast.UInt32
	case yang.Yuint64:
		return pbast.UInt64
	case yang.Ystring:
		return pbast.String
	case yang.Ybool:
		return pbast.Bool
	case yang.Yenum:
		return customEnum(name, typ.Enum)
	case yang.Ybits:
		return customBits(name, typ.Bit)
	case yang.Ybinary:
		return pbast.Bytes
	case yang.Yleafref:
		return leafRef
	case yang.Yidentityref:
		return identityRef
	case yang.Yempty:
		return pbast.Empty
	case yang.Yunion:
		return customUnion(name, typ.Type)
	case yang.YinstanceIdentifier:
		return instanceIdentifier
	case yang.Ydecimal64:
		return decimal64
	default:
		return nil
	}
}

func customBits(name string, bits *yang.EnumType) *pbast.Message {
	msg := pbast.NewMessage(name)
	for i, n := range bits.Names() {
		v := 1 << uint(bits.Values()[i])
		msg.AddField(pbast.NewMessageField(pbast.Bool, n, v))
	}

	return msg
}

func customEnum(name string, e *yang.EnumType) *pbast.Message {
	enum := pbast.NewEnum("Value")
	for _, v := range e.Values() {
		n := e.ValueMap()[v]
		enum.AddField(pbast.NewEnumField(constantName(n), int(v)))
	}
	msg := pbast.NewMessage(name).
		AddEnum(enum).
		AddField(pbast.NewMessageField(enum, "value", 1))

	return msg
}

func customUnion(name string, types []*yang.YangType) *pbast.Message {
	scope := newScope()
	pbTypes := unionFields(types, nil, scope)

	oneof := pbast.NewOneOf("value")
	for i, typ := range pbTypes {
		oneof.AddField(pbast.NewOneOfField(typ, underscoreCase(typ.TypeName()), i+1))
	}

	msg := pbast.NewMessage(name).AddOneOf(oneof)
	scope.reflectTo(msg)

	return msg
}

func unionFields(types []*yang.YangType, pbTypes []pbast.Type, scope *scope) []pbast.Type {
	for _, typ := range types {
		pbtype := convertType(typ, typ.Name)
		if pbtype == nil {
			continue
		}

		if typ.Kind == yang.Yunion {
			pbTypes = unionFields(typ.Type, pbTypes, scope)
			continue
		}

		pbTypes = append(pbTypes, pbtype)
		scope.addType(pbtype)
	}

	return pbTypes
}

func (t *transformer) directory(scope *scope, e entry, index int, repeated bool) *pbast.MessageField {
	fieldName := underscoreCase(e.Name)
	typeName := CamelCase(e.Name)

	inner := t.buildMessage(typeName, e)
	field := &pbast.MessageField{
		Repeated: repeated,
		Type:     inner.TypeName(),
		Name:     fieldName,
		Index:    index,
		Comment:  t.genericComments(e),
	}

	if err := scope.addType(inner); err != nil {
		fmt.Fprintln(os.Stderr, err)
	}

	return field
}
<commit_msg>Refactor<commit_after>package yang

import (
	"fmt"
	"os"
	"strings"

	"github.com/openconfig/goyang/pkg/yang"
	"github.com/oshothebig/pbast"
)

var builtinMap = map[yang.TypeKind]pbast.Type{
	yang.Yint8:   pbast.Int32,
	yang.Yint16:  pbast.Int32,
	yang.Yint32:  pbast.Int32,
	yang.Yint64:  pbast.Int64,
	yang.Yuint8:  pbast.UInt32,
	yang.Yuint16: pbast.UInt32,
	yang.Yuint32: pbast.UInt32,
	yang.Yuint64: pbast.UInt64,
	yang.Ystring: pbast.String,
	yang.Ybool:   pbast.Bool,
	yang.Ybinary: pbast.Bytes,
}

var builtinTypes = stringSet{
	"int8":                struct{}{},
	"int16":               struct{}{},
	"int32":               struct{}{},
	"int64":               struct{}{},
	"uint8":               struct{}{},
	"uint16":              struct{}{},
	"unit32":              struct{}{},
	"uint64":              struct{}{},
	"string":              struct{}{},
	"boolean":             struct{}{},
	"enumeration":         struct{}{},
	"bits":                struct{}{},
	"binary":              struct{}{},
	"leafref":             struct{}{},
	"identityref":         struct{}{},
	"empty":               struct{}{},
	"union":               struct{}{},
	"instance-identifier": struct{}{},
}

type transformer struct {
	topScope           *scope
	decimal64          *pbast.Message
	leafRef            *pbast.Message
	identityRef        *pbast.Message
	instanceIdentifier *pbast.Message
	emptyNeeded        bool
}

// e must be YANG module
func Transform(e *yang.Entry) *pbast.File {
	if _, ok := e.Node.(*yang.Module); !ok {
		return nil
	}

	t := &transformer{
		topScope: newScope(),
	}

	return t.module(entry{e})
}

func (t *transformer) declare(m *pbast.Message) {
	if err := t.topScope.addType(m); err != nil {
		fmt.Fprintln(os.Stderr, err)
	}
}

func (t *transformer) reflectTo(f *pbast.File) {
	t.topScope.reflectTo(f)
	f.AddMessage(t.decimal64)
	f.AddMessage(t.leafRef)
	f.AddMessage(t.identityRef)
	f.AddMessage(t.instanceIdentifier)
	if t.emptyNeeded {
		f.AddImport(pbast.NewImport("google/protobuf/empty.proto"))
	}
}

func (t *transformer) module(e entry) *pbast.File {
	namespace := e.Namespace().Name
	f := pbast.NewFile(pbast.NewPackageWithElements(guessElements(namespace)))

	f.Comment = t.moduleComment(e)

	root := t.buildMessage("Root", e)
	// Clear Root messgage comment because it overlaps with
	// the file level comment being generated from module description too
	root.Comment = nil
	// Child nodes are enclosed with Root message
	f.AddMessage(root)

	// RPCs
	s := t.rpcs(e)
	f.AddService(s)

	// Notifications
	n := t.notifications(e)
	f.AddService(n)

	t.reflectTo(f)

	return f
}

func (t *transformer) moduleComment(e entry) pbast.Comment {
	description := t.description(e)
	namespace := t.namespace(e)
	revisions := t.revisions(e)
	reference := t.reference(e)

	var comment []string
	comment = append(comment, description...)
	comment = append(comment, namespace...)
	comment = append(comment, revisions...)
	comment = append(comment, reference...)

	return comment
}

func (t *transformer) genericComments(e entry) pbast.Comment {
	description := t.description(e)
	reference := t.reference(e)

	comments := append(description, reference...)
	return comments
}

func (t *transformer) description(e entry) pbast.Comment {
	description := e.Description
	if e.Description == "" {
		return nil
	}

	lines := strings.Split(strings.TrimRight(description, "\n "), "\n")

	ret := make([]string, 0, len(lines)+1)
	ret = append(ret, "Description:")
	ret = append(ret, lines...)
	return ret
}

func (t *transformer) revisions(e entry) pbast.Comment {
	var lines []string
	if v := e.Extra["revision"]; len(v) > 0 {
		for _, rev := range v[0].([]*yang.Revision) {
			lines = append(lines, "Revision: "+rev.Name)
		}
	}

	return lines
}

func (t *transformer) namespace(e entry) pbast.Comment {
	namespace := e.Namespace().Name
	if namespace == "" {
		return nil
	}

	return []string{"Namespace: " + namespace}
}

func (t *transformer) reference(e entry) pbast.Comment {
	v := e.Extra["reference"]
	if len(v) == 0 {
		return nil
	}

	ref := v[0].(*yang.Value)
	if ref == nil {
		return nil
	}
	if ref.Name == "" {
		return nil
	}

	lines := strings.Split(strings.TrimRight(ref.Name, "\n "), "\n")

	ret := make([]string, 0, len(lines)+1)
	ret = append(ret, "Reference:")
	ret = append(ret, lines...)
	return ret
}

func (t *transformer) rpcs(e entry) *pbast.Service {
	rpcs := e.rpcs()
	if len(rpcs) == 0 {
		return nil
	}

	s := pbast.NewService(CamelCase(e.Name))
	for _, rpc := range rpcs {
		r := t.rpc(rpc)
		s.AddRPC(r)
	}

	return s
}

func (t *transformer) rpc(e entry) *pbast.RPC {
	method := CamelCase(e.Name)
	in := method + "Request"
	out := method + "Response"

	rpc := pbast.NewRPC(
		method,
		pbast.NewReturnType(in),
		pbast.NewReturnType(out),
	)
	rpc.Comment = t.genericComments(e)

	t.declare(t.buildMessage(in, entry{e.RPC.Input}))
	t.declare(t.buildMessage(out, entry{e.RPC.Output}))

	return rpc
}

func (t *transformer) notifications(e entry) *pbast.Service {
	notifications := e.notifications()
	if len(notifications) == 0 {
		return nil
	}

	s := pbast.NewService(CamelCase(e.Name + "Notification"))
	for _, notification := range notifications {
		n := t.notification(notification)
		n.Comment = t.genericComments(notification)
		s.AddRPC(n)
	}

	return s
}

func (t *transformer) notification(e entry) *pbast.RPC {
	const common = "Notification"
	method := CamelCase(e.Name)
	in := buildName(method, common, "Request")
	out := buildName(method, common, "Response")

	rpc := pbast.NewRPC(method, pbast.NewReturnType(in), pbast.NewReturnType(out))

	// notification statement doesn't have an input parameter equivalent,
	// then empty message is used for input as RPC
	t.declare(pbast.NewMessage(in))
	t.declare(t.buildMessage(out, e))

	return rpc
}

func (t *transformer) buildMessage(name string, e entry) *pbast.Message {
	if e.Entry == nil {
		return nil
	}

	msg := pbast.NewMessage(name)
	scope := newScope()
	msg.Comment = t.genericComments(e)
	for index, child := range e.children() {
		fieldNum := index + 1
		var field *pbast.MessageField
		switch {
		case child.Type != nil:
			typ := t.leaf(scope, child.Type, child.Name)

			if !t.useType(typ) {
				if err := scope.addType(typ); err != nil {
					fmt.Fprintln(os.Stderr, err)
				}
			}

			field = &pbast.MessageField{
				Repeated: child.ListAttr != nil,
				Type:     typ.TypeName(),
				Name:     underscoreCase(child.Name),
				Index:    fieldNum,
				Comment:  t.genericComments(child),
			}
		// list case
		case child.ListAttr != nil:
			field = t.directory(scope, child, fieldNum, true)
		// others might be container case
		default:
			field = t.directory(scope, child, fieldNum, false)
		}
		msg.AddField(field)
	}

	scope.reflectTo(msg)

	return msg
}

func (t *transformer) leaf(scope *scope, typ *yang.YangType, name string) pbast.Type {
	if isBuiltinType(typ) {
		return t.convertType(typ, CamelCase(name))
	}
	switch typ.Kind {
	case yang.Yenum, yang.Ybits, yang.Yunion:
		return t.convertType(typ, CamelCase(typ.Name))
	default:
		inner := t.convertType(typ, "Value")
		msg := pbast.NewMessage(CamelCase(typ.Name)).
			AddField(pbast.NewMessageField(inner, "value", 1))
		if !t.useType(inner) {
			msg.AddType(inner)
		}
		return msg
	}
}

func (t *transformer) useType(typ pbast.Type) bool {
	switch typ {
	case decimal64:
		t.decimal64 = decimal64
	case leafRef:
		t.leafRef = leafRef
	case identityRef:
		t.identityRef = identityRef
	case instanceIdentifier:
		t.instanceIdentifier = instanceIdentifier
	case pbast.Empty:
		t.emptyNeeded = true
	default:
		return false
	}

	return true
}

func isBuiltinType(typ *yang.YangType) bool {
	if _, ok := yang.TypeKindFromName[typ.Name]; !ok {
		return false
	}
	return true
}

func (t *transformer) convertType(typ *yang.YangType, name string) pbast.Type {
	switch typ.Kind {
	case yang.Yint8, yang.Yint16, yang.Yint32:
		return pbast.Int32
	case yang.Yint64:
		return pbast.Int64
	case yang.Yuint8, yang.Yuint16, yang.Yuint32:
		return pbast.UInt32
	case yang.Yuint64:
		return pbast.UInt64
	case yang.Ystring:
		return pbast.String
	case yang.Ybool:
		return pbast.Bool
	case yang.Yenum:
		return t.customEnum(name, typ.Enum)
	case yang.Ybits:
		return t.customBits(name, typ.Bit)
	case yang.Ybinary:
		return pbast.Bytes
	case yang.Yleafref:
		return leafRef
	case yang.Yidentityref:
		return identityRef
	case yang.Yempty:
		return pbast.Empty
	case yang.Yunion:
		return t.customUnion(name, typ.Type)
	case yang.YinstanceIdentifier:
		return instanceIdentifier
	case yang.Ydecimal64:
		return decimal64
	default:
		return nil
	}
}

func (t *transformer) customBits(name string, bits *yang.EnumType) *pbast.Message {
	msg := pbast.NewMessage(name)
	for i, n := range bits.Names() {
		v := 1 << uint(bits.Values()[i])
		msg.AddField(pbast.NewMessageField(pbast.Bool, n, v))
	}

	return msg
}

func (t *transformer) customEnum(name string, e *yang.EnumType) *pbast.Message {
	enum := pbast.NewEnum("Value")
	for _, v := range e.Values() {
		n := e.ValueMap()[v]
		enum.AddField(pbast.NewEnumField(constantName(n), int(v)))
	}
	msg := pbast.NewMessage(name).
		AddEnum(enum).
		AddField(pbast.NewMessageField(enum, "value", 1))

	return msg
}

func (t *transformer) customUnion(name string, types []*yang.YangType) *pbast.Message {
	scope := newScope()
	pbTypes := t.unionFields(types, nil, scope)

	oneof := pbast.NewOneOf("value")
	for i, typ := range pbTypes {
		oneof.AddField(pbast.NewOneOfField(typ, underscoreCase(typ.TypeName()), i+1))
	}

	msg := pbast.NewMessage(name).AddOneOf(oneof)
	scope.reflectTo(msg)

	return msg
}

func (t *transformer) unionFields(types []*yang.YangType, pbTypes []pbast.Type, scope *scope) []pbast.Type {
	for _, typ := range types {
		pbtype := t.convertType(typ, typ.Name)
		if pbtype == nil {
			continue
		}

		if typ.Kind == yang.Yunion {
			pbTypes = t.unionFields(typ.Type, pbTypes, scope)
			continue
		}

		pbTypes = append(pbTypes, pbtype)
		scope.addType(pbtype)
	}

	return pbTypes
}

func (t *transformer) directory(scope *scope, e entry, index int, repeated bool) *pbast.MessageField {
	fieldName := underscoreCase(e.Name)
	typeName := CamelCase(e.Name)

	inner := t.buildMessage(typeName, e)
	field := &pbast.MessageField{
		Repeated: repeated,
		Type:     inner.TypeName(),
		Name:     fieldName,
		Index:    index,
		Comment:  t.genericComments(e),
	}

	if err := scope.addType(inner); err != nil {
		fmt.Fprintln(os.Stderr, err)
	}

	return field
}
<|endoftext|>


<commit_before><commit_msg>resolved review comments<commit_after><|endoftext|>


<commit_before>package terraform

import (
	"fmt"
	"strings"

	"github.com/hashicorp/terraform/config"
	"github.com/hashicorp/terraform/dag"
)

// NodeApplyableOutput represents an output that is "applyable":
// it is ready to be applied.
type NodeApplyableOutput struct {
	PathValue []string
	Config    *config.Output // Config is the output in the config
}

func (n *NodeApplyableOutput) Name() string {
	result := fmt.Sprintf("output.%s", n.Config.Name)
	if len(n.PathValue) > 1 {
		result = fmt.Sprintf("%s.%s", modulePrefixStr(n.PathValue), result)
	}

	return result
}

// GraphNodeSubPath
func (n *NodeApplyableOutput) Path() []string {
	return n.PathValue
}

// RemovableIfNotTargeted
func (n *NodeApplyableOutput) RemoveIfNotTargeted() bool {
	// We need to add this so that this node will be removed if
	// it isn't targeted or a dependency of a target.
	return true
}

// GraphNodeTargetDownstream
func (n *NodeApplyableOutput) TargetDownstream(targetedDeps, untargetedDeps *dag.Set) bool {
	// If any of the direct dependencies of an output are targeted then
	// the output must always be targeted as well, so its value will always
	// be up-to-date at the completion of an apply walk.
	return true
}

// GraphNodeReferenceable
func (n *NodeApplyableOutput) ReferenceableName() []string {
	name := fmt.Sprintf("output.%s", n.Config.Name)
	return []string{name}
}

// GraphNodeReferencer
func (n *NodeApplyableOutput) References() []string {
	var result []string
	result = append(result, n.Config.DependsOn...)
	result = append(result, ReferencesFromConfig(n.Config.RawConfig)...)
	for _, v := range result {
		split := strings.Split(v, "/")
		for i, s := range split {
			split[i] = s + ".destroy"
		}

		result = append(result, strings.Join(split, "/"))
	}

	return result
}

// GraphNodeEvalable
func (n *NodeApplyableOutput) EvalTree() EvalNode {
	return &EvalOpFilter{
		Ops: []walkOperation{walkRefresh, walkPlan, walkApply,
			walkDestroy, walkInput, walkValidate},
		Node: &EvalSequence{
			Nodes: []EvalNode{
				&EvalWriteOutput{
					Name:      n.Config.Name,
					Sensitive: n.Config.Sensitive,
					Value:     n.Config.RawConfig,
				},
			},
		},
	}
}
<commit_msg>"destroy" outputs too<commit_after>package terraform

import (
	"fmt"
	"strings"

	"github.com/hashicorp/terraform/config"
	"github.com/hashicorp/terraform/dag"
)

// NodeApplyableOutput represents an output that is "applyable":
// it is ready to be applied.
type NodeApplyableOutput struct {
	PathValue []string
	Config    *config.Output // Config is the output in the config
}

func (n *NodeApplyableOutput) Name() string {
	result := fmt.Sprintf("output.%s", n.Config.Name)
	if len(n.PathValue) > 1 {
		result = fmt.Sprintf("%s.%s", modulePrefixStr(n.PathValue), result)
	}

	return result
}

// GraphNodeSubPath
func (n *NodeApplyableOutput) Path() []string {
	return n.PathValue
}

// RemovableIfNotTargeted
func (n *NodeApplyableOutput) RemoveIfNotTargeted() bool {
	// We need to add this so that this node will be removed if
	// it isn't targeted or a dependency of a target.
	return true
}

// GraphNodeTargetDownstream
func (n *NodeApplyableOutput) TargetDownstream(targetedDeps, untargetedDeps *dag.Set) bool {
	// If any of the direct dependencies of an output are targeted then
	// the output must always be targeted as well, so its value will always
	// be up-to-date at the completion of an apply walk.
	return true
}

// GraphNodeReferenceable
func (n *NodeApplyableOutput) ReferenceableName() []string {
	name := fmt.Sprintf("output.%s", n.Config.Name)
	return []string{name}
}

// GraphNodeReferencer
func (n *NodeApplyableOutput) References() []string {
	var result []string
	result = append(result, n.Config.DependsOn...)
	result = append(result, ReferencesFromConfig(n.Config.RawConfig)...)
	for _, v := range result {
		split := strings.Split(v, "/")
		for i, s := range split {
			split[i] = s + ".destroy"
		}

		result = append(result, strings.Join(split, "/"))
	}

	return result
}

// GraphNodeEvalable
func (n *NodeApplyableOutput) EvalTree() EvalNode {
	return &EvalSequence{
		Nodes: []EvalNode{
			&EvalOpFilter{
				Ops: []walkOperation{walkRefresh, walkPlan, walkApply,
					walkInput, walkValidate},
				Node: &EvalWriteOutput{
					Name:      n.Config.Name,
					Sensitive: n.Config.Sensitive,
					Value:     n.Config.RawConfig,
				},
			},
			&EvalOpFilter{
				Ops: []walkOperation{walkDestroy, walkPlanDestroy},
				Node: &EvalDeleteOutput{
					Name: n.Config.Name,
				},
			},
		},
	}
}
<|endoftext|>


<commit_before>// Copyright (C) 2015  The GoHBase Authors.  All rights reserved.
// This file is part of GoHBase.
// Use of this source code is governed by the Apache License 2.0
// that can be found in the COPYING file.

package region

import (
	"encoding/binary"
	"errors"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"net"
	"sync"
	"time"

	"github.com/golang/protobuf/proto"
	"github.com/tsuna/gohbase/hrpc"
	"github.com/tsuna/gohbase/pb"
)

var (
	// ErrShortWrite is used when the writer thread only succeeds in writing
	// part of its buffer to the socket, and not all of the buffer was sent
	ErrShortWrite = errors.New("short write occurred while writing to socket")

	// ErrMissingCallID is used when HBase sends us a response message for a
	// request that we didn't send
	ErrMissingCallID = errors.New("HBase responded to a nonsensical call ID")

	// javaRetryableExceptions is a map where all Java exceptions that signify
	// the RPC should be sent again are listed (as keys). If a Java exception
	// listed here is returned by HBase, the client should attempt to resend
	// the RPC message, potentially via a different region client.
	javaRetryableExceptions = map[string]struct{}{
		"org.apache.hadoop.hbase.NotServingRegionException":         struct{}{},
		"org.apache.hadoop.hbase.exceptions.RegionMovedException":   struct{}{},
		"org.apache.hadoop.hbase.exceptions.RegionOpeningException": struct{}{},
	}
)

// UnrecoverableError is an error that this region.Client can't recover from.
// The connection to the RegionServer has to be closed and all queued and
// outstanding RPCs will be failed / retried.
type UnrecoverableError struct {
	error
}

func (e UnrecoverableError) Error() string {
	return error(e).Error()
}

// RetryableError is an error that indicates the RPC should be retried because
// the error is transient (e.g. a region being momentarily unavailable).
type RetryableError struct {
	error
}

func (e RetryableError) Error() string {
	return error(e).Error()
}

// Client manages a connection to a RegionServer.
type Client struct {
	id uint32

	conn net.Conn

	// Hostname or IP address of the RegionServer.
	host string

	// Port of the RegionServer.
	port uint16

	// writeMutex is used to prevent multiple threads from writing to the
	// socket at the same time.
	writeMutex *sync.Mutex

	// sendErr is set once a write fails.
	sendErr error

	rpcs []hrpc.Call

	// Once the rpcs list has grown to a large enough size, this channel is
	// written to to notify the writer thread that it should stop sleeping and
	// process the list
	process chan struct{}

	// sentRPCs contains the mapping of sent call IDs to RPC calls, so that when
	// a response is received it can be tied to the correct RPC
	sentRPCs      map[uint32]hrpc.Call
	sentRPCsMutex *sync.Mutex

	rpcQueueSize  int
	flushInterval time.Duration
}

// NewClient creates a new RegionClient.
func NewClient(host string, port uint16, queueSize int, flushInterval time.Duration) (*Client, error) {
	addr := fmt.Sprintf("%s:%d", host, port)
	conn, err := net.Dial("tcp", addr)
	if err != nil {
		return nil,
			fmt.Errorf("failed to connect to the RegionServer at %s: %s", addr, err)
	}
	c := &Client{
		conn:          conn,
		host:          host,
		port:          port,
		writeMutex:    &sync.Mutex{},
		process:       make(chan struct{}),
		sentRPCsMutex: &sync.Mutex{},
		sentRPCs:      make(map[uint32]hrpc.Call),
		rpcQueueSize:  queueSize,
		flushInterval: flushInterval,
	}
	err = c.sendHello()
	if err != nil {
		return nil, err
	}
	go c.processRpcs() // Writer goroutine
	go c.receiveRpcs() // Reader goroutine
	return c, nil
}

func (c *Client) processRpcs() {
	for {
		if c.sendErr != nil {
			return
		}

		select {
		case <-time.After(c.flushInterval):
			c.writeMutex.Lock()
		case <-c.process:
			// We don't acquire the lock here, because the thread that sent
			// something on the process channel will have locked the mutex,
			// and will not release it so as to transfer ownership
		}

		rpcs := make([]hrpc.Call, len(c.rpcs))
		for i, rpc := range c.rpcs {
			rpcs[i] = rpc
		}
		c.rpcs = nil
		c.writeMutex.Unlock()

		for i, rpc := range rpcs {
			// If the deadline has been exceeded, don't bother sending the
			// request. The function that placed the RPC in our queue should
			// stop waiting for a result and return an error.
			select {
			case _, ok := <-rpc.GetContext().Done():
				if !ok {
					continue
				}
			default:
			}

			err := c.sendRPC(rpc)
			if err != nil {
				_, ok := err.(UnrecoverableError)
				if ok {
					c.sendErr = err

					c.writeMutex.Lock()
					c.rpcs = append(c.rpcs, rpcs[i:]...)
					c.writeMutex.Unlock()

					c.errorEncountered()
					return
				}
				rpc.GetResultChan() <- hrpc.RPCResult{nil, err}
			}
		}
	}
}

func (c *Client) receiveRpcs() {
	var sz [4]byte
	for {
		err := c.readFully(sz[:])
		if err != nil {
			c.sendErr = err
			c.errorEncountered()
			return
		}

		buf := make([]byte, binary.BigEndian.Uint32(sz[:]))
		err = c.readFully(buf)
		if err != nil {
			c.sendErr = err
			c.errorEncountered()
			return
		}

		resp := &pb.ResponseHeader{}
		respLen, nb := proto.DecodeVarint(buf)
		buf = buf[nb:]
		err = proto.UnmarshalMerge(buf[:respLen], resp)
		buf = buf[respLen:]
		if err != nil {
			// Failed to deserialize the response header
			c.sendErr = err
			c.errorEncountered()
			return
		}
		if resp.CallId == nil {
			// Response doesn't have a call ID
			log.Error("Response doesn't have a call ID!")
			c.sendErr = ErrMissingCallID
			c.errorEncountered()
			return
		}

		c.sentRPCsMutex.Lock()
		rpc, ok := c.sentRPCs[*resp.CallId]
		c.sentRPCsMutex.Unlock()

		if !ok {
			log.WithFields(log.Fields{
				"CallId": *resp.CallId,
			}).Error("Received a response with an unexpected call ID")

			log.Warn("Waiting for responses to the following calls: ")
			c.sentRPCsMutex.Lock()
			for k := range c.sentRPCs {
				log.Error("\t\t%d, ", k)
			}
			c.sentRPCsMutex.Unlock()

			c.sendErr = fmt.Errorf("HBase sent a response with an unexpected call ID: %d", resp.CallId)
			c.errorEncountered()
			return
		}

		var rpcResp proto.Message
		if resp.Exception == nil {
			respLen, nb = proto.DecodeVarint(buf)
			buf = buf[nb:]
			rpcResp = rpc.NewResponse()
			err = proto.UnmarshalMerge(buf, rpcResp)
			buf = buf[respLen:]
		} else {
			javaClass := *resp.Exception.ExceptionClassName
			err = fmt.Errorf("HBase Java exception %s: \n%s", javaClass,
				*resp.Exception.StackTrace)
			if _, ok := javaRetryableExceptions[javaClass]; ok {
				// This is a recoverable error. The client should retry.
				err = RetryableError{err}
			}
		}
		rpc.GetResultChan() <- hrpc.RPCResult{rpcResp, err}

		c.sentRPCsMutex.Lock()
		delete(c.sentRPCs, *resp.CallId)
		c.sentRPCsMutex.Unlock()
	}
}

func (c *Client) errorEncountered() {
	c.writeMutex.Lock()
	res := hrpc.RPCResult{nil, UnrecoverableError{c.sendErr}}
	for _, rpc := range c.rpcs {
		rpc.GetResultChan() <- res
	}
	c.rpcs = nil
	c.writeMutex.Unlock()

	c.sentRPCsMutex.Lock()
	for _, rpc := range c.sentRPCs {
		rpc.GetResultChan() <- res
	}
	c.sentRPCs = nil
	c.sentRPCsMutex.Unlock()

	c.conn.Close()
}

// Sends the given buffer to the RegionServer.
func (c *Client) write(buf []byte) error {
	n, err := c.conn.Write(buf)

	if err != nil {
		// There was an error while writing
		return err
	}
	if n != len(buf) {
		// We failed to write the entire buffer
		// TODO: Perhaps handle this in another way than closing down
		return ErrShortWrite
	}
	return nil
}

// Tries to read enough data to fully fill up the given buffer.
func (c *Client) readFully(buf []byte) error {
	// TODO: Handle short reads.
	n, err := c.conn.Read(buf)
	if err != nil {
		return fmt.Errorf("Failed to read from the RS: %s", err)
	} else if n != len(buf) {
		return fmt.Errorf("Failed to read everything from the RS: %s", err)
	}
	return nil
}

// Sends the "hello" message needed when opening a new connection.
func (c *Client) sendHello() error {
	connHeader := &pb.ConnectionHeader{
		UserInfo: &pb.UserInformation{
			EffectiveUser: proto.String("gopher"),
		},
		ServiceName: proto.String("ClientService"),
		//CellBlockCodecClass: "org.apache.hadoop.hbase.codec.KeyValueCodec",
	}
	data, err := proto.Marshal(connHeader)
	if err != nil {
		return fmt.Errorf("failed to marshal connection header: %s", err)
	}

	const header = "HBas\x00\x50" // \x50 = Simple Auth.
	buf := make([]byte, 0, len(header)+4+len(data))
	buf = append(buf, header...)
	buf = buf[:len(header)+4]
	binary.BigEndian.PutUint32(buf[6:], uint32(len(data)))
	buf = append(buf, data...)

	return c.write(buf)
}

// QueueRPC will add an rpc call to the queue for processing by the writer
// goroutine
func (c *Client) QueueRPC(rpc hrpc.Call) error {
	if c.sendErr != nil {
		return c.sendErr
	}
	c.writeMutex.Lock()
	c.rpcs = append(c.rpcs, rpc)
	if len(c.rpcs) > c.rpcQueueSize {
		c.process <- struct{}{}
		// We don't release the lock here, because we want to transfer ownership
		// of the lock to the goroutine that processes the RPCs
	} else {
		c.writeMutex.Unlock()
	}
	return nil
}

// sendRPC sends an RPC out to the wire.
// Returns the response (for now, as the call is synchronous).
func (c *Client) sendRPC(rpc hrpc.Call) error {
	// Header.
	c.id++
	reqheader := &pb.RequestHeader{
		CallId:       &c.id,
		MethodName:   proto.String(rpc.GetName()),
		RequestParam: proto.Bool(true),
	}

	payload, err := rpc.Serialize()
	if err != nil {
		return fmt.Errorf("Failed to serialize RPC: %s", err)
	}
	payloadLen := proto.EncodeVarint(uint64(len(payload)))

	headerData, err := proto.Marshal(reqheader)
	if err != nil {
		return fmt.Errorf("Failed to marshal Get request: %s", err)
	}

	buf := make([]byte, 5, 4+1+len(headerData)+len(payloadLen)+len(payload))
	binary.BigEndian.PutUint32(buf, uint32(cap(buf)-4))
	buf[4] = byte(len(headerData))
	buf = append(buf, headerData...)
	buf = append(buf, payloadLen...)
	buf = append(buf, payload...)

	c.sentRPCsMutex.Lock()
	c.sentRPCs[c.id] = rpc
	c.sentRPCsMutex.Unlock()

	err = c.write(buf)
	if err != nil {
		return UnrecoverableError{err}
	}

	return nil
}
<commit_msg>region/client: Fix deadlock in the RPC queue.<commit_after>// Copyright (C) 2015  The GoHBase Authors.  All rights reserved.
// This file is part of GoHBase.
// Use of this source code is governed by the Apache License 2.0
// that can be found in the COPYING file.

package region

import (
	"encoding/binary"
	"errors"
	"fmt"
	"net"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"

	"github.com/golang/protobuf/proto"
	"github.com/tsuna/gohbase/hrpc"
	"github.com/tsuna/gohbase/pb"
)

var (
	// ErrShortWrite is used when the writer thread only succeeds in writing
	// part of its buffer to the socket, and not all of the buffer was sent
	ErrShortWrite = errors.New("short write occurred while writing to socket")

	// ErrMissingCallID is used when HBase sends us a response message for a
	// request that we didn't send
	ErrMissingCallID = errors.New("HBase responded to a nonsensical call ID")

	// javaRetryableExceptions is a map where all Java exceptions that signify
	// the RPC should be sent again are listed (as keys). If a Java exception
	// listed here is returned by HBase, the client should attempt to resend
	// the RPC message, potentially via a different region client.
	javaRetryableExceptions = map[string]struct{}{
		"org.apache.hadoop.hbase.NotServingRegionException":         struct{}{},
		"org.apache.hadoop.hbase.exceptions.RegionMovedException":   struct{}{},
		"org.apache.hadoop.hbase.exceptions.RegionOpeningException": struct{}{},
	}
)

// UnrecoverableError is an error that this region.Client can't recover from.
// The connection to the RegionServer has to be closed and all queued and
// outstanding RPCs will be failed / retried.
type UnrecoverableError struct {
	error
}

func (e UnrecoverableError) Error() string {
	return error(e).Error()
}

// RetryableError is an error that indicates the RPC should be retried because
// the error is transient (e.g. a region being momentarily unavailable).
type RetryableError struct {
	error
}

func (e RetryableError) Error() string {
	return error(e).Error()
}

// Client manages a connection to a RegionServer.
type Client struct {
	id uint32

	conn net.Conn

	// Hostname or IP address of the RegionServer.
	host string

	// Port of the RegionServer.
	port uint16

	// writeMutex is used to prevent multiple threads from writing to the
	// socket at the same time.
	writeMutex *sync.Mutex

	// sendErr is set once a write fails.
	sendErr error

	rpcs []hrpc.Call

	// Once the rpcs list has grown to a large enough size, this channel is
	// written to to notify the writer thread that it should stop sleeping and
	// process the list
	process chan struct{}

	// sentRPCs contains the mapping of sent call IDs to RPC calls, so that when
	// a response is received it can be tied to the correct RPC
	sentRPCs      map[uint32]hrpc.Call
	sentRPCsMutex *sync.Mutex

	rpcQueueSize  int
	flushInterval time.Duration
}

// NewClient creates a new RegionClient.
func NewClient(host string, port uint16, queueSize int, flushInterval time.Duration) (*Client, error) {
	addr := fmt.Sprintf("%s:%d", host, port)
	conn, err := net.Dial("tcp", addr)
	if err != nil {
		return nil,
			fmt.Errorf("failed to connect to the RegionServer at %s: %s", addr, err)
	}
	c := &Client{
		conn:          conn,
		host:          host,
		port:          port,
		writeMutex:    &sync.Mutex{},
		process:       make(chan struct{}),
		sentRPCsMutex: &sync.Mutex{},
		sentRPCs:      make(map[uint32]hrpc.Call),
		rpcQueueSize:  queueSize,
		flushInterval: flushInterval,
	}
	err = c.sendHello()
	if err != nil {
		return nil, err
	}
	go c.processRpcs() // Writer goroutine
	go c.receiveRpcs() // Reader goroutine
	return c, nil
}

func (c *Client) processRpcs() {
	for {
		if c.sendErr != nil {
			return
		}

		select {
		case <-time.After(c.flushInterval):
			select {
			case <-c.process:
			// If we got a message on c.process at the same time as our
			// timeout elapsed, we'll non-deterministically land in either
			// cases of this outer select.  Here we double-check whether
			// something was written onto c.process, in which case we don't
			// grab the lock (see comment below in the other case).
			default:
				c.writeMutex.Lock()
			}
		case <-c.process:
			// We don't acquire the lock here, because the thread that sent
			// something on the process channel will have locked the mutex,
			// and will not release it so as to transfer ownership
		}

		rpcs := make([]hrpc.Call, len(c.rpcs))
		for i, rpc := range c.rpcs {
			rpcs[i] = rpc
		}
		c.rpcs = nil
		c.writeMutex.Unlock()

		for i, rpc := range rpcs {
			// If the deadline has been exceeded, don't bother sending the
			// request. The function that placed the RPC in our queue should
			// stop waiting for a result and return an error.
			select {
			case _, ok := <-rpc.GetContext().Done():
				if !ok {
					continue
				}
			default:
			}

			err := c.sendRPC(rpc)
			if err != nil {
				_, ok := err.(UnrecoverableError)
				if ok {
					c.sendErr = err

					c.writeMutex.Lock()
					c.rpcs = append(c.rpcs, rpcs[i:]...)
					c.writeMutex.Unlock()

					c.errorEncountered()
					return
				}
				rpc.GetResultChan() <- hrpc.RPCResult{nil, err}
			}
		}
	}
}

func (c *Client) receiveRpcs() {
	var sz [4]byte
	for {
		err := c.readFully(sz[:])
		if err != nil {
			c.sendErr = err
			c.errorEncountered()
			return
		}

		buf := make([]byte, binary.BigEndian.Uint32(sz[:]))
		err = c.readFully(buf)
		if err != nil {
			c.sendErr = err
			c.errorEncountered()
			return
		}

		resp := &pb.ResponseHeader{}
		respLen, nb := proto.DecodeVarint(buf)
		buf = buf[nb:]
		err = proto.UnmarshalMerge(buf[:respLen], resp)
		buf = buf[respLen:]
		if err != nil {
			// Failed to deserialize the response header
			c.sendErr = err
			c.errorEncountered()
			return
		}
		if resp.CallId == nil {
			// Response doesn't have a call ID
			log.Error("Response doesn't have a call ID!")
			c.sendErr = ErrMissingCallID
			c.errorEncountered()
			return
		}

		c.sentRPCsMutex.Lock()
		rpc, ok := c.sentRPCs[*resp.CallId]
		c.sentRPCsMutex.Unlock()

		if !ok {
			log.WithFields(log.Fields{
				"CallId": *resp.CallId,
			}).Error("Received a response with an unexpected call ID")

			log.Warn("Waiting for responses to the following calls: ")
			c.sentRPCsMutex.Lock()
			for k := range c.sentRPCs {
				log.Error("\t\t%d, ", k)
			}
			c.sentRPCsMutex.Unlock()

			c.sendErr = fmt.Errorf("HBase sent a response with an unexpected call ID: %d", resp.CallId)
			c.errorEncountered()
			return
		}

		var rpcResp proto.Message
		if resp.Exception == nil {
			respLen, nb = proto.DecodeVarint(buf)
			buf = buf[nb:]
			rpcResp = rpc.NewResponse()
			err = proto.UnmarshalMerge(buf, rpcResp)
			buf = buf[respLen:]
		} else {
			javaClass := *resp.Exception.ExceptionClassName
			err = fmt.Errorf("HBase Java exception %s: \n%s", javaClass,
				*resp.Exception.StackTrace)
			if _, ok := javaRetryableExceptions[javaClass]; ok {
				// This is a recoverable error. The client should retry.
				err = RetryableError{err}
			}
		}
		rpc.GetResultChan() <- hrpc.RPCResult{rpcResp, err}

		c.sentRPCsMutex.Lock()
		delete(c.sentRPCs, *resp.CallId)
		c.sentRPCsMutex.Unlock()
	}
}

func (c *Client) errorEncountered() {
	c.writeMutex.Lock()
	res := hrpc.RPCResult{nil, UnrecoverableError{c.sendErr}}
	for _, rpc := range c.rpcs {
		rpc.GetResultChan() <- res
	}
	c.rpcs = nil
	c.writeMutex.Unlock()

	c.sentRPCsMutex.Lock()
	for _, rpc := range c.sentRPCs {
		rpc.GetResultChan() <- res
	}
	c.sentRPCs = nil
	c.sentRPCsMutex.Unlock()

	c.conn.Close()
}

// Sends the given buffer to the RegionServer.
func (c *Client) write(buf []byte) error {
	n, err := c.conn.Write(buf)

	if err != nil {
		// There was an error while writing
		return err
	}
	if n != len(buf) {
		// We failed to write the entire buffer
		// TODO: Perhaps handle this in another way than closing down
		return ErrShortWrite
	}
	return nil
}

// Tries to read enough data to fully fill up the given buffer.
func (c *Client) readFully(buf []byte) error {
	// TODO: Handle short reads.
	n, err := c.conn.Read(buf)
	if err != nil {
		return fmt.Errorf("Failed to read from the RS: %s", err)
	} else if n != len(buf) {
		return fmt.Errorf("Failed to read everything from the RS: %s", err)
	}
	return nil
}

// Sends the "hello" message needed when opening a new connection.
func (c *Client) sendHello() error {
	connHeader := &pb.ConnectionHeader{
		UserInfo: &pb.UserInformation{
			EffectiveUser: proto.String("gopher"),
		},
		ServiceName: proto.String("ClientService"),
		//CellBlockCodecClass: "org.apache.hadoop.hbase.codec.KeyValueCodec",
	}
	data, err := proto.Marshal(connHeader)
	if err != nil {
		return fmt.Errorf("failed to marshal connection header: %s", err)
	}

	const header = "HBas\x00\x50" // \x50 = Simple Auth.
	buf := make([]byte, 0, len(header)+4+len(data))
	buf = append(buf, header...)
	buf = buf[:len(header)+4]
	binary.BigEndian.PutUint32(buf[6:], uint32(len(data)))
	buf = append(buf, data...)

	return c.write(buf)
}

// QueueRPC will add an rpc call to the queue for processing by the writer
// goroutine
func (c *Client) QueueRPC(rpc hrpc.Call) error {
	if c.sendErr != nil {
		return c.sendErr
	}
	c.writeMutex.Lock()
	c.rpcs = append(c.rpcs, rpc)
	if len(c.rpcs) > c.rpcQueueSize {
		c.process <- struct{}{}
		// We don't release the lock here, because we want to transfer ownership
		// of the lock to the goroutine that processes the RPCs
	} else {
		c.writeMutex.Unlock()
	}
	return nil
}

// sendRPC sends an RPC out to the wire.
// Returns the response (for now, as the call is synchronous).
func (c *Client) sendRPC(rpc hrpc.Call) error {
	// Header.
	c.id++
	reqheader := &pb.RequestHeader{
		CallId:       &c.id,
		MethodName:   proto.String(rpc.GetName()),
		RequestParam: proto.Bool(true),
	}

	payload, err := rpc.Serialize()
	if err != nil {
		return fmt.Errorf("Failed to serialize RPC: %s", err)
	}
	payloadLen := proto.EncodeVarint(uint64(len(payload)))

	headerData, err := proto.Marshal(reqheader)
	if err != nil {
		return fmt.Errorf("Failed to marshal Get request: %s", err)
	}

	buf := make([]byte, 5, 4+1+len(headerData)+len(payloadLen)+len(payload))
	binary.BigEndian.PutUint32(buf, uint32(cap(buf)-4))
	buf[4] = byte(len(headerData))
	buf = append(buf, headerData...)
	buf = append(buf, payloadLen...)
	buf = append(buf, payload...)

	c.sentRPCsMutex.Lock()
	c.sentRPCs[c.id] = rpc
	c.sentRPCsMutex.Unlock()

	err = c.write(buf)
	if err != nil {
		return UnrecoverableError{err}
	}

	return nil
}
<|endoftext|>


<commit_before>package metrics_pipeline_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"testing"
)

func TestMetricsBuffer(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "MetricsBuffer Suite")
}
<commit_msg>metrics_pipeline: correct suite name<commit_after>package metrics_pipeline_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"testing"
)

func TestMetricsBuffer(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "MetricsPipeline Suite")
}
<|endoftext|>


<commit_before>package portsbinding

import (
	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
)

// Get retrieves a specific port based on its unique ID.
func Get(c *gophercloud.ServiceClient, id string) (r GetResult) {
	_, r.Err = c.Get(getURL(c, id), &r.Body, nil)
	return
}

// CreateOpts represents the attributes used when creating a new
// port with extended attributes.
type CreateOpts struct {
	// CreateOptsBuilder is the interface options structs have to satisfy in order
	// to be used in the main Create operation in this package.
	ports.CreateOptsBuilder `json:"-"`
	// The ID of the host where the port is allocated
	HostID string `json:"binding:host_id,omitempty"`
	// The virtual network interface card (vNIC) type that is bound to the
	// neutron port
	VNICType string `json:"binding:vnic_type,omitempty"`
	// A dictionary that enables the application running on the specified
	// host to pass and receive virtual network interface (VIF) port-specific
	// information to the plug-in
	Profile map[string]string `json:"binding:profile,omitempty"`
}

// ToPortCreateMap casts a CreateOpts struct to a map.
func (opts CreateOpts) ToPortCreateMap() (map[string]interface{}, error) {
	b1, err := opts.CreateOptsBuilder.ToPortCreateMap()
	if err != nil {
		return nil, err
	}

	b2, err := gophercloud.BuildRequestBody(opts, "")
	if err != nil {
		return nil, err
	}

	port := b1["port"].(map[string]interface{})

	for k, v := range b2 {
		port[k] = v
	}

	return map[string]interface{}{"port": port}, nil
}

// Create accepts a CreateOpts struct and creates a new port with extended attributes.
// You must remember to provide a NetworkID value.
func Create(c *gophercloud.ServiceClient, opts ports.CreateOptsBuilder) (r CreateResult) {
	b, err := opts.ToPortCreateMap()
	if err != nil {
		r.Err = err
		return
	}
	_, r.Err = c.Post(createURL(c), b, &r.Body, nil)
	return
}

// UpdateOpts represents the attributes used when updating an existing port.
type UpdateOpts struct {
	// UpdateOptsBuilder is the interface options structs have to satisfy in order
	// to be used in the main Update operation in this package.
	ports.UpdateOptsBuilder `json:"-"`
	// The ID of the host where the port is allocated
	HostID string `json:"binding:host_id,omitempty"`
	// The virtual network interface card (vNIC) type that is bound to the
	// neutron port
	VNICType string `json:"binding:vnic_type,omitempty"`
	// A dictionary that enables the application running on the specified
	// host to pass and receive virtual network interface (VIF) port-specific
	// information to the plug-in
	Profile map[string]string `json:"binding:profile,omitempty"`
}

// ToPortUpdateMap casts an UpdateOpts struct to a map.
func (opts UpdateOpts) ToPortUpdateMap() (map[string]interface{}, error) {
	b1, err := opts.UpdateOptsBuilder.ToPortUpdateMap()
	if err != nil {
		return nil, err
	}

	b2, err := gophercloud.BuildRequestBody(opts, "")
	if err != nil {
		return nil, err
	}

	port := b1["port"].(map[string]interface{})

	for k, v := range b2 {
		port[k] = v
	}

	return map[string]interface{}{"port": port}, nil
}

// Update accepts a UpdateOpts struct and updates an existing port using the
// values provided.
func Update(c *gophercloud.ServiceClient, id string, opts ports.UpdateOptsBuilder) (r UpdateResult) {
	b, err := opts.ToPortUpdateMap()
	if err != nil {
		r.Err = err
		return r
	}
	_, r.Err = c.Put(updateURL(c, id), b, &r.Body, &gophercloud.RequestOpts{
		OkCodes: []int{200, 201},
	})
	return
}
<commit_msg>Add dns_name to portsbinding<commit_after>package portsbinding

import (
	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
)

// Get retrieves a specific port based on its unique ID.
func Get(c *gophercloud.ServiceClient, id string) (r GetResult) {
	_, r.Err = c.Get(getURL(c, id), &r.Body, nil)
	return
}

// CreateOpts represents the attributes used when creating a new
// port with extended attributes.
type CreateOpts struct {
	// CreateOptsBuilder is the interface options structs have to satisfy in order
	// to be used in the main Create operation in this package.
	ports.CreateOptsBuilder `json:"-"`
	// The ID of the host where the port is allocated
	HostID string `json:"binding:host_id,omitempty"`
	// The dns name
	DNSName string `json:"dns_name,omitempty"`
	// The virtual network interface card (vNIC) type that is bound to the
	// neutron port
	VNICType string `json:"binding:vnic_type,omitempty"`
	// A dictionary that enables the application running on the specified
	// host to pass and receive virtual network interface (VIF) port-specific
	// information to the plug-in
	Profile map[string]string `json:"binding:profile,omitempty"`
}

// ToPortCreateMap casts a CreateOpts struct to a map.
func (opts CreateOpts) ToPortCreateMap() (map[string]interface{}, error) {
	b1, err := opts.CreateOptsBuilder.ToPortCreateMap()
	if err != nil {
		return nil, err
	}

	b2, err := gophercloud.BuildRequestBody(opts, "")
	if err != nil {
		return nil, err
	}

	port := b1["port"].(map[string]interface{})

	for k, v := range b2 {
		port[k] = v
	}

	return map[string]interface{}{"port": port}, nil
}

// Create accepts a CreateOpts struct and creates a new port with extended attributes.
// You must remember to provide a NetworkID value.
func Create(c *gophercloud.ServiceClient, opts ports.CreateOptsBuilder) (r CreateResult) {
	b, err := opts.ToPortCreateMap()
	if err != nil {
		r.Err = err
		return
	}
	_, r.Err = c.Post(createURL(c), b, &r.Body, nil)
	return
}

// UpdateOpts represents the attributes used when updating an existing port.
type UpdateOpts struct {
	// UpdateOptsBuilder is the interface options structs have to satisfy in order
	// to be used in the main Update operation in this package.
	ports.UpdateOptsBuilder `json:"-"`
	// The ID of the host where the port is allocated
	HostID string `json:"binding:host_id,omitempty"`
	// The dns name
	DNSName string `json:"dns_name,omitempty"`
	// The virtual network interface card (vNIC) type that is bound to the
	// neutron port
	VNICType string `json:"binding:vnic_type,omitempty"`
	// A dictionary that enables the application running on the specified
	// host to pass and receive virtual network interface (VIF) port-specific
	// information to the plug-in
	Profile map[string]string `json:"binding:profile,omitempty"`
}

// ToPortUpdateMap casts an UpdateOpts struct to a map.
func (opts UpdateOpts) ToPortUpdateMap() (map[string]interface{}, error) {
	b1, err := opts.UpdateOptsBuilder.ToPortUpdateMap()
	if err != nil {
		return nil, err
	}

	b2, err := gophercloud.BuildRequestBody(opts, "")
	if err != nil {
		return nil, err
	}

	port := b1["port"].(map[string]interface{})

	for k, v := range b2 {
		port[k] = v
	}

	return map[string]interface{}{"port": port}, nil
}

// Update accepts a UpdateOpts struct and updates an existing port using the
// values provided.
func Update(c *gophercloud.ServiceClient, id string, opts ports.UpdateOptsBuilder) (r UpdateResult) {
	b, err := opts.ToPortUpdateMap()
	if err != nil {
		r.Err = err
		return r
	}
	_, r.Err = c.Put(updateURL(c, id), b, &r.Body, &gophercloud.RequestOpts{
		OkCodes: []int{200, 201},
	})
	return
}
<|endoftext|>


<commit_before>package bosh

import (
	"errors"
	"io"
	"time"

	"github.com/skriptble/nine/element"
)

var ErrRequestClosed = errors.New("request has already been responded to")

type Request struct {
	rid  int
	wait time.Duration
	sid  string
	body Body

	proceed  chan struct{}
	closed   chan struct{}
	payload  []element.Element
	response Body
	spent    bool

	// The function should return the highest rid processed.
	// ack is a function called when the request is being spent and sending a
	// response. This can either happen due to a timeout or because data to
	// write has been received. This function should return the highest rid
	// processed by the session
	ack func() int
}

func NewRequest(rid int, wait time.Duration, sid string, b, response Body, ack func() int) *Request {
	return &Request{
		rid:      rid,
		wait:     wait,
		sid:      sid,
		body:     b,
		response: response,
		ack:      ack,
		proceed:  make(chan struct{}),
		closed:   make(chan struct{}),
	}
}

// Write adds the given elements as the payload for the response body.
func (r *Request) Write(els ...element.Element) error {
	if r.spent {
		return ErrRequestClosed
	}
	r.payload = els
	r.spent = true
	close(r.proceed)
	return nil
}

func (r *Request) Close() {
	close(r.closed)
}

// RID returns the request ID of this Request.
func (r *Request) RID() int { return r.rid }

func (r *Request) Elements() []element.Element {
	return r.body.Children
}

func (r *Request) Handle(w io.Writer) {
	select {
	case <-r.proceed:
	case <-r.closed:
		r.spent = true
	case <-time.After(r.wait):
		r.spent = true
	}
	r.response.Ack = r.ack()
	r.response.Children = r.payload
	w.Write(r.response.TransformElement().WriteBytes())
}
<commit_msg>Fixing race condition in request<commit_after>package bosh

import (
	"errors"
	"io"
	"sync"
	"time"

	"github.com/skriptble/nine/element"
)

var ErrRequestClosed = errors.New("request has already been responded to")

type Request struct {
	rid  int
	wait time.Duration
	sid  string
	body Body

	proceed  chan struct{}
	closed   chan struct{}
	payload  []element.Element
	response Body
	spent    bool

	// The function should return the highest rid processed.
	// ack is a function called when the request is being spent and sending a
	// response. This can either happen due to a timeout or because data to
	// write has been received. This function should return the highest rid
	// processed by the session
	ack func() int
	sync.Mutex
}

func NewRequest(rid int, wait time.Duration, sid string, b, response Body, ack func() int) *Request {
	return &Request{
		rid:      rid,
		wait:     wait,
		sid:      sid,
		body:     b,
		response: response,
		ack:      ack,
		proceed:  make(chan struct{}),
		closed:   make(chan struct{}),
	}
}

// Write adds the given elements as the payload for the response body.
func (r *Request) Write(els ...element.Element) error {
	r.Lock()
	defer r.Unlock()
	if r.spent {
		return ErrRequestClosed
	}
	r.payload = els
	r.spent = true
	close(r.proceed)
	return nil
}

func (r *Request) Close() {
	close(r.closed)
}

// RID returns the request ID of this Request.
func (r *Request) RID() int { return r.rid }

func (r *Request) Elements() []element.Element {
	return r.body.Children
}

func (r *Request) Handle(w io.Writer) {
	select {
	case <-r.proceed:
	case <-r.closed:
		r.Lock()
		defer r.Unlock()
		r.spent = true
	case <-time.After(r.wait):
		r.Lock()
		defer r.Unlock()
		r.spent = true
	}
	r.response.Ack = r.ack()
	r.response.Children = r.payload
	w.Write(r.response.TransformElement().WriteBytes())
}
<|endoftext|>


<commit_before>package fox

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"util"
)

func getFileName(uuid string) string {
	return util.GetConfig().Storage.Filepath + uuid
}

// StoreFox persists the fox instance to somewhere
func StoreFox(fox Fox, uuid string) UUID {
	f, err := os.Create(getFileName(uuid))

	if err != nil {
		panic(err)
	}

	defer f.Close()

	// Make sure the file name matches the uuid in the structure
	fox.Uuid = uuid
	if err := json.NewEncoder(f).Encode(fox); err != nil {
		panic(err)
	}

	return UUID{Uuid: uuid}
}

func ReadFox(uuid string) (Fox, error) {
	var fox Fox
	data, err := ioutil.ReadFile(getFileName(uuid))

	if err != nil {
		return fox, err
	}

	if err := json.Unmarshal(data, &fox); err != nil {
		return fox, err
	}

	return fox, nil
}

func GetFoxes() ([]Fox, error) {
	var foxes []Fox

	foxes = make([]Fox, 0)
	fname := util.GetConfig().Storage.Filepath
	files, _ := ioutil.ReadDir(fname)
	for _, f := range files {
		fox, err := ReadFox(f.Name())
		if err != nil {
			return foxes, err
		}
		foxes = append(foxes, fox)
	}

	return foxes, nil
}

func FoxExists(uuid string) bool {
	if _, err := os.Stat(getFileName(uuid)); os.IsNotExist(err) {
		return false
	}
	return true
}

// Deletes the fox, if it exists. Does nothing if it does not
func DeleteFoxFromStorage(uuid string) {

	if !FoxExists(uuid) {
		// Exit quietly, if the fox is not there
		return
	}

	// Attempt to remove the file if it is there. Panic if it fails
	if err := os.Remove(getFileName(uuid)); err != nil {
		panic(err)
	}
}
<commit_msg>added chmod 0744 to StoreFox() for all the tests to work<commit_after>package fox

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"util"
)

func getFileName(uuid string) string {
	return util.GetConfig().Storage.Filepath + uuid
}

// StoreFox persists the fox instance to somewhere
func StoreFox(fox Fox, uuid string) UUID {

	err := os.Chmod(getFileName(""), 0744)

	if err != nil {
		panic(err)
	}

	f, err := os.Create(getFileName(uuid))

	if err != nil {
		panic(err)
	}

	defer f.Close()

	// Make sure the file name matches the uuid in the structure
	fox.Uuid = uuid
	if err := json.NewEncoder(f).Encode(fox); err != nil {
		panic(err)
	}

	return UUID{Uuid: uuid}
}

func ReadFox(uuid string) (Fox, error) {
	var fox Fox
	data, err := ioutil.ReadFile(getFileName(uuid))

	if err != nil {
		return fox, err
	}

	if err := json.Unmarshal(data, &fox); err != nil {
		return fox, err
	}

	return fox, nil
}

func GetFoxes() ([]Fox, error) {
	var foxes []Fox

	foxes = make([]Fox, 0)
	fname := util.GetConfig().Storage.Filepath
	files, _ := ioutil.ReadDir(fname)
	for _, f := range files {
		fox, err := ReadFox(f.Name())
		if err != nil {
			return foxes, err
		}
		foxes = append(foxes, fox)
	}

	return foxes, nil
}

func FoxExists(uuid string) bool {
	if _, err := os.Stat(getFileName(uuid)); os.IsNotExist(err) {
		return false
	}
	return true
}

// Deletes the fox, if it exists. Does nothing if it does not
func DeleteFoxFromStorage(uuid string) {

	if !FoxExists(uuid) {
		// Exit quietly, if the fox is not there
		return
	}

	// Attempt to remove the file if it is there. Panic if it fails
	if err := os.Remove(getFileName(uuid)); err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"html/template"
	"log"
	"net/http"
	"time"
	"os"
	"io"
	"bufio"
	"bytes"
	"strconv"
	"strings"
)

var (
	port        = flag.String("port", "8497", "tail log port number")
	logFileName = flag.String("log", "", "tail log file path")
	homeTempl   = template.Must(template.New("").Parse(homeHTML))
)

func readFileIfModified(lastMod time.Time, seekPos, endPos int64, filterKeyword string) ([]byte, time.Time, int64, error) {
	fi, err := os.Stat(*logFileName)
	if err != nil {
		return nil, lastMod, 0, err
	}
	if !fi.ModTime().After(lastMod) {
		return nil, lastMod, fi.Size(), nil
	}

	input, err := os.Open(*logFileName)
	if err != nil {
		return nil, lastMod, fi.Size(), err
	}
	defer input.Close()

	if seekPos < 0 {
		seekPos = fi.Size() + seekPos
	}

	if _, err := input.Seek(seekPos, 0); err != nil {
		return nil, lastMod, fi.Size(), err
	}

	p, lastPos, err := readContent(input, seekPos, endPos, filterKeyword)
	return p, fi.ModTime(), lastPos, err
}

func readContent(input io.ReadSeeker, startPos, endPos int64, filterKeyword string) ([]byte, int64, error) {
	r := bufio.NewReader(input)

	var buffer bytes.Buffer
	firstLine := true
	pos := startPos
	for endPos < 0 || pos < endPos {
		data, err := r.ReadBytes('\n')
		len := len(data)
		pos += int64(len)
		if err == nil || err == io.EOF {
			if firstLine { // jump the first line because of it may be not full.
				firstLine = false
				continue;
			}
			if len > 0 {
				line := string(data)
				if filterKeyword == "" || strings.Contains(line, filterKeyword) {
					buffer.WriteString(line)
				}
			} else {
				break
			}
		} else if err != nil {
			if err == io.EOF {
				break
			}
			return nil, pos, err
		}
	}

	return buffer.Bytes(), pos, nil
}

func hexString(val int64) string {
	return strconv.FormatInt(val, 16)
}

func parseHex(val string) (int64, error) {
	return strconv.ParseInt(val, 16, 64)
}

func serveTail(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	var lastMod time.Time
	if n, err := parseHex(r.FormValue("lastMod")); err == nil {
		lastMod = time.Unix(0, n)
	}

	seekPos, err := parseHex(r.FormValue("seekPos"))

	filterKeyword := r.FormValue("filterKeyword")

	p, lastMod, seekPos, err := readFileIfModified(lastMod, seekPos, -1, filterKeyword)
	if err != nil {
		log.Println("readFileIfModified error", err)
		return
	}

	w.Header().Set("last-mod", hexString(lastMod.UnixNano()))
	w.Header().Set("seek-pos", hexString(seekPos))
	w.Write(p)
}

func serveHome(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path != "/" {
		http.Error(w, "Not found", 404)
		return
	}
	if r.Method != "GET" {
		http.Error(w, "Method not allowed", 405)
		return
	}
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	p, lastMod, fileSize, err := readFileIfModified(time.Time{}, -600, -1, "")
	if err != nil {
		log.Println("readFileIfModified error", err)
		p = []byte(err.Error())
		lastMod = time.Unix(0, 0)
	}

	var v = struct {
		Data        string
		SeekPos     string
		LastMod     string
		LogFileName string
	}{
		string(p),
		hexString(fileSize),
		hexString(lastMod.UnixNano()),
		*logFileName,
	}
	homeTempl.Execute(w, &v)
}

func main() {
	flag.Parse()

	http.HandleFunc("/", serveHome)
	http.HandleFunc("/tail", serveTail)
	if err := http.ListenAndServe(":" + *port, nil); err != nil {
		log.Fatal(err)
	}
}

const homeHTML = `<!DOCTYPE html>

<title>{{.LogFileName}}</title>

{{.Data}}

	<input type="text" id="filterKeyword" placeholder="请输入过滤关键字"></input>
	<input type="checkbox" id="toggleWrapCheckbox" checked="checked">自动换行</input>
	<input type="checkbox" id="autoRefreshCheckbox" checked="checked">自动刷新</input>
	<button id="refreshButton">刷新</button>
	<button id="clearButton">清空</button>

`
<commit_msg>add window.location.pathname to ajax url<commit_after>package main

import (
	"flag"
	"html/template"
	"log"
	"net/http"
	"time"
	"os"
	"io"
	"bufio"
	"bytes"
	"strconv"
	"strings"
)

var (
	port        = flag.String("port", "8497", "tail log port number")
	logFileName = flag.String("log", "", "tail log file path")
	homeTempl   = template.Must(template.New("").Parse(homeHTML))
)

func readFileIfModified(lastMod time.Time, seekPos, endPos int64, filterKeyword string) ([]byte, time.Time, int64, error) {
	fi, err := os.Stat(*logFileName)
	if err != nil {
		return nil, lastMod, 0, err
	}
	if !fi.ModTime().After(lastMod) {
		return nil, lastMod, fi.Size(), nil
	}

	input, err := os.Open(*logFileName)
	if err != nil {
		return nil, lastMod, fi.Size(), err
	}
	defer input.Close()

	if seekPos < 0 {
		seekPos = fi.Size() + seekPos
	} else if seekPos > fi.Size() {
		seekPos = 0
	}

	if _, err := input.Seek(seekPos, 0); err != nil {
		return nil, lastMod, fi.Size(), err
	}

	p, lastPos, err := readContent(input, seekPos, endPos, filterKeyword)
	return p, fi.ModTime(), lastPos, err
}

func readContent(input io.ReadSeeker, startPos, endPos int64, filterKeyword string) ([]byte, int64, error) {
	r := bufio.NewReader(input)

	var buffer bytes.Buffer
	firstLine := true
	pos := startPos
	for endPos < 0 || pos < endPos {
		data, err := r.ReadBytes('\n')
		len := len(data)
		pos += int64(len)
		if err == nil || err == io.EOF {
			if firstLine { // jump the first line because of it may be not full.
				firstLine = false
				continue
			}
			if len > 0 {
				line := string(data)
				if filterKeyword == "" || strings.Contains(line, filterKeyword) {
					buffer.WriteString(line)
				}
			} else {
				break
			}
		} else if err != nil {
			if err == io.EOF {
				break
			}
			return nil, pos, err
		}
	}

	return buffer.Bytes(), pos, nil
}

func hexString(val int64) string {
	return strconv.FormatInt(val, 16)
}

func parseHex(val string) (int64, error) {
	return strconv.ParseInt(val, 16, 64)
}

func serveTail(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	var lastMod time.Time
	if n, err := parseHex(r.FormValue("lastMod")); err == nil {
		lastMod = time.Unix(0, n)
	}

	seekPos, err := parseHex(r.FormValue("seekPos"))

	filterKeyword := r.FormValue("filterKeyword")

	p, lastMod, seekPos, err := readFileIfModified(lastMod, seekPos, -1, filterKeyword)
	if err != nil {
		log.Println("readFileIfModified error", err)
		return
	}

	w.Header().Set("last-mod", hexString(lastMod.UnixNano()))
	w.Header().Set("seek-pos", hexString(seekPos))
	w.Write(p)
}

func serveHome(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path != "/" {
		http.Error(w, "Not found", 404)
		return
	}
	if r.Method != "GET" {
		http.Error(w, "Method not allowed", 405)
		return
	}
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	p, lastMod, fileSize, err := readFileIfModified(time.Time{}, -600, -1, "")
	if err != nil {
		log.Println("readFileIfModified error", err)
		p = []byte(err.Error())
		lastMod = time.Unix(0, 0)
	}

	var v = struct {
		Data        string
		SeekPos     string
		LastMod     string
		LogFileName string
	}{
		string(p),
		hexString(fileSize),
		hexString(lastMod.UnixNano()),
		*logFileName,
	}
	homeTempl.Execute(w, &v)
}

func main() {
	flag.Parse()

	http.HandleFunc("/", serveHome)
	http.HandleFunc("/tail", serveTail)
	if err := http.ListenAndServe(":" + *port, nil); err != nil {
		log.Fatal(err)
	}
}

const homeHTML = `<!DOCTYPE html>

<title>{{.LogFileName}}</title>

{{.Data}}

	<input type="text" id="filterKeyword" placeholder="请输入过滤关键字"></input>
	<input type="checkbox" id="toggleWrapCheckbox" checked="checked">自动换行</input>
	<input type="checkbox" id="autoRefreshCheckbox" checked="checked">自动刷新</input>
	<button id="refreshButton">刷新</button>
	<button id="clearButton">清空</button>

`
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package Printer

import Scanner "scanner"
import AST "ast"

// Printer implements AST.Visitor
type Printer struct {
	level int;  // true scope level
	indent int;  // indentation level
	semi bool;  // pending ";"
	newl bool;  // pending "\n"
	prec int;  // operator precedence
}

func (P *Printer) String(s string) {
	if P.semi && P.level > 0 {  // no semicolons at level 0
		print(";");
	}
	if P.newl {
		print("\n");
		for i := P.indent; i > 0; i-- {
			print("\t");
		}
	}
	print(s);
	P.newl, P.semi = false, false;
}

func (P *Printer) NewLine() {  // explicit "\n"
	print("\n");
	P.semi, P.newl = false, true;
}

func (P *Printer) OpenScope(paren string) {
	P.semi, P.newl = false, false;
	P.String(paren);
	P.level++;
	P.indent++;
	P.newl = true;
}

func (P *Printer) CloseScope(paren string) {
	P.level--;
	P.indent--;
	P.newl = true;
	P.String(paren);
	P.semi, P.newl = false, true;
}

func (P *Printer) Print(x AST.Node) {
	outer := P.prec;
	P.prec = 0;
	x.Visit(P);
	P.prec = outer;
}

func (P *Printer) PrintList(p *AST.List) {
	for i := 0; i < p.len(); i++ {
		if i > 0 {
			P.String(", ");
		}
		P.Print(p.at(i));
	}
}

// ----------------------------------------------------------------------------
// Basics

func (P *Printer) DoNil(x *AST.Nil) {
	P.String("<NIL>");
}

func (P *Printer) DoIdent(x *AST.Ident) {
	P.String(x.val);
}

// ----------------------------------------------------------------------------
// Types

func (P *Printer) DoFunctionType(x *AST.FunctionType) {
	P.String("(");
	P.PrintList(x.params);
	P.String(")");
	if x.result != nil {
		P.String(" (");
		P.PrintList(x.result);
		P.String(")");
	}
}

func (P *Printer) DoArrayType(x *AST.ArrayType) {
	P.String("[");
	P.Print(x.len_);
	P.String("] ");
	P.Print(x.elt);
}

func (P *Printer) DoStructType(x *AST.StructType) {
	P.String("struct ");
	P.OpenScope("{");
	for i := 0; i < x.fields.len(); i++ {
		P.Print(x.fields.at(i));
		P.newl, P.semi = true, true;
	}
	P.CloseScope("}");
}

func (P *Printer) DoMapType(x *AST.MapType) {
	P.String("[");
	P.Print(x.key);
	P.String("] ");
	P.Print(x.val);
}

func (P *Printer) DoChannelType(x *AST.ChannelType) {
	switch x.mode {
	case AST.FULL: P.String("chan ");
	case AST.RECV: P.String("<-chan ");
	case AST.SEND: P.String("chan <- ");
	}
	P.Print(x.elt);
}

func (P *Printer) DoInterfaceType(x *AST.InterfaceType) {
	P.String("interface ");
	P.OpenScope("{");
	for i := 0; i < x.methods.len(); i++ {
		P.Print(x.methods.at(i));
		P.newl, P.semi = true, true;
	}
	P.CloseScope("}");
}

func (P *Printer) DoPointerType(x *AST.PointerType) {
	P.String("*");
	P.Print(x.base);
}

// ----------------------------------------------------------------------------
// Declarations

func (P *Printer) DoBlock(x *AST.Block);

func (P *Printer) DoImportDecl(x *AST.ImportDecl) {
	if x.ident != nil {
		P.Print(x.ident);
		P.String(" ");
	}
	P.String(x.file);
}

func (P *Printer) DoConstDecl(x *AST.ConstDecl) {
	P.Print(x.ident);
	P.String(" ");
	P.Print(x.typ);
	P.String(" = ");
	P.Print(x.val);
	P.semi = true;
}

func (P *Printer) DoTypeDecl(x *AST.TypeDecl) {
	P.Print(x.ident);
	P.String(" ");
	P.Print(x.typ);
	P.semi = true;
}

func (P *Printer) DoVarDecl(x *AST.VarDecl) {
	P.PrintList(x.idents);
	P.String(" ");
	P.Print(x.typ);
	if x.vals != nil {
		P.String(" = ");
		P.PrintList(x.vals);
	}
	P.semi = true;
}

func (P *Printer) DoVarDeclList(x *AST.VarDeclList) {
	if x.idents != nil {
		P.PrintList(x.idents);
		P.String(" ");
	}
	P.Print(x.typ);
}

func (P *Printer) DoFuncDecl(x *AST.FuncDecl) {
	P.String("func ");
	if x.typ.recv != nil {
		P.String("(");
		P.DoVarDeclList(x.typ.recv);
		P.String(") ");
	}
	P.DoIdent(x.ident);
	P.DoFunctionType(x.typ);
	if x.body != nil {
		P.DoBlock(x.body);
	} else {
		P.String(" ;");
	}
	P.NewLine();
	P.NewLine();
}

func (P *Printer) DoMethodDecl(x *AST.MethodDecl) {
	P.DoIdent(x.ident);
	P.DoFunctionType(x.typ);
}

func (P *Printer) DoDeclaration(x *AST.Declaration) {
	P.String(Scanner.TokenName(x.tok));
	P.String(" ");
	switch x.decls.len() {
	case 0:
		P.String("()");
	case 1:
		P.Print(x.decls.at(0));
	default:
		P.OpenScope(" (");
		for i := 0; i < x.decls.len(); i++ {
			P.Print(x.decls.at(i));
			P.newl, P.semi = true, true;
		}
		P.CloseScope(")");
	}
	if P.level == 0 {
		P.NewLine();
	}
	P.newl = true;
}

// ----------------------------------------------------------------------------
// Expressions

func (P *Printer) DoBinary(x *AST.Binary) {
	outer := P.prec;
	P.prec = Scanner.Precedence(x.tok);

	if P.prec < outer {
		print("(");
	}

	P.Print(x.x);
	P.String(" " + Scanner.TokenName(x.tok) + " ");
	P.Print(x.y);

	if P.prec < outer {
		print(")");
	}

	P.prec = outer;
}

func (P *Printer) DoUnary(x *AST.Unary) {
	P.String(Scanner.TokenName(x.tok));
	P.Print(x.x);
}

func (P *Printer) DoLiteral(x *AST.Literal) {
	P.String(x.val);
}

func (P *Printer) DoPair(x *AST.Pair) {
	P.Print(x.x);
	P.String(" : ");
	P.Print(x.y);
}

func (P *Printer) DoIndex(x *AST.Index) {
	P.Print(x.x);
	P.String("[");
	P.Print(x.index);
	P.String("]");
}

func (P *Printer) DoCall(x *AST.Call) {
	P.Print(x.fun);
	P.String("(");
	P.PrintList(x.args);
	P.String(")");
}

func (P *Printer) DoSelector(x *AST.Selector) {
	P.Print(x.x);
	P.String(".");
	P.String(x.field);
}

func (P *Printer) DoCompositeLit(x *AST.CompositeLit) {
	P.Print(x.typ);
	P.String("{");
	P.PrintList(x.vals);
	P.String("}");
}

func (P *Printer) DoFunctionLit(x *AST.FunctionLit) {
	P.String("func ");
	P.Print(x.typ);
	P.String(" ");
	P.Print(x.body);
}

// ----------------------------------------------------------------------------
// Statements

func (P *Printer) DoBlock(x *AST.Block) {
	P.OpenScope("{");
	for i := 0; i < x.stats.len(); i++ {
		P.Print(x.stats.at(i));
		P.newl = true;
	}
	P.CloseScope("}");
}

func (P *Printer) DoLabel(x *AST.Label) {
	P.indent--;
	P.newl = true;
	P.Print(x.ident);
	P.String(":");
	P.indent++;
}

func (P *Printer) DoExprStat(x *AST.ExprStat) {
	P.Print(x.expr);
	P.semi = true;
}

func (P *Printer) DoAssignment(x *AST.Assignment) {
	P.PrintList(x.lhs);
	P.String(" " + Scanner.TokenName(x.tok) + " ");
	P.PrintList(x.rhs);
	P.semi = true;
}

func (P *Printer) PrintControlClause(x *AST.ControlClause) {
	if x.has_init {
		P.String(" ");
		P.Print(x.init);
		P.semi = true;
		P.String("");
	}
	if x.has_expr {
		P.String(" ");
		P.Print(x.expr);
		P.semi = false;
	}
	if x.has_post {
		P.semi = true;
		P.String(" ");
		P.Print(x.post);
		P.semi = false;
	}
	P.String(" ");
}

func (P *Printer) DoIfStat(x *AST.IfStat) {
	P.String("if");
	P.PrintControlClause(x.ctrl);
	P.DoBlock(x.then);
	if x.has_else {
		P.newl = false;
		P.String(" else ");
		P.Print(x.else_);
	}
}

func (P *Printer) DoForStat(x *AST.ForStat) {
	P.String("for");
	P.PrintControlClause(x.ctrl);
	P.DoBlock(x.body);
}

func (P *Printer) DoCaseClause(x *AST.CaseClause) {
	if x.exprs != nil {
		P.String("case ");
		P.PrintList(x.exprs);
		P.String(":");
	} else {
		P.String("default:");
	}

	P.OpenScope("");
	for i := 0; i < x.stats.len(); i++ {
		P.Print(x.stats.at(i));
		P.newl = true;
	}
	if x.falls {
		P.String("fallthrough");
	}
	P.CloseScope("");
}

func (P *Printer) DoSwitchStat(x *AST.SwitchStat) {
	P.String("switch ");
	P.PrintControlClause(x.ctrl);
	P.OpenScope("{");
	P.indent--;
	for i := 0; i < x.cases.len(); i++ {
		P.Print(x.cases.at(i));
	}
	P.indent++;
	P.CloseScope("}");
}

func (P *Printer) DoReturnStat(x *AST.ReturnStat) {
	P.String("return ");
	P.PrintList(x.res);
	P.semi = true;
}

func (P *Printer) DoIncDecStat(x *AST.IncDecStat) {
	P.Print(x.expr);
	P.String(Scanner.TokenName(x.tok));
	P.semi = true;
}

func (P *Printer) DoControlFlowStat(x *AST.ControlFlowStat) {
	P.String(Scanner.TokenName(x.tok));
	if x.label != nil {
		P.String(" ");
		P.Print(x.label);
	}
	P.semi = true;
}

func (P *Printer) DoGoStat(x *AST.GoStat) {
	P.String("go ");
	P.Print(x.expr);
	P.semi = true;
}

// ----------------------------------------------------------------------------
// Program

func (P *Printer) DoProgram(x *AST.Program) {
	P.String("package ");
	P.DoIdent(x.ident);
	P.NewLine();
	for i := 0; i < x.decls.len(); i++ {
		P.Print(x.decls.at(i));
	}
	P.newl = true;
	P.String("");
}

// ----------------------------------------------------------------------------
// Driver

export func Print(x AST.Node) {
	var P Printer;
	(&P).Print(x);
	print("\n");
}

<commit_msg>more fine-tuning of ;'s<commit_after>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package Printer

import Scanner "scanner"
import AST "ast"

// Printer implements AST.Visitor
type Printer struct {
	level int;  // true scope level
	indent int;  // indentation level
	semi bool;  // pending ";"
	newl bool;  // pending "\n"
	prec int;  // operator precedence
}

func (P *Printer) String(s string) {
	if P.semi && P.level > 0 {  // no semicolons at level 0
		print(";");
	}
	if P.newl {
		print("\n");
		for i := P.indent; i > 0; i-- {
			print("\t");
		}
	}
	print(s);
	P.newl, P.semi = false, false;
}

func (P *Printer) NewLine() {  // explicit "\n"
	print("\n");
	P.semi, P.newl = false, true;
}

func (P *Printer) OpenScope(paren string) {
	P.semi, P.newl = false, false;
	P.String(paren);
	P.level++;
	P.indent++;
	P.newl = true;
}

func (P *Printer) CloseScope(paren string) {
	P.indent--;
	P.semi = false;
	P.String(paren);
	P.level--;
	P.semi, P.newl = false, true;
}

func (P *Printer) Print(x AST.Node) {
	outer := P.prec;
	P.prec = 0;
	x.Visit(P);
	P.prec = outer;
}

func (P *Printer) PrintList(p *AST.List) {
	for i := 0; i < p.len(); i++ {
		if i > 0 {
			P.String(", ");
		}
		P.Print(p.at(i));
	}
}

// ----------------------------------------------------------------------------
// Basics

func (P *Printer) DoNil(x *AST.Nil) {
	P.String("<NIL>");
}

func (P *Printer) DoIdent(x *AST.Ident) {
	P.String(x.val);
}

// ----------------------------------------------------------------------------
// Types

func (P *Printer) DoFunctionType(x *AST.FunctionType) {
	P.String("(");
	P.PrintList(x.params);
	P.String(")");
	if x.result != nil {
		P.String(" (");
		P.PrintList(x.result);
		P.String(")");
	}
}

func (P *Printer) DoArrayType(x *AST.ArrayType) {
	P.String("[");
	P.Print(x.len_);
	P.String("] ");
	P.Print(x.elt);
}

func (P *Printer) DoStructType(x *AST.StructType) {
	P.String("struct ");
	P.OpenScope("{");
	for i := 0; i < x.fields.len(); i++ {
		P.Print(x.fields.at(i));
		P.newl, P.semi = true, true;
	}
	P.CloseScope("}");
}

func (P *Printer) DoMapType(x *AST.MapType) {
	P.String("[");
	P.Print(x.key);
	P.String("] ");
	P.Print(x.val);
}

func (P *Printer) DoChannelType(x *AST.ChannelType) {
	switch x.mode {
	case AST.FULL: P.String("chan ");
	case AST.RECV: P.String("<-chan ");
	case AST.SEND: P.String("chan <- ");
	}
	P.Print(x.elt);
}

func (P *Printer) DoInterfaceType(x *AST.InterfaceType) {
	P.String("interface ");
	P.OpenScope("{");
	for i := 0; i < x.methods.len(); i++ {
		P.Print(x.methods.at(i));
		P.newl, P.semi = true, true;
	}
	P.CloseScope("}");
}

func (P *Printer) DoPointerType(x *AST.PointerType) {
	P.String("*");
	P.Print(x.base);
}

// ----------------------------------------------------------------------------
// Declarations

func (P *Printer) DoBlock(x *AST.Block);

func (P *Printer) DoImportDecl(x *AST.ImportDecl) {
	if x.ident != nil {
		P.Print(x.ident);
		P.String(" ");
	}
	P.String(x.file);
}

func (P *Printer) DoConstDecl(x *AST.ConstDecl) {
	P.Print(x.ident);
	P.String(" ");
	P.Print(x.typ);
	P.String(" = ");
	P.Print(x.val);
	P.semi = true;
}

func (P *Printer) DoTypeDecl(x *AST.TypeDecl) {
	P.Print(x.ident);
	P.String(" ");
	P.Print(x.typ);
	P.semi = true;
}

func (P *Printer) DoVarDecl(x *AST.VarDecl) {
	P.PrintList(x.idents);
	P.String(" ");
	P.Print(x.typ);
	if x.vals != nil {
		P.String(" = ");
		P.PrintList(x.vals);
	}
	P.semi = true;
}

func (P *Printer) DoVarDeclList(x *AST.VarDeclList) {
	if x.idents != nil {
		P.PrintList(x.idents);
		P.String(" ");
	}
	P.Print(x.typ);
}

func (P *Printer) DoFuncDecl(x *AST.FuncDecl) {
	P.String("func ");
	if x.typ.recv != nil {
		P.String("(");
		P.DoVarDeclList(x.typ.recv);
		P.String(") ");
	}
	P.DoIdent(x.ident);
	P.DoFunctionType(x.typ);
	if x.body != nil {
		P.String(" ");
		P.DoBlock(x.body);
	} else {
		P.String(" ;");
	}
	P.NewLine();
	P.NewLine();
}

func (P *Printer) DoMethodDecl(x *AST.MethodDecl) {
	P.DoIdent(x.ident);
	P.DoFunctionType(x.typ);
}

func (P *Printer) DoDeclaration(x *AST.Declaration) {
	P.String(Scanner.TokenName(x.tok));
	P.String(" ");
	switch x.decls.len() {
	case 0:
		P.String("()");
	case 1:
		P.Print(x.decls.at(0));
	default:
		P.OpenScope(" (");
		for i := 0; i < x.decls.len(); i++ {
			P.Print(x.decls.at(i));
			P.newl, P.semi = true, true;
		}
		P.CloseScope(")");
	}
	if P.level == 0 {
		P.NewLine();
	}
	P.newl = true;
}

// ----------------------------------------------------------------------------
// Expressions

func (P *Printer) DoBinary(x *AST.Binary) {
	outer := P.prec;
	P.prec = Scanner.Precedence(x.tok);

	if P.prec < outer {
		print("(");
	}

	P.Print(x.x);
	P.String(" " + Scanner.TokenName(x.tok) + " ");
	P.Print(x.y);

	if P.prec < outer {
		print(")");
	}

	P.prec = outer;
}

func (P *Printer) DoUnary(x *AST.Unary) {
	P.String(Scanner.TokenName(x.tok));
	P.Print(x.x);
}

func (P *Printer) DoLiteral(x *AST.Literal) {
	P.String(x.val);
}

func (P *Printer) DoPair(x *AST.Pair) {
	P.Print(x.x);
	P.String(" : ");
	P.Print(x.y);
}

func (P *Printer) DoIndex(x *AST.Index) {
	P.Print(x.x);
	P.String("[");
	P.Print(x.index);
	P.String("]");
}

func (P *Printer) DoCall(x *AST.Call) {
	P.Print(x.fun);
	P.String("(");
	P.PrintList(x.args);
	P.String(")");
}

func (P *Printer) DoSelector(x *AST.Selector) {
	P.Print(x.x);
	P.String(".");
	P.String(x.field);
}

func (P *Printer) DoCompositeLit(x *AST.CompositeLit) {
	P.Print(x.typ);
	P.String("{");
	P.PrintList(x.vals);
	P.String("}");
}

func (P *Printer) DoFunctionLit(x *AST.FunctionLit) {
	P.String("func ");
	P.Print(x.typ);
	P.String(" ");
	P.Print(x.body);
}

// ----------------------------------------------------------------------------
// Statements

func (P *Printer) DoBlock(x *AST.Block) {
	P.OpenScope("{");
	for i := 0; i < x.stats.len(); i++ {
		P.Print(x.stats.at(i));
		P.newl = true;
	}
	P.CloseScope("}");
}

func (P *Printer) DoLabel(x *AST.Label) {
	P.indent--;
	P.newl = true;
	P.Print(x.ident);
	P.String(":");
	P.indent++;
}

func (P *Printer) DoExprStat(x *AST.ExprStat) {
	P.Print(x.expr);
	P.semi = true;
}

func (P *Printer) DoAssignment(x *AST.Assignment) {
	P.PrintList(x.lhs);
	P.String(" " + Scanner.TokenName(x.tok) + " ");
	P.PrintList(x.rhs);
	P.semi = true;
}

func (P *Printer) PrintControlClause(x *AST.ControlClause) {
	if x.has_init {
		P.String(" ");
		P.Print(x.init);
		P.semi = true;
		P.String("");
	}
	if x.has_expr {
		P.String(" ");
		P.Print(x.expr);
		P.semi = false;
	}
	if x.has_post {
		P.semi = true;
		P.String(" ");
		P.Print(x.post);
		P.semi = false;
	}
	P.String(" ");
}

func (P *Printer) DoIfStat(x *AST.IfStat) {
	P.String("if");
	P.PrintControlClause(x.ctrl);
	P.DoBlock(x.then);
	if x.has_else {
		P.newl = false;
		P.String(" else ");
		P.Print(x.else_);
	}
}

func (P *Printer) DoForStat(x *AST.ForStat) {
	P.String("for");
	P.PrintControlClause(x.ctrl);
	P.DoBlock(x.body);
}

func (P *Printer) DoCaseClause(x *AST.CaseClause) {
	if x.exprs != nil {
		P.String("case ");
		P.PrintList(x.exprs);
		P.String(":");
	} else {
		P.String("default:");
	}

	P.OpenScope("");
	for i := 0; i < x.stats.len(); i++ {
		P.Print(x.stats.at(i));
		P.newl = true;
	}
	if x.falls {
		P.String("fallthrough");
	}
	P.CloseScope("");
}

func (P *Printer) DoSwitchStat(x *AST.SwitchStat) {
	P.String("switch ");
	P.PrintControlClause(x.ctrl);
	P.OpenScope("{");
	P.indent--;
	for i := 0; i < x.cases.len(); i++ {
		P.Print(x.cases.at(i));
	}
	P.indent++;
	P.CloseScope("}");
}

func (P *Printer) DoReturnStat(x *AST.ReturnStat) {
	P.String("return ");
	P.PrintList(x.res);
	P.semi = true;
}

func (P *Printer) DoIncDecStat(x *AST.IncDecStat) {
	P.Print(x.expr);
	P.String(Scanner.TokenName(x.tok));
	P.semi = true;
}

func (P *Printer) DoControlFlowStat(x *AST.ControlFlowStat) {
	P.String(Scanner.TokenName(x.tok));
	if x.label != nil {
		P.String(" ");
		P.Print(x.label);
	}
	P.semi = true;
}

func (P *Printer) DoGoStat(x *AST.GoStat) {
	P.String("go ");
	P.Print(x.expr);
	P.semi = true;
}

// ----------------------------------------------------------------------------
// Program

func (P *Printer) DoProgram(x *AST.Program) {
	P.String("package ");
	P.DoIdent(x.ident);
	P.NewLine();
	for i := 0; i < x.decls.len(); i++ {
		P.Print(x.decls.at(i));
	}
	P.newl = true;
	P.String("");
}

// ----------------------------------------------------------------------------
// Driver

export func Print(x AST.Node) {
	var P Printer;
	(&P).Print(x);
	print("\n");
}

<|endoftext|>


<commit_before>package awsv1shim

import (
	"bytes"
	"context"
	"io"
	"io/ioutil"
	"log"

	"github.com/aws/aws-sdk-go-v2/config"
)

func resolveCustomCABundle(ctx context.Context, configSources []interface{}) (value io.Reader, found bool, err error) {
	for _, source := range configSources {
		switch cfg := source.(type) {
		case config.LoadOptions:
			value, found, err = loadOptionsGetCustomCABundle(ctx, cfg)
		case config.EnvConfig:
			value, found, err = envConfigGetCustomCABundle(ctx, cfg)
		case config.SharedConfig:
			value, found, err = sharedConfigGetCustomCABundle(ctx, cfg)
		default:
			log.Printf("[WARN] Unrecognized config source: %T", source)
			continue
		}
		if err != nil || found {
			break
		}
	}

	return
}

// Copied from https://github.com/aws/aws-sdk-go-v2/blob/889e1da2776ae5bd6d056cf44f6ce6d043237769/config/load_options.go#L334-L340
func loadOptionsGetCustomCABundle(_ context.Context, o config.LoadOptions) (io.Reader, bool, error) { //nolint:unparam
	if o.CustomCABundle == nil {
		return nil, false, nil
	}

	return o.CustomCABundle, true, nil
}

// Copied from https://github.com/aws/aws-sdk-go-v2/blob/889e1da2776ae5bd6d056cf44f6ce6d043237769/config/env_config.go#L463-L473
func envConfigGetCustomCABundle(_ context.Context, c config.EnvConfig) (io.Reader, bool, error) {
	if len(c.CustomCABundle) == 0 {
		return nil, false, nil
	}

	b, err := ioutil.ReadFile(c.CustomCABundle)
	if err != nil {
		return nil, false, err
	}
	return bytes.NewReader(b), true, nil
}

// Copied from https://github.com/aws/aws-sdk-go-v2/blob/889e1da2776ae5bd6d056cf44f6ce6d043237769/config/shared_config.go#L350-L360
func sharedConfigGetCustomCABundle(_ context.Context, c config.SharedConfig) (io.Reader, bool, error) {
	if len(c.CustomCABundle) == 0 {
		return nil, false, nil
	}

	b, err := ioutil.ReadFile(c.CustomCABundle)
	if err != nil {
		return nil, false, err
	}
	return bytes.NewReader(b), true, nil
}
<commit_msg>Fixes semgrep<commit_after>package awsv1shim

import ( // nosemgrep: no-sdkv2-imports-in-awsv1shim
	"bytes"
	"context"
	"io"
	"io/ioutil"
	"log"

	"github.com/aws/aws-sdk-go-v2/config"
)

func resolveCustomCABundle(ctx context.Context, configSources []interface{}) (value io.Reader, found bool, err error) {
	for _, source := range configSources {
		switch cfg := source.(type) {
		case config.LoadOptions:
			value, found, err = loadOptionsGetCustomCABundle(ctx, cfg)
		case config.EnvConfig:
			value, found, err = envConfigGetCustomCABundle(ctx, cfg)
		case config.SharedConfig:
			value, found, err = sharedConfigGetCustomCABundle(ctx, cfg)
		default:
			log.Printf("[WARN] Unrecognized config source: %T", source)
			continue
		}
		if err != nil || found {
			break
		}
	}

	return
}

// Copied from https://github.com/aws/aws-sdk-go-v2/blob/889e1da2776ae5bd6d056cf44f6ce6d043237769/config/load_options.go#L334-L340
func loadOptionsGetCustomCABundle(_ context.Context, o config.LoadOptions) (io.Reader, bool, error) { //nolint:unparam
	if o.CustomCABundle == nil {
		return nil, false, nil
	}

	return o.CustomCABundle, true, nil
}

// Copied from https://github.com/aws/aws-sdk-go-v2/blob/889e1da2776ae5bd6d056cf44f6ce6d043237769/config/env_config.go#L463-L473
func envConfigGetCustomCABundle(_ context.Context, c config.EnvConfig) (io.Reader, bool, error) {
	if len(c.CustomCABundle) == 0 {
		return nil, false, nil
	}

	b, err := ioutil.ReadFile(c.CustomCABundle)
	if err != nil {
		return nil, false, err
	}
	return bytes.NewReader(b), true, nil
}

// Copied from https://github.com/aws/aws-sdk-go-v2/blob/889e1da2776ae5bd6d056cf44f6ce6d043237769/config/shared_config.go#L350-L360
func sharedConfigGetCustomCABundle(_ context.Context, c config.SharedConfig) (io.Reader, bool, error) {
	if len(c.CustomCABundle) == 0 {
		return nil, false, nil
	}

	b, err := ioutil.ReadFile(c.CustomCABundle)
	if err != nil {
		return nil, false, err
	}
	return bytes.NewReader(b), true, nil
}
<|endoftext|>


<commit_before>package client

import (
	"net/url"
	"strings"
)

func encodeParameter(value string) string {
	return strings.Replace(url.QueryEscape(value), "/", "%2F", 0)
}

// Tag returns current tag for push event hook payload
// This function returns empty string for any other events
func (h *HookPayload) Tag() string {
	return strings.TrimPrefix(h.Ref, "refs/tags/")
}

// Branch returns current branch for push event hook payload
// This function returns empty string for any other events
func (h *HookPayload) Branch() string {
	return strings.TrimPrefix(h.Ref, "refs/heads/")
}

// Head returns the latest changeset for push event hook payload
func (h *HookPayload) Head() hCommit {
	c := hCommit{}
	for _, cm := range h.Commits {
		if h.After == cm.Id {
			return cm
		}
	}
	return c
}
<commit_msg>Dot names fix<commit_after>package client

import (
	"net/url"
	"strings"
)

var encodeMap = map[string]string{
	".": "%252E",
}

func encodeParameter(value string) string {
	value = url.QueryEscape(value)

	for before, after := range encodeMap {
		value = strings.Replace(value, before, after, -1)
	}

	return value
}

// Tag returns current tag for push event hook payload
// This function returns empty string for any other events
func (h *HookPayload) Tag() string {
	return strings.TrimPrefix(h.Ref, "refs/tags/")
}

// Branch returns current branch for push event hook payload
// This function returns empty string for any other events
func (h *HookPayload) Branch() string {
	return strings.TrimPrefix(h.Ref, "refs/heads/")
}

// Head returns the latest changeset for push event hook payload
func (h *HookPayload) Head() hCommit {
	c := hCommit{}
	for _, cm := range h.Commits {
		if h.After == cm.Id {
			return cm
		}
	}
	return c
}
<|endoftext|>


<commit_before>package anteater

import (
	"time"
	"runtime"
	"sort"
	"sync/atomic"
)

var FiveSecondsCounters []*StateHttpCounters = make([]*StateHttpCounters, 61)
var FiveSecondsCursor int

type State struct {
	Main  *StateMain
	Files *StateFiles
	Counters *StateHttpCounters
	CountersLast5Seconds *StateHttpCounters
	CountersLastMinute   *StateHttpCounters
	CountersLast5Minutes *StateHttpCounters
	Alloc *StateAllocateCounters
}

type StateMain struct {
	Time          int64
	StartTime     int64
	Goroutines    int
	MemoryUsage   uint64
	LastDump      int64
	LastDumpTime  int64
	IndexFileSize int64
}

type StateFiles struct {
	ContainersCount int
	FilesCount      int64
	FilesSize       int64
	SpacesCount    int64
	SpacesSize     int64
}

type StateHttpCounters struct {
	Get      int64
	Add      int64
	Delete   int64
	NotFound int64
}

type StateAllocateCounters struct {
	ReplaceSpace int64
	AddToSpace   int64
	ToEnd        int64
}

func init() {
	FiveSecondsTick()
	go func() {
		ch := time.Tick(5 * time.Second)
		for _ = range ch {
			func () {
				FiveSecondsTick()
			}()
		}
	}()
}

func GetState() *State {
	ms := &runtime.MemStats{}
	runtime.ReadMemStats(ms)
	m := &StateMain{
		Time         : time.Now().Unix(),
		StartTime    : StartTime.Unix(),
		Goroutines   : runtime.NumGoroutine(),
		MemoryUsage  : ms.TotalAlloc,
		LastDump     : LastDump.Unix(),
		LastDumpTime : LastDumpTime,
		IndexFileSize: IndexFileSize,
	}

	GetFileLock.Lock()
	defer GetFileLock.Unlock()

	var ids []int = make([]int, 0)
	for _, cn := range(FileContainers) {
		ids = append(ids, int(cn.Id))
	}
	sort.Ints(ids)
	var cnt *Container

	var totalSize, totalFileSize, fileCount, spacesCount, spacesTotalSize int64

	for _, id := range(ids) {
		cnt = FileContainers[int32(id)]
		_, sc, st := cnt.Spaces.ToHtml(10, cnt.Size)
		totalSize += cnt.Size
		fileCount += cnt.Count
		spacesCount += sc
		spacesTotalSize += st
		allocated := cnt.Size - (cnt.Size - cnt.Offset) - st
		totalFileSize += allocated
	}

	f := &StateFiles{
		ContainersCount : len(ids),
		FilesCount      : fileCount,
		FilesSize       : totalFileSize,
		SpacesCount     : spacesCount,
		SpacesSize      : spacesTotalSize,
	}

	return &State{
		Main     : m,
		Files    : f,
		Counters : HttpCn,
		CountersLast5Seconds : GetHttpStateByPeriod(1),
		CountersLastMinute : GetHttpStateByPeriod(12),
		CountersLast5Minutes : GetHttpStateByPeriod(60),
		Alloc    : AllocCn,
	}
}

func FiveSecondsTick() {
	if FiveSecondsCursor == 61 {
		FiveSecondsCursor = 0
	}
	if FiveSecondsCounters[FiveSecondsCursor] == nil {
		FiveSecondsCounters[FiveSecondsCursor] = &StateHttpCounters{}
	}
	FiveSecondsCounters[FiveSecondsCursor].SetData(HttpCn)
	FiveSecondsCursor++
}

func GetHttpStateByPeriod(period int) (result *StateHttpCounters) {
	curCursor := FiveSecondsCursor - 1
	diffCursor := curCursor
	result = &StateHttpCounters{}
	i := 0
	for {
		diffCursor--
		i++
		if diffCursor == -1 {
			diffCursor = 60
		}
		if FiveSecondsCounters[diffCursor] == nil {
			diffCursor++
			if diffCursor == 61 {
				diffCursor = 0
			}
			break
		}
		if i >= period {
			break
		}
	}

	cur  := FiveSecondsCounters[curCursor]
	diff := FiveSecondsCounters[diffCursor]

	result.Add = cur.Add - diff.Add
	result.Get = cur.Get - diff.Get
	result.Delete = cur.Delete - diff.Delete
	result.NotFound = cur.NotFound - diff.NotFound
	return
}

func (s *StateHttpCounters) CGet() {
	atomic.AddInt64(&s.Get, 1)
}

func (s *StateHttpCounters) CAdd() {
	atomic.AddInt64(&s.Add, 1)
}

func (s *StateHttpCounters) CDelete() {
	atomic.AddInt64(&s.Delete, 1)
}

func (s *StateHttpCounters) CNotFound() {
	atomic.AddInt64(&s.NotFound, 1)
}

func (s *StateHttpCounters) SetData(otherS *StateHttpCounters) {
	s.Get = otherS.Get
	s.Add = otherS.Add
	s.Delete = otherS.Delete
	s.NotFound = otherS.NotFound
}

func (s *StateAllocateCounters) CTarget(target int) {
	switch target {
		case TARGET_SPACE_EQ:
			atomic.AddInt64(&s.ReplaceSpace, 1)
		case TARGET_SPACE_FREE:
			atomic.AddInt64(&s.AddToSpace, 1)
	 	case TARGET_NEW:
	 		atomic.AddInt64(&s.ToEnd, 1)
	}
}
<commit_msg>rem memory usage<commit_after>package anteater

import (
	"time"
	"runtime"
	"sort"
	"sync/atomic"
)

var FiveSecondsCounters []*StateHttpCounters = make([]*StateHttpCounters, 61)
var FiveSecondsCursor int

type State struct {
	Main  *StateMain
	Files *StateFiles
	Counters *StateHttpCounters
	CountersLast5Seconds *StateHttpCounters
	CountersLastMinute   *StateHttpCounters
	CountersLast5Minutes *StateHttpCounters
	Alloc *StateAllocateCounters
}

type StateMain struct {
	Time          int64
	StartTime     int64
	Goroutines    int
	LastDump      int64
	LastDumpTime  int64
	IndexFileSize int64
}

type StateFiles struct {
	ContainersCount int
	FilesCount      int64
	FilesSize       int64
	SpacesCount    int64
	SpacesSize     int64
}

type StateHttpCounters struct {
	Get      int64
	Add      int64
	Delete   int64
	NotFound int64
}

type StateAllocateCounters struct {
	ReplaceSpace int64
	AddToSpace   int64
	ToEnd        int64
}

func init() {
	FiveSecondsTick()
	go func() {
		ch := time.Tick(5 * time.Second)
		for _ = range ch {
			func () {
				FiveSecondsTick()
			}()
		}
	}()
}

func GetState() *State {
	m := &StateMain{
		Time         : time.Now().Unix(),
		StartTime    : StartTime.Unix(),
		Goroutines   : runtime.NumGoroutine(),
		LastDump     : LastDump.Unix(),
		LastDumpTime : LastDumpTime,
		IndexFileSize: IndexFileSize,
	}

	GetFileLock.Lock()
	defer GetFileLock.Unlock()

	var ids []int = make([]int, 0)
	for _, cn := range(FileContainers) {
		ids = append(ids, int(cn.Id))
	}
	sort.Ints(ids)
	var cnt *Container

	var totalSize, totalFileSize, fileCount, spacesCount, spacesTotalSize int64

	for _, id := range(ids) {
		cnt = FileContainers[int32(id)]
		_, sc, st := cnt.Spaces.ToHtml(10, cnt.Size)
		totalSize += cnt.Size
		fileCount += cnt.Count
		spacesCount += sc
		spacesTotalSize += st
		allocated := cnt.Size - (cnt.Size - cnt.Offset) - st
		totalFileSize += allocated
	}

	f := &StateFiles{
		ContainersCount : len(ids),
		FilesCount      : fileCount,
		FilesSize       : totalFileSize,
		SpacesCount     : spacesCount,
		SpacesSize      : spacesTotalSize,
	}

	return &State{
		Main     : m,
		Files    : f,
		Counters : HttpCn,
		CountersLast5Seconds : GetHttpStateByPeriod(1),
		CountersLastMinute : GetHttpStateByPeriod(12),
		CountersLast5Minutes : GetHttpStateByPeriod(60),
		Alloc    : AllocCn,
	}
}

func FiveSecondsTick() {
	if FiveSecondsCursor == 61 {
		FiveSecondsCursor = 0
	}
	if FiveSecondsCounters[FiveSecondsCursor] == nil {
		FiveSecondsCounters[FiveSecondsCursor] = &StateHttpCounters{}
	}
	FiveSecondsCounters[FiveSecondsCursor].SetData(HttpCn)
	FiveSecondsCursor++
}

func GetHttpStateByPeriod(period int) (result *StateHttpCounters) {
	curCursor := FiveSecondsCursor - 1
	diffCursor := curCursor
	result = &StateHttpCounters{}
	i := 0
	for {
		diffCursor--
		i++
		if diffCursor == -1 {
			diffCursor = 60
		}
		if FiveSecondsCounters[diffCursor] == nil {
			diffCursor++
			if diffCursor == 61 {
				diffCursor = 0
			}
			break
		}
		if i >= period {
			break
		}
	}

	cur  := FiveSecondsCounters[curCursor]
	diff := FiveSecondsCounters[diffCursor]

	result.Add = cur.Add - diff.Add
	result.Get = cur.Get - diff.Get
	result.Delete = cur.Delete - diff.Delete
	result.NotFound = cur.NotFound - diff.NotFound
	return
}

func (s *StateHttpCounters) CGet() {
	atomic.AddInt64(&s.Get, 1)
}

func (s *StateHttpCounters) CAdd() {
	atomic.AddInt64(&s.Add, 1)
}

func (s *StateHttpCounters) CDelete() {
	atomic.AddInt64(&s.Delete, 1)
}

func (s *StateHttpCounters) CNotFound() {
	atomic.AddInt64(&s.NotFound, 1)
}

func (s *StateHttpCounters) SetData(otherS *StateHttpCounters) {
	s.Get = otherS.Get
	s.Add = otherS.Add
	s.Delete = otherS.Delete
	s.NotFound = otherS.NotFound
}

func (s *StateAllocateCounters) CTarget(target int) {
	switch target {
		case TARGET_SPACE_EQ:
			atomic.AddInt64(&s.ReplaceSpace, 1)
		case TARGET_SPACE_FREE:
			atomic.AddInt64(&s.AddToSpace, 1)
	 	case TARGET_NEW:
	 		atomic.AddInt64(&s.ToEnd, 1)
	}
}
<|endoftext|>