<commit_before>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package gRIBIServer is an implementation of the gRIBI gRPC RIB programming interface.
// It interacts with a fake backend which is stored using ygot generated structs.
package gRIBIServer

import (
	"context"
	"sync"

	"github.com/openconfig/gribi/oc"

	"github.com/openconfig/ygot/ygot"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	rpb "github.com/openconfig/gribi/proto/service"

	gpb "github.com/openconfig/gnmi/proto/gnmi"
)

// Server acts as a fake gRIBI server. It can be used to test a gRIBI client
// implementation against, and serves as a reference implementation describing
// the behaviour that is expected of a gRIBI server.
//
// It implements both the gRIBI service, and the gNMI service. When a modifications
// to the RIB is written, it is stored in a per-client instance of the AFT model.
// A decision process is applied based on the clients that exist, and the
// "best" entry is then stored in the overall AFT for the device. When modifications
// to the selected AFT entries are made, these are transmitted as gNMI notifications.
type Server struct {
	// aft stores the backend implementation's selected set of AFT entries.
	aft *internalAFT
	// changes is a channel which is written to by the server when changes occur
	// within the AFT. This allows event-based subscriptions to be supplied with
	// the changes to the AFT entry. The channel passes pointers to a ygot.GoStruct
	// which can be serialised to the relevant format by the RPC handler.
	changes chan *ygot.GoStruct
	// clientAFT stores the backend implementat's per-client AFT entries. It is
	// keyed by a per-client identifier which may be derived from the client address
	// or explicitly specified by the client.
	// TODO(robjs): Clarify how the explicit specification by the client works.
	clientAFT map[string]*internalAFT
}

// internalAFT is an internally stored abstract forwarding table (AFT), which is
// associated with a particular context within the fake gRIBI server.
type internalAFT struct {
	// mu is a read-write mutex that protects accesses to the AFT stored by the internal AFT.
	mu sync.RWMutex
	// aft is the contents of the AFT.
	aft *oc.GribiAft_Afts
}

// Modify is a bidirectional streaming RPC handler which provides a mechanism to
// modify the RIB entries on a device via the gRIBI gRPC service.
func (*Server) Modify(rpb.GRIBI_ModifyServer) error {
	return status.Errorf(codes.Unimplemented, "TODO")
}

// Capabilities ensures that Server implements the gNMI server interface. It is
// unimplemented in this fake.
func (*Server) Capabilities(_ context.Context, _ *gpb.CapabilityRequest) (*gpb.CapabilityResponse, error) {
	return nil, status.Errorf(codes.Unimplemented, "Capabilities RPC is unimplemented for the gRIBI fake")
}

// Get ensures that Server implements the gNMI server interface. It is unimplemented
// in this fake.
func (*Server) Get(_ context.Context, _ *gpb.GetRequest) (*gpb.GetResponse, error) {
	return nil, status.Errorf(codes.Unimplemented, "Get RPC is unimplemented for the gRIBI fake")
}

// Set ensures that the Server implements the gNMI server interface. It is unimplemented
// in this fake.
func (*Server) Set(_ context.Context, _ *gpb.SetRequest) (*gpb.SetResponse, error) {
	return nil, status.Errorf(codes.Unimplemented, "Set RPC is unimplemented for the gRIBI fake")
}

// Subscribe implements the gNMI subscribe RPC. It can be used by a gRIBI client to
// determine whether an entry that was injected into the system was written to the
// active AFT of the system - through creating a subscription to the AFTs on the system.
// Alternatively, a client may retrieve the entire set of active AFT entries through
// use of the Subscribe ONCE mode, whereby a complete set of the referenced path is
// serialised and sent to the client.
func (s *Server) Subscribe(gpb.GNMI_SubscribeServer) error {
	return status.Errorf(codes.Unimplemented, "TODO")
}
<commit_msg>Fix tests.<commit_after>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package gRIBIServer is an implementation of the gRIBI gRPC RIB programming interface.
// It interacts with a fake backend which is stored using ygot generated structs.
package gRIBIServer

import (
	"context"
	"sync"

	"github.com/openconfig/gribi/oc"

	"github.com/openconfig/ygot/ygot"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	rpb "github.com/openconfig/gribi/proto/service"

	gpb "github.com/openconfig/gnmi/proto/gnmi"
)

// Server acts as a fake gRIBI server. It can be used to test a gRIBI client
// implementation against, and serves as a reference implementation describing
// the behaviour that is expected of a gRIBI server.
//
// It implements both the gRIBI service, and the gNMI service. When a modifications
// to the RIB is written, it is stored in a per-client instance of the AFT model.
// A decision process is applied based on the clients that exist, and the
// "best" entry is then stored in the overall AFT for the device. When modifications
// to the selected AFT entries are made, these are transmitted as gNMI notifications.
type Server struct {
	// aft stores the backend implementation's selected set of AFT entries.
	aft *internalAFT
	// changes is a channel which is written to by the server when changes occur
	// within the AFT. This allows event-based subscriptions to be supplied with
	// the changes to the AFT entry. The channel passes pointers to a ygot.GoStruct
	// which can be serialised to the relevant format by the RPC handler.
	changes chan *ygot.GoStruct
	// clientAFT stores the backend implementat's per-client AFT entries. It is
	// keyed by a per-client identifier which may be derived from the client address
	// or explicitly specified by the client.
	// TODO(robjs): Clarify how the explicit specification by the client works.
	clientAFT map[string]*internalAFT

	*rpb.UnimplementedGRIBIServer
}

// internalAFT is an internally stored abstract forwarding table (AFT), which is
// associated with a particular context within the fake gRIBI server.
type internalAFT struct {
	// mu is a read-write mutex that protects accesses to the AFT stored by the internal AFT.
	mu sync.RWMutex
	// aft is the contents of the AFT.
	aft *oc.GribiAft_Afts
}

// Modify is a bidirectional streaming RPC handler which provides a mechanism to
// modify the RIB entries on a device via the gRIBI gRPC service.
func (*Server) Modify(rpb.GRIBI_ModifyServer) error {
	return status.Errorf(codes.Unimplemented, "TODO")
}

// Capabilities ensures that Server implements the gNMI server interface. It is
// unimplemented in this fake.
func (*Server) Capabilities(_ context.Context, _ *gpb.CapabilityRequest) (*gpb.CapabilityResponse, error) {
	return nil, status.Errorf(codes.Unimplemented, "Capabilities RPC is unimplemented for the gRIBI fake")
}

// Get ensures that Server implements the gNMI server interface. It is unimplemented
// in this fake.
func (*Server) Get(_ context.Context, _ *gpb.GetRequest) (*gpb.GetResponse, error) {
	return nil, status.Errorf(codes.Unimplemented, "Get RPC is unimplemented for the gRIBI fake")
}

// Set ensures that the Server implements the gNMI server interface. It is unimplemented
// in this fake.
func (*Server) Set(_ context.Context, _ *gpb.SetRequest) (*gpb.SetResponse, error) {
	return nil, status.Errorf(codes.Unimplemented, "Set RPC is unimplemented for the gRIBI fake")
}

// Subscribe implements the gNMI subscribe RPC. It can be used by a gRIBI client to
// determine whether an entry that was injected into the system was written to the
// active AFT of the system - through creating a subscription to the AFTs on the system.
// Alternatively, a client may retrieve the entire set of active AFT entries through
// use of the Subscribe ONCE mode, whereby a complete set of the referenced path is
// serialised and sent to the client.
func (s *Server) Subscribe(gpb.GNMI_SubscribeServer) error {
	return status.Errorf(codes.Unimplemented, "TODO")
}
<|endoftext|>


<commit_before>package zk

import (
	"strconv"
	"sync"

	"github.com/funkygao/dbus/pkg/cluster"
	log "github.com/funkygao/log4go"
	"github.com/funkygao/zkclient"
)

type leader struct {
	ctx *controller

	lastDecision cluster.Decision

	epoch          int // should never overflow
	epochZkVersion int32

	pcl zkclient.ZkChildListener // leader watches live participants
	rcl zkclient.ZkChildListener // leader watches resources

	rbLockStep sync.Mutex
}

func newLeader(ctx *controller) *leader {
	return &leader{
		ctx: ctx,
		pcl: newParticipantChangeListener(ctx),
		rcl: newResourceChangeListener(ctx),
	}
}

func (l *leader) fetchEpoch() {
	data, stat, err := l.ctx.zc.GetWithStat(l.ctx.kb.leaderEpoch())
	if err != nil {
		if !zkclient.IsErrNoNode(err) {
			log.Error("%v", err)
		}

		return
	}

	l.epoch, _ = strconv.Atoi(string(data))
	l.epochZkVersion = stat.Version
}

func (l *leader) incrementEpoch() (ok bool) {
	newEpoch := l.epoch + 1
	data := []byte(strconv.Itoa(newEpoch))
	// CAS
	newStat, err := l.ctx.zc.SetWithVersion(l.ctx.kb.leaderEpoch(), data, l.epochZkVersion)
	if err != nil {
		switch {
		case zkclient.IsErrNoNode(err):
			// if path doesn't exist, this is the first controller whose epoch should be 1
			// the following call can still fail if another controller gets elected between checking if the path exists and
			// trying to create the controller epoch path
			if err := l.ctx.zc.CreatePersistent(l.ctx.kb.leaderEpoch(), data); err != nil {
				if zkclient.IsErrNodeExists(err) {
					log.Warn("leader moved to another participant! abort rebalance")
					return
				}

				// unexpected zk err
				log.Error("Error while incrementing controller epoch: %v", err)
				return
			}

			// will go to ok

		case zkclient.IsErrVersionConflict(err):
			log.Warn("leader moved to another participant! abort rebalance")
			return

		default:
			// unexpected zk err
			log.Error("Error while incrementing controller epoch: %v", err)
			return
		}
	}

	ok = true
	l.epoch = newEpoch
	l.epochZkVersion = newStat.Version
	return
}

func (l *leader) onResigningAsLeader() {
	l.ctx.zc.UnsubscribeChildChanges(l.ctx.kb.participants(), l.pcl)
	l.ctx.zc.UnsubscribeChildChanges(l.ctx.kb.resources(), l.rcl)

	l.lastDecision = nil
	l.ctx.elector.leaderID = ""
	l.epoch = 0
	l.epochZkVersion = 0

	log.Trace("[%s] resigned as leader", l.ctx.participant)
}

func (l *leader) onBecomingLeader() {
	l.ctx.zc.SubscribeChildChanges(l.ctx.kb.participants(), l.pcl)
	l.ctx.zc.SubscribeChildChanges(l.ctx.kb.resources(), l.rcl)

	l.fetchEpoch()
	if !l.incrementEpoch() {
		return
	}

	log.Trace("[%s] become controller leader and trigger rebalance!", l.ctx.participant)
	l.doRebalance()
}

/// rebalance happens on controller leader when:
// 1. participants change
// 2. resources change
// 3. becoming leader
// rebalance runs sequentially
func (l *leader) doRebalance() {
	l.rbLockStep.Lock()
	defer l.rbLockStep.Unlock()

	liveParticipants, err := l.ctx.LiveParticipants()
	if err != nil {
		// TODO
		log.Critical("[%s] %s", l.ctx.participant, err)
		return
	}
	if len(liveParticipants) == 0 {
		log.Critical("[%s] no live participants found", l.ctx.participant)
		return
	}

	resources, err := l.ctx.RegisteredResources()
	if err != nil {
		// TODO
		log.Critical("[%s] %s", l.ctx.participant, err)
		return
	}

	newDecision := l.ctx.strategyFunc(liveParticipants, resources)
	if !newDecision.Equals(l.lastDecision) {
		l.lastDecision = newDecision

		// WAL
		walFailure := false
		for participant, resources := range newDecision {
			for _, resource := range resources {
				rs := cluster.NewResourceState()
				rs.LeaderEpoch = l.epoch
				rs.Owner = participant.Endpoint
				// TODO add random sleep here to test race condition
				if err := l.ctx.zc.Set(l.ctx.kb.resourceState(resource.Name), rs.Marshal()); err != nil {
					// zk conn lost? timeout?
					// TODO
					log.Critical("[%s] %s %v", l.ctx.participant, resource.Name, err)
					walFailure = true
					break
				}
			}

			if walFailure {
				break
			}
		}

		l.ctx.onRebalance(l.epoch, newDecision)
	} else {
		log.Trace("[%s] decision stay unchanged, quit rebalance", l.ctx.participant)
	}
}
<commit_msg>only when WAL succeed will do real rebalance<commit_after>package zk

import (
	"strconv"
	"sync"

	"github.com/funkygao/dbus/pkg/cluster"
	log "github.com/funkygao/log4go"
	"github.com/funkygao/zkclient"
)

type leader struct {
	ctx *controller

	lastDecision cluster.Decision

	epoch          int // should never overflow
	epochZkVersion int32

	pcl zkclient.ZkChildListener // leader watches live participants
	rcl zkclient.ZkChildListener // leader watches resources

	rbLockStep sync.Mutex
}

func newLeader(ctx *controller) *leader {
	return &leader{
		ctx: ctx,
		pcl: newParticipantChangeListener(ctx),
		rcl: newResourceChangeListener(ctx),
	}
}

func (l *leader) fetchEpoch() {
	data, stat, err := l.ctx.zc.GetWithStat(l.ctx.kb.leaderEpoch())
	if err != nil {
		if !zkclient.IsErrNoNode(err) {
			log.Error("%v", err)
		}

		return
	}

	l.epoch, _ = strconv.Atoi(string(data))
	l.epochZkVersion = stat.Version
}

func (l *leader) incrementEpoch() (ok bool) {
	newEpoch := l.epoch + 1
	data := []byte(strconv.Itoa(newEpoch))
	// CAS
	newStat, err := l.ctx.zc.SetWithVersion(l.ctx.kb.leaderEpoch(), data, l.epochZkVersion)
	if err != nil {
		switch {
		case zkclient.IsErrNoNode(err):
			// if path doesn't exist, this is the first controller whose epoch should be 1
			// the following call can still fail if another controller gets elected between checking if the path exists and
			// trying to create the controller epoch path
			if err := l.ctx.zc.CreatePersistent(l.ctx.kb.leaderEpoch(), data); err != nil {
				if zkclient.IsErrNodeExists(err) {
					log.Warn("leader moved to another participant! abort rebalance")
					return
				}

				// unexpected zk err
				log.Error("Error while incrementing controller epoch: %v", err)
				return
			}

			// will go to ok

		case zkclient.IsErrVersionConflict(err):
			log.Warn("leader moved to another participant! abort rebalance")
			return

		default:
			// unexpected zk err
			log.Error("Error while incrementing controller epoch: %v", err)
			return
		}
	}

	ok = true
	l.epoch = newEpoch
	l.epochZkVersion = newStat.Version
	return
}

func (l *leader) onResigningAsLeader() {
	l.ctx.zc.UnsubscribeChildChanges(l.ctx.kb.participants(), l.pcl)
	l.ctx.zc.UnsubscribeChildChanges(l.ctx.kb.resources(), l.rcl)

	l.lastDecision = nil
	l.ctx.elector.leaderID = ""
	l.epoch = 0
	l.epochZkVersion = 0

	log.Trace("[%s] resigned as leader", l.ctx.participant)
}

func (l *leader) onBecomingLeader() {
	l.ctx.zc.SubscribeChildChanges(l.ctx.kb.participants(), l.pcl)
	l.ctx.zc.SubscribeChildChanges(l.ctx.kb.resources(), l.rcl)

	l.fetchEpoch()
	if !l.incrementEpoch() {
		return
	}

	log.Trace("[%s] become controller leader and trigger rebalance!", l.ctx.participant)
	l.doRebalance()
}

/// rebalance happens on controller leader when:
// 1. participants change
// 2. resources change
// 3. becoming leader
// rebalance runs sequentially
func (l *leader) doRebalance() {
	l.rbLockStep.Lock()
	defer l.rbLockStep.Unlock()

	liveParticipants, err := l.ctx.LiveParticipants()
	if err != nil {
		// TODO
		log.Critical("[%s] %s", l.ctx.participant, err)
		return
	}
	if len(liveParticipants) == 0 {
		log.Critical("[%s] no live participants found", l.ctx.participant)
		return
	}

	resources, err := l.ctx.RegisteredResources()
	if err != nil {
		// TODO
		log.Critical("[%s] %s", l.ctx.participant, err)
		return
	}

	newDecision := l.ctx.strategyFunc(liveParticipants, resources)
	if !newDecision.Equals(l.lastDecision) {
		l.lastDecision = newDecision

		WALok := true
		for participant, resources := range newDecision {
			for _, resource := range resources {
				rs := cluster.NewResourceState()
				rs.LeaderEpoch = l.epoch
				rs.Owner = participant.Endpoint
				// TODO add random sleep here to test race condition
				if err := l.ctx.zc.Set(l.ctx.kb.resourceState(resource.Name), rs.Marshal()); err != nil {
					log.Critical("[%s] %s %v", l.ctx.participant, resource.Name, err)
					WALok = false
					break
				}
			}

			if !WALok {
				break
			}
		}

		if WALok {
			// WAL fails means zk conn got wrong, let listeners wait next event
			l.ctx.onRebalance(l.epoch, newDecision)
		}
	} else {
		log.Trace("[%s] decision stay unchanged, quit rebalance", l.ctx.participant)
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// Package compiler generates sys descriptions of syscalls, types and resources
// from textual descriptions.
package compiler

import (
	"fmt"
	"sort"
	"strconv"
	"strings"

	"github.com/google/syzkaller/pkg/ast"
	"github.com/google/syzkaller/prog"
	"github.com/google/syzkaller/sys/targets"
)

// Overview of compilation process:
// 1. ast.Parse on text file does tokenization and builds AST.
//    This step catches basic syntax errors. AST contains full debug info.
// 2. ExtractConsts as AST returns set of constant identifiers.
//    This step also does verification of include/incdir/define AST nodes.
// 3. User translates constants to values.
// 4. Compile on AST and const values does the rest of the work and returns Prog
//    containing generated prog objects.
// 4.1. assignSyscallNumbers: uses consts to assign syscall numbers.
//      This step also detects unsupported syscalls and discards no longer
//      needed AST nodes (inlcude, define, comments, etc).
// 4.2. patchConsts: patches Int nodes referring to consts with corresponding values.
//      Also detects unsupported syscalls, structs, resources due to missing consts.
// 4.3. check: does extensive semantical checks of AST.
// 4.4. gen: generates prog objects from AST.

// Prog is description compilation result.
type Prog struct {
	Resources   []*prog.ResourceDesc
	Syscalls    []*prog.Syscall
	StructDescs []*prog.KeyedStruct
	// Set of unsupported syscalls/flags.
	Unsupported map[string]bool
	// Returned if consts was nil.
	fileConsts map[string]*ConstInfo
}

func createCompiler(desc *ast.Description, target *targets.Target, eh ast.ErrorHandler) *compiler {
	if eh == nil {
		eh = ast.LoggingHandler
	}
	comp := &compiler{
		desc:         desc,
		target:       target,
		eh:           eh,
		ptrSize:      target.PtrSize,
		unsupported:  make(map[string]bool),
		resources:    make(map[string]*ast.Resource),
		typedefs:     make(map[string]*ast.TypeDef),
		structs:      make(map[string]*ast.Struct),
		intFlags:     make(map[string]*ast.IntFlags),
		strFlags:     make(map[string]*ast.StrFlags),
		used:         make(map[string]bool),
		usedTypedefs: make(map[string]bool),
		structDescs:  make(map[prog.StructKey]*prog.StructDesc),
		structNodes:  make(map[*prog.StructDesc]*ast.Struct),
		structVarlen: make(map[string]bool),
	}
	for name, n := range builtinTypedefs {
		comp.typedefs[name] = n
		comp.usedTypedefs[name] = true
	}
	for name, n := range builtinStrFlags {
		comp.strFlags[name] = n
	}
	return comp
}

// Compile compiles sys description.
func Compile(desc *ast.Description, consts map[string]uint64, target *targets.Target, eh ast.ErrorHandler) *Prog {
	comp := createCompiler(desc.Clone(), target, eh)
	comp.typecheck()
	// The subsequent, more complex, checks expect basic validity of the tree,
	// in particular corrent number of type arguments. If there were errors,
	// don't proceed to avoid out-of-bounds references to type arguments.
	if comp.errors != 0 {
		return nil
	}
	if consts == nil {
		fileConsts := comp.extractConsts()
		if comp.errors != 0 {
			return nil
		}
		return &Prog{fileConsts: fileConsts}
	}
	if comp.target.SyscallNumbers {
		comp.assignSyscallNumbers(consts)
	}
	comp.patchConsts(consts)
	comp.check()
	if comp.errors != 0 {
		return nil
	}
	for _, w := range comp.warnings {
		eh(w.pos, w.msg)
	}
	syscalls := comp.genSyscalls()
	prg := &Prog{
		Resources:   comp.genResources(),
		Syscalls:    syscalls,
		StructDescs: comp.genStructDescs(syscalls),
		Unsupported: comp.unsupported,
	}
	if comp.errors != 0 {
		return nil
	}
	return prg
}

type compiler struct {
	desc     *ast.Description
	target   *targets.Target
	eh       ast.ErrorHandler
	errors   int
	warnings []warn
	ptrSize  uint64

	unsupported  map[string]bool
	resources    map[string]*ast.Resource
	typedefs     map[string]*ast.TypeDef
	structs      map[string]*ast.Struct
	intFlags     map[string]*ast.IntFlags
	strFlags     map[string]*ast.StrFlags
	used         map[string]bool // contains used structs/resources
	usedTypedefs map[string]bool

	structDescs  map[prog.StructKey]*prog.StructDesc
	structNodes  map[*prog.StructDesc]*ast.Struct
	structVarlen map[string]bool
}

type warn struct {
	pos ast.Pos
	msg string
}

func (comp *compiler) error(pos ast.Pos, msg string, args ...interface{}) {
	comp.errors++
	comp.eh(pos, fmt.Sprintf(msg, args...))
}

func (comp *compiler) warning(pos ast.Pos, msg string, args ...interface{}) {
	comp.warnings = append(comp.warnings, warn{pos, fmt.Sprintf(msg, args...)})
}

func (comp *compiler) structIsVarlen(name string) bool {
	if varlen, ok := comp.structVarlen[name]; ok {
		return varlen
	}
	s := comp.structs[name]
	if s.IsUnion {
		if varlen, _ := comp.parseUnionAttrs(s); varlen {
			comp.structVarlen[name] = true
			return true
		}
	}
	comp.structVarlen[name] = false // to not hang on recursive types
	varlen := false
	for _, fld := range s.Fields {
		if comp.isVarlen(fld.Type) {
			varlen = true
			break
		}
	}
	comp.structVarlen[name] = varlen
	return varlen
}

func (comp *compiler) parseUnionAttrs(n *ast.Struct) (varlen bool, size uint64) {
	size = sizeUnassigned
	for _, attr := range n.Attrs {
		switch attr.Ident {
		case "varlen":
			if len(attr.Args) != 0 {
				comp.error(attr.Pos, "%v attribute has args", attr.Ident)
			}
			varlen = true
		case "size":
			size = comp.parseSizeAttr(attr)
		default:
			comp.error(attr.Pos, "unknown union %v attribute %v",
				n.Name.Name, attr.Ident)
		}
	}
	return
}

func (comp *compiler) parseStructAttrs(n *ast.Struct) (packed bool, size, align uint64) {
	size = sizeUnassigned
	for _, attr := range n.Attrs {
		switch {
		case attr.Ident == "packed":
			if len(attr.Args) != 0 {
				comp.error(attr.Pos, "%v attribute has args", attr.Ident)
			}
			packed = true
		case attr.Ident == "align_ptr":
			if len(attr.Args) != 0 {
				comp.error(attr.Pos, "%v attribute has args", attr.Ident)
			}
			align = comp.ptrSize
		case strings.HasPrefix(attr.Ident, "align_"):
			if len(attr.Args) != 0 {
				comp.error(attr.Pos, "%v attribute has args", attr.Ident)
			}
			a, err := strconv.ParseUint(attr.Ident[6:], 10, 64)
			if err != nil {
				comp.error(attr.Pos, "bad struct %v alignment %v",
					n.Name.Name, attr.Ident[6:])
				continue
			}
			if a&(a-1) != 0 || a == 0 || a > 1<<30 {
				comp.error(attr.Pos, "bad struct %v alignment %v (must be a sane power of 2)",
					n.Name.Name, a)
			}
			align = a
		case attr.Ident == "size":
			size = comp.parseSizeAttr(attr)
		default:
			comp.error(attr.Pos, "unknown struct %v attribute %v",
				n.Name.Name, attr.Ident)
		}
	}
	return
}

func (comp *compiler) parseSizeAttr(attr *ast.Type) uint64 {
	if len(attr.Args) != 1 {
		comp.error(attr.Pos, "%v attribute is expected to have 1 argument", attr.Ident)
		return sizeUnassigned
	}
	sz := attr.Args[0]
	if unexpected, _, ok := checkTypeKind(sz, kindInt); !ok {
		comp.error(sz.Pos, "unexpected %v, expect int", unexpected)
		return sizeUnassigned
	}
	if len(sz.Colon) != 0 || len(sz.Args) != 0 {
		comp.error(sz.Pos, "size attribute has colon or args")
		return sizeUnassigned
	}
	return sz.Value
}

func (comp *compiler) getTypeDesc(t *ast.Type) *typeDesc {
	if desc := builtinTypes[t.Ident]; desc != nil {
		return desc
	}
	if comp.resources[t.Ident] != nil {
		return typeResource
	}
	if comp.structs[t.Ident] != nil {
		return typeStruct
	}
	if comp.typedefs[t.Ident] != nil {
		return typeTypedef
	}
	return nil
}

func (comp *compiler) getArgsBase(t *ast.Type, field string, dir prog.Dir, isArg bool) (
	*typeDesc, []*ast.Type, prog.IntTypeCommon) {
	desc := comp.getTypeDesc(t)
	if desc == nil {
		panic(fmt.Sprintf("no type desc for %#v", *t))
	}
	args, opt := removeOpt(t)
	com := genCommon(t.Ident, field, sizeUnassigned, dir, opt != nil)
	base := genIntCommon(com, 0, false)
	if desc.NeedBase {
		base.TypeSize = comp.ptrSize
		if !isArg {
			baseType := args[len(args)-1]
			args = args[:len(args)-1]
			base = typeInt.Gen(comp, baseType, nil, base).(*prog.IntType).IntTypeCommon
		}
	}
	return desc, args, base
}

func (comp *compiler) derefPointers(t *ast.Type) (*ast.Type, *typeDesc) {
	for {
		desc := comp.getTypeDesc(t)
		if desc != typePtr {
			return t, desc
		}
		t = t.Args[1]
	}
}

func (comp *compiler) foreachType(n0 ast.Node,
	cb func(*ast.Type, *typeDesc, []*ast.Type, prog.IntTypeCommon)) {
	switch n := n0.(type) {
	case *ast.Call:
		for _, arg := range n.Args {
			comp.foreachSubType(arg.Type, true, cb)
		}
		if n.Ret != nil {
			comp.foreachSubType(n.Ret, true, cb)
		}
	case *ast.Resource:
		comp.foreachSubType(n.Base, false, cb)
	case *ast.Struct:
		for _, f := range n.Fields {
			comp.foreachSubType(f.Type, false, cb)
		}
	case *ast.TypeDef:
		if len(n.Args) == 0 {
			comp.foreachSubType(n.Type, false, cb)
		}
	default:
		panic(fmt.Sprintf("unexpected node %#v", n0))
	}
}

func (comp *compiler) foreachSubType(t *ast.Type, isArg bool,
	cb func(*ast.Type, *typeDesc, []*ast.Type, prog.IntTypeCommon)) {
	desc, args, base := comp.getArgsBase(t, "", prog.DirIn, isArg)
	cb(t, desc, args, base)
	for i, arg := range args {
		if desc.Args[i].Type == typeArgType {
			comp.foreachSubType(arg, desc.Args[i].IsArg, cb)
		}
	}
}

func removeOpt(t *ast.Type) ([]*ast.Type, *ast.Type) {
	args := t.Args
	if last := len(args) - 1; last >= 0 && args[last].Ident == "opt" {
		return args[:last], args[last]
	}
	return args, nil
}

func (comp *compiler) parseIntType(name string) (size uint64, bigEndian bool) {
	be := strings.HasSuffix(name, "be")
	if be {
		name = name[:len(name)-len("be")]
	}
	size = comp.ptrSize
	if name != "intptr" {
		size, _ = strconv.ParseUint(name[3:], 10, 64)
		size /= 8
	}
	return size, be
}

func toArray(m map[string]bool) []string {
	delete(m, "")
	var res []string
	for v := range m {
		if v != "" {
			res = append(res, v)
		}
	}
	sort.Strings(res)
	return res
}

func arrayContains(a []string, v string) bool {
	for _, s := range a {
		if s == v {
			return true
		}
	}
	return false
}
<commit_msg>pkg/compiler: emit warnings after generate stage<commit_after>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// Package compiler generates sys descriptions of syscalls, types and resources
// from textual descriptions.
package compiler

import (
	"fmt"
	"sort"
	"strconv"
	"strings"

	"github.com/google/syzkaller/pkg/ast"
	"github.com/google/syzkaller/prog"
	"github.com/google/syzkaller/sys/targets"
)

// Overview of compilation process:
// 1. ast.Parse on text file does tokenization and builds AST.
//    This step catches basic syntax errors. AST contains full debug info.
// 2. ExtractConsts as AST returns set of constant identifiers.
//    This step also does verification of include/incdir/define AST nodes.
// 3. User translates constants to values.
// 4. Compile on AST and const values does the rest of the work and returns Prog
//    containing generated prog objects.
// 4.1. assignSyscallNumbers: uses consts to assign syscall numbers.
//      This step also detects unsupported syscalls and discards no longer
//      needed AST nodes (inlcude, define, comments, etc).
// 4.2. patchConsts: patches Int nodes referring to consts with corresponding values.
//      Also detects unsupported syscalls, structs, resources due to missing consts.
// 4.3. check: does extensive semantical checks of AST.
// 4.4. gen: generates prog objects from AST.

// Prog is description compilation result.
type Prog struct {
	Resources   []*prog.ResourceDesc
	Syscalls    []*prog.Syscall
	StructDescs []*prog.KeyedStruct
	// Set of unsupported syscalls/flags.
	Unsupported map[string]bool
	// Returned if consts was nil.
	fileConsts map[string]*ConstInfo
}

func createCompiler(desc *ast.Description, target *targets.Target, eh ast.ErrorHandler) *compiler {
	if eh == nil {
		eh = ast.LoggingHandler
	}
	comp := &compiler{
		desc:         desc,
		target:       target,
		eh:           eh,
		ptrSize:      target.PtrSize,
		unsupported:  make(map[string]bool),
		resources:    make(map[string]*ast.Resource),
		typedefs:     make(map[string]*ast.TypeDef),
		structs:      make(map[string]*ast.Struct),
		intFlags:     make(map[string]*ast.IntFlags),
		strFlags:     make(map[string]*ast.StrFlags),
		used:         make(map[string]bool),
		usedTypedefs: make(map[string]bool),
		structDescs:  make(map[prog.StructKey]*prog.StructDesc),
		structNodes:  make(map[*prog.StructDesc]*ast.Struct),
		structVarlen: make(map[string]bool),
	}
	for name, n := range builtinTypedefs {
		comp.typedefs[name] = n
		comp.usedTypedefs[name] = true
	}
	for name, n := range builtinStrFlags {
		comp.strFlags[name] = n
	}
	return comp
}

// Compile compiles sys description.
func Compile(desc *ast.Description, consts map[string]uint64, target *targets.Target, eh ast.ErrorHandler) *Prog {
	comp := createCompiler(desc.Clone(), target, eh)
	comp.typecheck()
	// The subsequent, more complex, checks expect basic validity of the tree,
	// in particular corrent number of type arguments. If there were errors,
	// don't proceed to avoid out-of-bounds references to type arguments.
	if comp.errors != 0 {
		return nil
	}
	if consts == nil {
		fileConsts := comp.extractConsts()
		if comp.errors != 0 {
			return nil
		}
		return &Prog{fileConsts: fileConsts}
	}
	if comp.target.SyscallNumbers {
		comp.assignSyscallNumbers(consts)
	}
	comp.patchConsts(consts)
	comp.check()
	if comp.errors != 0 {
		return nil
	}
	syscalls := comp.genSyscalls()
	prg := &Prog{
		Resources:   comp.genResources(),
		Syscalls:    syscalls,
		StructDescs: comp.genStructDescs(syscalls),
		Unsupported: comp.unsupported,
	}
	if comp.errors != 0 {
		return nil
	}
	for _, w := range comp.warnings {
		eh(w.pos, w.msg)
	}
	return prg
}

type compiler struct {
	desc     *ast.Description
	target   *targets.Target
	eh       ast.ErrorHandler
	errors   int
	warnings []warn
	ptrSize  uint64

	unsupported  map[string]bool
	resources    map[string]*ast.Resource
	typedefs     map[string]*ast.TypeDef
	structs      map[string]*ast.Struct
	intFlags     map[string]*ast.IntFlags
	strFlags     map[string]*ast.StrFlags
	used         map[string]bool // contains used structs/resources
	usedTypedefs map[string]bool

	structDescs  map[prog.StructKey]*prog.StructDesc
	structNodes  map[*prog.StructDesc]*ast.Struct
	structVarlen map[string]bool
}

type warn struct {
	pos ast.Pos
	msg string
}

func (comp *compiler) error(pos ast.Pos, msg string, args ...interface{}) {
	comp.errors++
	comp.eh(pos, fmt.Sprintf(msg, args...))
}

func (comp *compiler) warning(pos ast.Pos, msg string, args ...interface{}) {
	comp.warnings = append(comp.warnings, warn{pos, fmt.Sprintf(msg, args...)})
}

func (comp *compiler) structIsVarlen(name string) bool {
	if varlen, ok := comp.structVarlen[name]; ok {
		return varlen
	}
	s := comp.structs[name]
	if s.IsUnion {
		if varlen, _ := comp.parseUnionAttrs(s); varlen {
			comp.structVarlen[name] = true
			return true
		}
	}
	comp.structVarlen[name] = false // to not hang on recursive types
	varlen := false
	for _, fld := range s.Fields {
		if comp.isVarlen(fld.Type) {
			varlen = true
			break
		}
	}
	comp.structVarlen[name] = varlen
	return varlen
}

func (comp *compiler) parseUnionAttrs(n *ast.Struct) (varlen bool, size uint64) {
	size = sizeUnassigned
	for _, attr := range n.Attrs {
		switch attr.Ident {
		case "varlen":
			if len(attr.Args) != 0 {
				comp.error(attr.Pos, "%v attribute has args", attr.Ident)
			}
			varlen = true
		case "size":
			size = comp.parseSizeAttr(attr)
		default:
			comp.error(attr.Pos, "unknown union %v attribute %v",
				n.Name.Name, attr.Ident)
		}
	}
	return
}

func (comp *compiler) parseStructAttrs(n *ast.Struct) (packed bool, size, align uint64) {
	size = sizeUnassigned
	for _, attr := range n.Attrs {
		switch {
		case attr.Ident == "packed":
			if len(attr.Args) != 0 {
				comp.error(attr.Pos, "%v attribute has args", attr.Ident)
			}
			packed = true
		case attr.Ident == "align_ptr":
			if len(attr.Args) != 0 {
				comp.error(attr.Pos, "%v attribute has args", attr.Ident)
			}
			align = comp.ptrSize
		case strings.HasPrefix(attr.Ident, "align_"):
			if len(attr.Args) != 0 {
				comp.error(attr.Pos, "%v attribute has args", attr.Ident)
			}
			a, err := strconv.ParseUint(attr.Ident[6:], 10, 64)
			if err != nil {
				comp.error(attr.Pos, "bad struct %v alignment %v",
					n.Name.Name, attr.Ident[6:])
				continue
			}
			if a&(a-1) != 0 || a == 0 || a > 1<<30 {
				comp.error(attr.Pos, "bad struct %v alignment %v (must be a sane power of 2)",
					n.Name.Name, a)
			}
			align = a
		case attr.Ident == "size":
			size = comp.parseSizeAttr(attr)
		default:
			comp.error(attr.Pos, "unknown struct %v attribute %v",
				n.Name.Name, attr.Ident)
		}
	}
	return
}

func (comp *compiler) parseSizeAttr(attr *ast.Type) uint64 {
	if len(attr.Args) != 1 {
		comp.error(attr.Pos, "%v attribute is expected to have 1 argument", attr.Ident)
		return sizeUnassigned
	}
	sz := attr.Args[0]
	if unexpected, _, ok := checkTypeKind(sz, kindInt); !ok {
		comp.error(sz.Pos, "unexpected %v, expect int", unexpected)
		return sizeUnassigned
	}
	if len(sz.Colon) != 0 || len(sz.Args) != 0 {
		comp.error(sz.Pos, "size attribute has colon or args")
		return sizeUnassigned
	}
	return sz.Value
}

func (comp *compiler) getTypeDesc(t *ast.Type) *typeDesc {
	if desc := builtinTypes[t.Ident]; desc != nil {
		return desc
	}
	if comp.resources[t.Ident] != nil {
		return typeResource
	}
	if comp.structs[t.Ident] != nil {
		return typeStruct
	}
	if comp.typedefs[t.Ident] != nil {
		return typeTypedef
	}
	return nil
}

func (comp *compiler) getArgsBase(t *ast.Type, field string, dir prog.Dir, isArg bool) (
	*typeDesc, []*ast.Type, prog.IntTypeCommon) {
	desc := comp.getTypeDesc(t)
	if desc == nil {
		panic(fmt.Sprintf("no type desc for %#v", *t))
	}
	args, opt := removeOpt(t)
	com := genCommon(t.Ident, field, sizeUnassigned, dir, opt != nil)
	base := genIntCommon(com, 0, false)
	if desc.NeedBase {
		base.TypeSize = comp.ptrSize
		if !isArg {
			baseType := args[len(args)-1]
			args = args[:len(args)-1]
			base = typeInt.Gen(comp, baseType, nil, base).(*prog.IntType).IntTypeCommon
		}
	}
	return desc, args, base
}

func (comp *compiler) derefPointers(t *ast.Type) (*ast.Type, *typeDesc) {
	for {
		desc := comp.getTypeDesc(t)
		if desc != typePtr {
			return t, desc
		}
		t = t.Args[1]
	}
}

func (comp *compiler) foreachType(n0 ast.Node,
	cb func(*ast.Type, *typeDesc, []*ast.Type, prog.IntTypeCommon)) {
	switch n := n0.(type) {
	case *ast.Call:
		for _, arg := range n.Args {
			comp.foreachSubType(arg.Type, true, cb)
		}
		if n.Ret != nil {
			comp.foreachSubType(n.Ret, true, cb)
		}
	case *ast.Resource:
		comp.foreachSubType(n.Base, false, cb)
	case *ast.Struct:
		for _, f := range n.Fields {
			comp.foreachSubType(f.Type, false, cb)
		}
	case *ast.TypeDef:
		if len(n.Args) == 0 {
			comp.foreachSubType(n.Type, false, cb)
		}
	default:
		panic(fmt.Sprintf("unexpected node %#v", n0))
	}
}

func (comp *compiler) foreachSubType(t *ast.Type, isArg bool,
	cb func(*ast.Type, *typeDesc, []*ast.Type, prog.IntTypeCommon)) {
	desc, args, base := comp.getArgsBase(t, "", prog.DirIn, isArg)
	cb(t, desc, args, base)
	for i, arg := range args {
		if desc.Args[i].Type == typeArgType {
			comp.foreachSubType(arg, desc.Args[i].IsArg, cb)
		}
	}
}

func removeOpt(t *ast.Type) ([]*ast.Type, *ast.Type) {
	args := t.Args
	if last := len(args) - 1; last >= 0 && args[last].Ident == "opt" {
		return args[:last], args[last]
	}
	return args, nil
}

func (comp *compiler) parseIntType(name string) (size uint64, bigEndian bool) {
	be := strings.HasSuffix(name, "be")
	if be {
		name = name[:len(name)-len("be")]
	}
	size = comp.ptrSize
	if name != "intptr" {
		size, _ = strconv.ParseUint(name[3:], 10, 64)
		size /= 8
	}
	return size, be
}

func toArray(m map[string]bool) []string {
	delete(m, "")
	var res []string
	for v := range m {
		if v != "" {
			res = append(res, v)
		}
	}
	sort.Strings(res)
	return res
}

func arrayContains(a []string, v string) bool {
	for _, s := range a {
		if s == v {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package kubelego

import (
	"errors"
	"fmt"
	"net"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/jetstack/kube-lego/pkg/acme"
	"github.com/jetstack/kube-lego/pkg/ingress"
	"github.com/jetstack/kube-lego/pkg/kubelego_const"
	"github.com/jetstack/kube-lego/pkg/provider/gce"
	"github.com/jetstack/kube-lego/pkg/provider/nginx"
	"github.com/jetstack/kube-lego/pkg/secret"

	log "github.com/Sirupsen/logrus"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/client-go/kubernetes"
	k8sApi "k8s.io/client-go/pkg/api/v1"
)

var _ kubelego.KubeLego = &KubeLego{}

func New(version string) *KubeLego {
	return &KubeLego{
		version:   version,
		stopCh:    make(chan struct{}),
		waitGroup: sync.WaitGroup{},
	}
}

func (kl *KubeLego) Log() *log.Entry {
	loglevel := strings.ToLower(os.Getenv("LEGO_LOG_LEVEL"))
	if len(loglevel) == 0 {
		log.SetLevel(log.InfoLevel)
	} else if loglevel == "debug" {
		log.SetLevel(log.DebugLevel)
	} else if loglevel == "info" {
		log.SetLevel(log.InfoLevel)
	} else if loglevel == "warn" {
		log.SetLevel(log.WarnLevel)
	} else if loglevel == "error" {
		log.SetLevel(log.ErrorLevel)
	} else {
		log.SetLevel(log.InfoLevel)
	}
	return log.WithField("context", "kubelego")
}

func (kl *KubeLego) Stop() {
	kl.Log().Info("shutting things down")
	close(kl.stopCh)
}

func (kl *KubeLego) IngressProvider(name string) (provider kubelego.IngressProvider, err error) {
	provider, ok := kl.legoIngressProvider[name]
	if !ok {
		return nil, fmt.Errorf("Ingress provider '%s' not found", name)
	}
	return
}

func (kl *KubeLego) Init() {
	kl.Log().Infof("kube-lego %s starting", kl.version)

	// handle sigterm correctly
	k := make(chan os.Signal, 1)
	signal.Notify(k, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		s := <-k
		logger := kl.Log().WithField("signal", s.String())
		logger.Debug("received signal")
		kl.Stop()
	}()

	// parse env vars
	err := kl.paramsLego()
	if err != nil {
		kl.Log().Fatal(err)
	}

	// initialising ingress providers
	kl.legoIngressProvider = map[string]kubelego.IngressProvider{
		"gce":   gce.New(kl),
		"nginx": nginx.New(kl),
	}

	// start workers
	kl.WatchReconfigure()

	// intialize kube api
	err = kl.InitKube()
	if err != nil {
		kl.Log().Fatal(err)
	}

	// run acme http server
	myAcme := acme.New(kl)
	go func() {
		kl.waitGroup.Add(1)
		defer kl.waitGroup.Done()
		myAcme.RunServer(kl.stopCh)
	}()
	kl.acmeClient = myAcme

	// run ticker to check certificates periodically
	ticker := time.NewTicker(kl.legoCheckInterval)
	go func() {
		for timestamp := range ticker.C {
			kl.Log().Infof("Periodically check certificates at %s", timestamp)
			kl.requestReconfigure()
		}
	}()

	// watch for ingress controller events
	kl.WatchEvents()

	// wait for stop signal
	<-kl.stopCh
	ticker.Stop()
	kl.Log().Infof("exiting")
	kl.waitGroup.Wait()
}

func (kl *KubeLego) AcmeClient() kubelego.Acme {
	return kl.acmeClient
}

func (kl *KubeLego) KubeClient() *kubernetes.Clientset {
	return kl.kubeClient
}

func (kl *KubeLego) Version() string {
	return kl.version
}

func (kl *KubeLego) LegoHTTPPort() intstr.IntOrString {
	return kl.legoHTTPPort
}

func (kl *KubeLego) LegoURL() string {
	return kl.legoURL
}

func (kl *KubeLego) LegoEmail() string {
	return kl.legoEmail
}

func (kl *KubeLego) LegoNamespace() string {
	return kl.legoNamespace
}

func (kl *KubeLego) LegoPodIP() net.IP {
	return kl.legoPodIP
}

func (kl *KubeLego) LegoDefaultIngressClass() string {
	return kl.legoDefaultIngressClass
}

func (kl *KubeLego) LegoIngressNameNginx() string {
	return kl.legoIngressNameNginx
}
func (kl *KubeLego) LegoSupportedIngressClass() []string {
	return kl.legoSupportedIngressClass
}

func (kl *KubeLego) LegoServiceNameNginx() string {
	return kl.legoServiceNameNginx
}

func (kl *KubeLego) LegoServiceNameGce() string {
	return kl.legoServiceNameGce
}

func (kl *KubeLego) LegoMinimumValidity() time.Duration {
	return kl.legoMinimumValidity
}

func (kl *KubeLego) LegoCheckInterval() time.Duration {
	return kl.legoCheckInterval
}

func (kl *KubeLego) LegoKubeApiURL() string {
	return kl.legoKubeApiURL
}

func (kl *KubeLego) acmeSecret() *secret.Secret {
	return secret.New(kl, kl.LegoNamespace(), kl.legoSecretName)
}

func (kl *KubeLego) AcmeUser() (map[string][]byte, error) {
	s := kl.acmeSecret()
	if !s.Exists() {
		return map[string][]byte{}, fmt.Errorf("no acme user found %s/%s", kl.LegoNamespace(), kl.legoSecretName)
	}
	return s.SecretApi.Data, nil
}

func (kl *KubeLego) SaveAcmeUser(data map[string][]byte) error {
	s := kl.acmeSecret()
	s.SecretApi.Data = data
	return s.Save()
}

// read config parameters from ENV vars
func (kl *KubeLego) paramsLego() error {

	kl.legoEmail = os.Getenv("LEGO_EMAIL")
	if len(kl.legoEmail) == 0 {
		return errors.New("Please provide an email address for cert recovery in LEGO_EMAIL")
	}

	kl.legoPodIP = net.ParseIP(os.Getenv("LEGO_POD_IP"))
	if kl.legoPodIP == nil {
		return errors.New("Please provide the pod's IP via environment variable LEGO_POD_IP using the downward API (http://kubernetes.io/docs/user-guide/downward-api/)")
	}

	kl.legoNamespace = os.Getenv("LEGO_NAMESPACE")
	if len(kl.legoNamespace) == 0 {
		kl.legoNamespace = k8sApi.NamespaceDefault
	}

	kl.legoURL = os.Getenv("LEGO_URL")
	if len(kl.legoURL) == 0 {
		kl.legoURL = "https://acme-staging.api.letsencrypt.org/directory"
	}

	kl.legoSecretName = os.Getenv("LEGO_SECRET_NAME")
	if len(kl.legoSecretName) == 0 {
		kl.legoSecretName = "kube-lego-account"
	}

	kl.legoServiceNameNginx = os.Getenv("LEGO_SERVICE_NAME_NGINX")
	if len(kl.legoServiceNameNginx) == 0 {
		kl.legoServiceNameNginx = os.Getenv("LEGO_SERVICE_NAME")
		if len(kl.legoServiceNameNginx) == 0 {
			kl.legoServiceNameNginx = "kube-lego-nginx"
		}
	}

	kl.legoServiceNameGce = os.Getenv("LEGO_SERVICE_NAME_GCE")
	if len(kl.legoServiceNameGce) == 0 {
		kl.legoServiceNameGce = "kube-lego-gce"
	}

	kl.legoSupportedIngressClass = strings.Split(os.Getenv("LEGO_SUPPORTED_INGRESS_CLASS"),",")
	if len(kl.legoSupportedIngressClass) == 1 {
		kl.legoSupportedIngressClass = kubelego.SupportedIngressClasses
	}

	legoDefaultIngressClass := os.Getenv("LEGO_DEFAULT_INGRESS_CLASS")
	if len(legoDefaultIngressClass) == 0 {
		kl.legoDefaultIngressClass = "nginx"
	} else {
		var err error = nil
		kl.legoDefaultIngressClass, err = ingress.IsSupportedIngressClass(kl.legoSupportedIngressClass, legoDefaultIngressClass)
		if err != nil {
			return fmt.Errorf("Unsupported default ingress class: '%s'", legoDefaultIngressClass)
		}
	}
	kl.legoIngressNameNginx = os.Getenv("LEGO_INGRESS_NAME_NGINX")
	if len(kl.legoIngressNameNginx) == 0 {
		kl.legoIngressNameNginx = os.Getenv("LEGO_INGRESS_NAME")
		if len(kl.legoIngressNameNginx) == 0 {
			kl.legoIngressNameNginx = "kube-lego-nginx"
		}
	}

	checkIntervalString := os.Getenv("LEGO_CHECK_INTERVAL")
	if len(checkIntervalString) == 0 {
		kl.legoCheckInterval = 8 * time.Hour
	} else {
		d, err := time.ParseDuration(checkIntervalString)
		if err != nil {
			return err
		}
		if d < 5*time.Minute {
			return fmt.Errorf("Minimum check interval is 5 minutes: %s", d)
		}
		kl.legoCheckInterval = d
	}

	kl.legoKubeApiURL = os.Getenv("LEGO_KUBE_API_URL")
	if len(kl.legoKubeApiURL) == 0 {
		kl.legoKubeApiURL = "http://127.0.0.1:8080"
	}

	minimumValidity := os.Getenv("LEGO_MINIMUM_VALIDITY")
	if len(minimumValidity) == 0 {
		kl.legoMinimumValidity = time.Hour * 24 * 30
	} else {
		d, err := time.ParseDuration(minimumValidity)
		if err != nil {
			return err
		}
		if d < 24*time.Hour {
			return fmt.Errorf("Smallest allowed minimum validity is 24 hours: %s", d)
		}
		kl.legoMinimumValidity = d
	}

	httpPortStr := os.Getenv("LEGO_PORT")
	if len(httpPortStr) == 0 {
		kl.legoHTTPPort = intstr.FromInt(8080)
	} else {
		i, err := strconv.Atoi(httpPortStr)
		if err != nil {
			return err
		}
		if i <= 0 || i >= 65535 {
			return fmt.Errorf("Wrong port: %d", i)
		}
		kl.legoHTTPPort = intstr.FromInt(i)
	}

	annotationEnabled := os.Getenv("LEGO_KUBE_ANNOTATION")
	if len(annotationEnabled) == 0 {
		kubelego.AnnotationEnabled = annotationEnabled
	}

	return nil
}
<commit_msg>fix logic typo<commit_after>package kubelego

import (
	"errors"
	"fmt"
	"net"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/jetstack/kube-lego/pkg/acme"
	"github.com/jetstack/kube-lego/pkg/ingress"
	"github.com/jetstack/kube-lego/pkg/kubelego_const"
	"github.com/jetstack/kube-lego/pkg/provider/gce"
	"github.com/jetstack/kube-lego/pkg/provider/nginx"
	"github.com/jetstack/kube-lego/pkg/secret"

	log "github.com/Sirupsen/logrus"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/client-go/kubernetes"
	k8sApi "k8s.io/client-go/pkg/api/v1"
)

var _ kubelego.KubeLego = &KubeLego{}

func New(version string) *KubeLego {
	return &KubeLego{
		version:   version,
		stopCh:    make(chan struct{}),
		waitGroup: sync.WaitGroup{},
	}
}

func (kl *KubeLego) Log() *log.Entry {
	loglevel := strings.ToLower(os.Getenv("LEGO_LOG_LEVEL"))
	if len(loglevel) == 0 {
		log.SetLevel(log.InfoLevel)
	} else if loglevel == "debug" {
		log.SetLevel(log.DebugLevel)
	} else if loglevel == "info" {
		log.SetLevel(log.InfoLevel)
	} else if loglevel == "warn" {
		log.SetLevel(log.WarnLevel)
	} else if loglevel == "error" {
		log.SetLevel(log.ErrorLevel)
	} else {
		log.SetLevel(log.InfoLevel)
	}
	return log.WithField("context", "kubelego")
}

func (kl *KubeLego) Stop() {
	kl.Log().Info("shutting things down")
	close(kl.stopCh)
}

func (kl *KubeLego) IngressProvider(name string) (provider kubelego.IngressProvider, err error) {
	provider, ok := kl.legoIngressProvider[name]
	if !ok {
		return nil, fmt.Errorf("Ingress provider '%s' not found", name)
	}
	return
}

func (kl *KubeLego) Init() {
	kl.Log().Infof("kube-lego %s starting", kl.version)

	// handle sigterm correctly
	k := make(chan os.Signal, 1)
	signal.Notify(k, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		s := <-k
		logger := kl.Log().WithField("signal", s.String())
		logger.Debug("received signal")
		kl.Stop()
	}()

	// parse env vars
	err := kl.paramsLego()
	if err != nil {
		kl.Log().Fatal(err)
	}

	// initialising ingress providers
	kl.legoIngressProvider = map[string]kubelego.IngressProvider{
		"gce":   gce.New(kl),
		"nginx": nginx.New(kl),
	}

	// start workers
	kl.WatchReconfigure()

	// intialize kube api
	err = kl.InitKube()
	if err != nil {
		kl.Log().Fatal(err)
	}

	// run acme http server
	myAcme := acme.New(kl)
	go func() {
		kl.waitGroup.Add(1)
		defer kl.waitGroup.Done()
		myAcme.RunServer(kl.stopCh)
	}()
	kl.acmeClient = myAcme

	// run ticker to check certificates periodically
	ticker := time.NewTicker(kl.legoCheckInterval)
	go func() {
		for timestamp := range ticker.C {
			kl.Log().Infof("Periodically check certificates at %s", timestamp)
			kl.requestReconfigure()
		}
	}()

	// watch for ingress controller events
	kl.WatchEvents()

	// wait for stop signal
	<-kl.stopCh
	ticker.Stop()
	kl.Log().Infof("exiting")
	kl.waitGroup.Wait()
}

func (kl *KubeLego) AcmeClient() kubelego.Acme {
	return kl.acmeClient
}

func (kl *KubeLego) KubeClient() *kubernetes.Clientset {
	return kl.kubeClient
}

func (kl *KubeLego) Version() string {
	return kl.version
}

func (kl *KubeLego) LegoHTTPPort() intstr.IntOrString {
	return kl.legoHTTPPort
}

func (kl *KubeLego) LegoURL() string {
	return kl.legoURL
}

func (kl *KubeLego) LegoEmail() string {
	return kl.legoEmail
}

func (kl *KubeLego) LegoNamespace() string {
	return kl.legoNamespace
}

func (kl *KubeLego) LegoPodIP() net.IP {
	return kl.legoPodIP
}

func (kl *KubeLego) LegoDefaultIngressClass() string {
	return kl.legoDefaultIngressClass
}

func (kl *KubeLego) LegoIngressNameNginx() string {
	return kl.legoIngressNameNginx
}
func (kl *KubeLego) LegoSupportedIngressClass() []string {
	return kl.legoSupportedIngressClass
}

func (kl *KubeLego) LegoServiceNameNginx() string {
	return kl.legoServiceNameNginx
}

func (kl *KubeLego) LegoServiceNameGce() string {
	return kl.legoServiceNameGce
}

func (kl *KubeLego) LegoMinimumValidity() time.Duration {
	return kl.legoMinimumValidity
}

func (kl *KubeLego) LegoCheckInterval() time.Duration {
	return kl.legoCheckInterval
}

func (kl *KubeLego) LegoKubeApiURL() string {
	return kl.legoKubeApiURL
}

func (kl *KubeLego) acmeSecret() *secret.Secret {
	return secret.New(kl, kl.LegoNamespace(), kl.legoSecretName)
}

func (kl *KubeLego) AcmeUser() (map[string][]byte, error) {
	s := kl.acmeSecret()
	if !s.Exists() {
		return map[string][]byte{}, fmt.Errorf("no acme user found %s/%s", kl.LegoNamespace(), kl.legoSecretName)
	}
	return s.SecretApi.Data, nil
}

func (kl *KubeLego) SaveAcmeUser(data map[string][]byte) error {
	s := kl.acmeSecret()
	s.SecretApi.Data = data
	return s.Save()
}

// read config parameters from ENV vars
func (kl *KubeLego) paramsLego() error {

	kl.legoEmail = os.Getenv("LEGO_EMAIL")
	if len(kl.legoEmail) == 0 {
		return errors.New("Please provide an email address for cert recovery in LEGO_EMAIL")
	}

	kl.legoPodIP = net.ParseIP(os.Getenv("LEGO_POD_IP"))
	if kl.legoPodIP == nil {
		return errors.New("Please provide the pod's IP via environment variable LEGO_POD_IP using the downward API (http://kubernetes.io/docs/user-guide/downward-api/)")
	}

	kl.legoNamespace = os.Getenv("LEGO_NAMESPACE")
	if len(kl.legoNamespace) == 0 {
		kl.legoNamespace = k8sApi.NamespaceDefault
	}

	kl.legoURL = os.Getenv("LEGO_URL")
	if len(kl.legoURL) == 0 {
		kl.legoURL = "https://acme-staging.api.letsencrypt.org/directory"
	}

	kl.legoSecretName = os.Getenv("LEGO_SECRET_NAME")
	if len(kl.legoSecretName) == 0 {
		kl.legoSecretName = "kube-lego-account"
	}

	kl.legoServiceNameNginx = os.Getenv("LEGO_SERVICE_NAME_NGINX")
	if len(kl.legoServiceNameNginx) == 0 {
		kl.legoServiceNameNginx = os.Getenv("LEGO_SERVICE_NAME")
		if len(kl.legoServiceNameNginx) == 0 {
			kl.legoServiceNameNginx = "kube-lego-nginx"
		}
	}

	kl.legoServiceNameGce = os.Getenv("LEGO_SERVICE_NAME_GCE")
	if len(kl.legoServiceNameGce) == 0 {
		kl.legoServiceNameGce = "kube-lego-gce"
	}

	kl.legoSupportedIngressClass = strings.Split(os.Getenv("LEGO_SUPPORTED_INGRESS_CLASS"),",")
	if len(kl.legoSupportedIngressClass) == 1 {
		kl.legoSupportedIngressClass = kubelego.SupportedIngressClasses
	}

	legoDefaultIngressClass := os.Getenv("LEGO_DEFAULT_INGRESS_CLASS")
	if len(legoDefaultIngressClass) == 0 {
		kl.legoDefaultIngressClass = "nginx"
	} else {
		var err error = nil
		kl.legoDefaultIngressClass, err = ingress.IsSupportedIngressClass(kl.legoSupportedIngressClass, legoDefaultIngressClass)
		if err != nil {
			return fmt.Errorf("Unsupported default ingress class: '%s'", legoDefaultIngressClass)
		}
	}
	kl.legoIngressNameNginx = os.Getenv("LEGO_INGRESS_NAME_NGINX")
	if len(kl.legoIngressNameNginx) == 0 {
		kl.legoIngressNameNginx = os.Getenv("LEGO_INGRESS_NAME")
		if len(kl.legoIngressNameNginx) == 0 {
			kl.legoIngressNameNginx = "kube-lego-nginx"
		}
	}

	checkIntervalString := os.Getenv("LEGO_CHECK_INTERVAL")
	if len(checkIntervalString) == 0 {
		kl.legoCheckInterval = 8 * time.Hour
	} else {
		d, err := time.ParseDuration(checkIntervalString)
		if err != nil {
			return err
		}
		if d < 5*time.Minute {
			return fmt.Errorf("Minimum check interval is 5 minutes: %s", d)
		}
		kl.legoCheckInterval = d
	}

	kl.legoKubeApiURL = os.Getenv("LEGO_KUBE_API_URL")
	if len(kl.legoKubeApiURL) == 0 {
		kl.legoKubeApiURL = "http://127.0.0.1:8080"
	}

	minimumValidity := os.Getenv("LEGO_MINIMUM_VALIDITY")
	if len(minimumValidity) == 0 {
		kl.legoMinimumValidity = time.Hour * 24 * 30
	} else {
		d, err := time.ParseDuration(minimumValidity)
		if err != nil {
			return err
		}
		if d < 24*time.Hour {
			return fmt.Errorf("Smallest allowed minimum validity is 24 hours: %s", d)
		}
		kl.legoMinimumValidity = d
	}

	httpPortStr := os.Getenv("LEGO_PORT")
	if len(httpPortStr) == 0 {
		kl.legoHTTPPort = intstr.FromInt(8080)
	} else {
		i, err := strconv.Atoi(httpPortStr)
		if err != nil {
			return err
		}
		if i <= 0 || i >= 65535 {
			return fmt.Errorf("Wrong port: %d", i)
		}
		kl.legoHTTPPort = intstr.FromInt(i)
	}

	annotationEnabled := os.Getenv("LEGO_KUBE_ANNOTATION")
	if len(annotationEnabled) != 0 {
		kubelego.AnnotationEnabled = annotationEnabled
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package playground registers an HTTP handler at "/compile" that
// proxies requests to the golang.org playground service.
package playground

import (
	"bytes"
	"fmt"
	"io"
	"net/http"
)

const runURL = "http://golang.org/compile"

func init() {
	http.HandleFunc("/compile", compile)
}

func compile(w http.ResponseWriter, r *http.Request) {
	b := new(bytes.Buffer)
	if err := passThru(b, r); err != nil {
		http.Error(w, "Compile server error.", http.StatusInternalServerError)
		report(r, err)
		return
	}
	io.Copy(w, b)
}

func passThru(w io.Writer, req *http.Request) error {
	defer req.Body.Close()
	r, err := client(req).Post(runURL, req.Header.Get("Content-type"), req.Body)
	if err != nil {
		return fmt.Errorf("making POST request: %v", err)
	}
	defer r.Body.Close()
	if _, err := io.Copy(w, r.Body); err != nil {
		return fmt.Errorf("copying response Body: %v", err)
	}
	return nil
}
<commit_msg>go.talks/pkg/playground: proxy share handler too<commit_after>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package playground registers an HTTP handler at "/compile" that
// proxies requests to the golang.org playground service.
package playground

import (
	"bytes"
	"fmt"
	"io"
	"net/http"
)

const baseURL = "http://play.golang.org"

func init() {
	http.HandleFunc("/compile", bounce)
	http.HandleFunc("/share", bounce)
}

func bounce(w http.ResponseWriter, r *http.Request) {
	b := new(bytes.Buffer)
	if err := passThru(b, r); err != nil {
		http.Error(w, "Server error.", http.StatusInternalServerError)
		report(r, err)
		return
	}
	io.Copy(w, b)
}

func passThru(w io.Writer, req *http.Request) error {
	defer req.Body.Close()
	url := baseURL + req.URL.Path
	r, err := client(req).Post(url, req.Header.Get("Content-type"), req.Body)
	if err != nil {
		return fmt.Errorf("making POST request: %v", err)
	}
	defer r.Body.Close()
	if _, err := io.Copy(w, r.Body); err != nil {
		return fmt.Errorf("copying response Body: %v", err)
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package aws

import (
	"fmt"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"k8s.io/klog/v2"

	"k8s.io/apimachinery/pkg/util/wait"
)

// TagNameKubernetesClusterPrefix is the tag name we use to differentiate multiple
// logically independent clusters running in the same AZ.
// The tag key = TagNameKubernetesClusterPrefix + clusterID
// The tag value is an ownership value
const TagNameKubernetesClusterPrefix = "kubernetes.io/cluster/"

// TagNameKubernetesClusterLegacy is the legacy tag name we use to differentiate multiple
// logically independent clusters running in the same AZ.  The problem with it was that it
// did not allow shared resources.
const TagNameKubernetesClusterLegacy = "KubernetesCluster"

// ResourceLifecycle is the cluster lifecycle state used in tagging
type ResourceLifecycle string

const (
	// ResourceLifecycleOwned is the value we use when tagging resources to indicate
	// that the resource is considered owned and managed by the cluster,
	// and in particular that the lifecycle is tied to the lifecycle of the cluster.
	ResourceLifecycleOwned = "owned"
	// ResourceLifecycleShared is the value we use when tagging resources to indicate
	// that the resource is shared between multiple clusters, and should not be destroyed
	// if the cluster is destroyed.
	ResourceLifecycleShared = "shared"
)

type awsTagging struct {
	// ClusterID is our cluster identifier: we tag AWS resources with this value,
	// and thus we can run two independent clusters in the same VPC or subnets.
	// This gives us similar functionality to GCE projects.
	ClusterID string

	// usesLegacyTags is true if we are using the legacy TagNameKubernetesClusterLegacy tags
	usesLegacyTags bool
}

func (t *awsTagging) init(legacyClusterID string, clusterID string) error {
	if legacyClusterID != "" {
		if clusterID != "" && legacyClusterID != clusterID {
			return fmt.Errorf("clusterID tags did not match: %q vs %q", clusterID, legacyClusterID)
		}
		t.usesLegacyTags = true
		clusterID = legacyClusterID
	}

	t.ClusterID = clusterID

	if clusterID != "" {
		klog.Infof("AWS cloud filtering on ClusterID: %v", clusterID)
	} else {
		return fmt.Errorf("AWS cloud failed to find ClusterID")
	}

	return nil
}

// Extracts a clusterID from the given tags, if one is present
// If no clusterID is found, returns "", nil
// If multiple (different) clusterIDs are found, returns an error
func (t *awsTagging) initFromTags(tags []*ec2.Tag) error {
	legacyClusterID, newClusterID, err := findClusterIDs(tags)
	if err != nil {
		return err
	}

	if legacyClusterID == "" && newClusterID == "" {
		klog.Errorf("Tag %q nor %q not found; Kubernetes may behave unexpectedly.", TagNameKubernetesClusterLegacy, TagNameKubernetesClusterPrefix+"...")
	}

	return t.init(legacyClusterID, newClusterID)
}

// Extracts the legacy & new cluster ids from the given tags, if they are present
// If duplicate tags are found, returns an error
func findClusterIDs(tags []*ec2.Tag) (string, string, error) {
	legacyClusterID := ""
	newClusterID := ""

	for _, tag := range tags {
		tagKey := aws.StringValue(tag.Key)
		if strings.HasPrefix(tagKey, TagNameKubernetesClusterPrefix) {
			id := strings.TrimPrefix(tagKey, TagNameKubernetesClusterPrefix)
			if newClusterID != "" {
				return "", "", fmt.Errorf("Found multiple cluster tags with prefix %s (%q and %q)", TagNameKubernetesClusterPrefix, newClusterID, id)
			}
			newClusterID = id
		}

		if tagKey == TagNameKubernetesClusterLegacy {
			id := aws.StringValue(tag.Value)
			if legacyClusterID != "" {
				return "", "", fmt.Errorf("Found multiple %s tags (%q and %q)", TagNameKubernetesClusterLegacy, legacyClusterID, id)
			}
			legacyClusterID = id
		}
	}

	return legacyClusterID, newClusterID, nil
}

func (t *awsTagging) clusterTagKey() string {
	return TagNameKubernetesClusterPrefix + t.ClusterID
}

func (t *awsTagging) hasClusterTag(tags []*ec2.Tag) bool {
	// if the clusterID is not configured -- we consider all instances.
	if len(t.ClusterID) == 0 {
		return true
	}
	clusterTagKey := t.clusterTagKey()
	for _, tag := range tags {
		tagKey := aws.StringValue(tag.Key)
		// For 1.6, we continue to recognize the legacy tags, for the 1.5 -> 1.6 upgrade
		// Note that we want to continue traversing tag list if we see a legacy tag with value != ClusterID
		if (tagKey == TagNameKubernetesClusterLegacy) && (aws.StringValue(tag.Value) == t.ClusterID) {
			return true
		}
		if tagKey == clusterTagKey {
			return true
		}
	}
	return false
}

func (t *awsTagging) hasNoClusterPrefixTag(tags []*ec2.Tag) bool {
	for _, tag := range tags {
		if strings.HasPrefix(aws.StringValue(tag.Key), TagNameKubernetesClusterPrefix) {
			return false
		}
	}
	return true
}

// Ensure that a resource has the correct tags
// If it has no tags, we assume that this was a problem caused by an error in between creation and tagging,
// and we add the tags.  If it has a different cluster's tags, that is an error.
func (t *awsTagging) readRepairClusterTags(client EC2, resourceID string, lifecycle ResourceLifecycle, additionalTags map[string]string, observedTags []*ec2.Tag) error {
	actualTagMap := make(map[string]string)
	for _, tag := range observedTags {
		actualTagMap[aws.StringValue(tag.Key)] = aws.StringValue(tag.Value)
	}

	expectedTags := t.buildTags(lifecycle, additionalTags)

	addTags := make(map[string]string)
	for k, expected := range expectedTags {
		actual := actualTagMap[k]
		if actual == expected {
			continue
		}
		if actual == "" {
			klog.Warningf("Resource %q was missing expected cluster tag %q.  Will add (with value %q)", resourceID, k, expected)
			addTags[k] = expected
		} else {
			return fmt.Errorf("resource %q has tag belonging to another cluster: %q=%q (expected %q)", resourceID, k, actual, expected)
		}
	}

	if len(addTags) == 0 {
		return nil
	}

	if err := t.createTags(client, resourceID, lifecycle, addTags); err != nil {
		return fmt.Errorf("error adding missing tags to resource %q: %q", resourceID, err)
	}

	return nil
}

// createTags calls EC2 CreateTags, but adds retry-on-failure logic
// We retry mainly because if we create an object, we cannot tag it until it is "fully created" (eventual consistency)
// The error code varies though (depending on what we are tagging), so we simply retry on all errors
func (t *awsTagging) createTags(client EC2, resourceID string, lifecycle ResourceLifecycle, additionalTags map[string]string) error {
	tags := t.buildTags(lifecycle, additionalTags)

	if tags == nil || len(tags) == 0 {
		return nil
	}

	var awsTags []*ec2.Tag
	for k, v := range tags {
		tag := &ec2.Tag{
			Key:   aws.String(k),
			Value: aws.String(v),
		}
		awsTags = append(awsTags, tag)
	}

	backoff := wait.Backoff{
		Duration: createTagInitialDelay,
		Factor:   createTagFactor,
		Steps:    createTagSteps,
	}
	request := &ec2.CreateTagsInput{}
	request.Resources = []*string{&resourceID}
	request.Tags = awsTags

	var lastErr error
	err := wait.ExponentialBackoff(backoff, func() (bool, error) {
		_, err := client.CreateTags(request)
		if err == nil {
			return true, nil
		}

		// We could check that the error is retryable, but the error code changes based on what we are tagging
		// SecurityGroup: InvalidGroup.NotFound
		klog.V(2).Infof("Failed to create tags; will retry.  Error was %q", err)
		lastErr = err
		return false, nil
	})
	if err == wait.ErrWaitTimeout {
		// return real CreateTags error instead of timeout
		err = lastErr
	}
	return err
}

// Add additional filters, to match on our tags
// This lets us run multiple k8s clusters in a single EC2 AZ
func (t *awsTagging) addFilters(filters []*ec2.Filter) []*ec2.Filter {
	// if there are no clusterID configured - no filtering by special tag names
	// should be applied to revert to legacy behaviour.
	if len(t.ClusterID) == 0 {
		if len(filters) == 0 {
			// We can't pass a zero-length Filters to AWS (it's an error)
			// So if we end up with no filters; just return nil
			return nil
		}
		return filters
	}

	f := newEc2Filter("tag-key", t.clusterTagKey())
	filters = append(filters, f)
	return filters
}

// Add additional filters, to match on our tags. This uses the tag for legacy
// 1.5 -> 1.6 clusters and exists for backwards compatibility
//
// This lets us run multiple k8s clusters in a single EC2 AZ
func (t *awsTagging) addLegacyFilters(filters []*ec2.Filter) []*ec2.Filter {
	// if there are no clusterID configured - no filtering by special tag names
	// should be applied to revert to legacy behaviour.
	if len(t.ClusterID) == 0 {
		if len(filters) == 0 {
			// We can't pass a zero-length Filters to AWS (it's an error)
			// So if we end up with no filters; just return nil
			return nil
		}
		return filters
	}

	f := newEc2Filter(fmt.Sprintf("tag:%s", TagNameKubernetesClusterLegacy), t.ClusterID)

	// We can't pass a zero-length Filters to AWS (it's an error)
	// So if we end up with no filters; we need to return nil
	filters = append(filters, f)
	return filters
}

func (t *awsTagging) buildTags(lifecycle ResourceLifecycle, additionalTags map[string]string) map[string]string {
	tags := make(map[string]string)
	for k, v := range additionalTags {
		tags[k] = v
	}

	// no clusterID is a sign of misconfigured cluster, but we can't be tagging the resources with empty
	// strings
	if len(t.ClusterID) == 0 {
		return tags
	}

	// We only create legacy tags if we are using legacy tags, i.e. if we have seen a legacy tag on our instance
	if t.usesLegacyTags {
		tags[TagNameKubernetesClusterLegacy] = t.ClusterID
	}
	tags[t.clusterTagKey()] = string(lifecycle)

	return tags
}

func (t *awsTagging) clusterID() string {
	return t.ClusterID
}

func (c *Cloud) TagResource(resourceId string, tags map[string]string) error {
	request := &ec2.CreateTagsInput{
		Resources: []*string{aws.String(resourceId)},
		Tags:      buildAwsTags(tags),
	}

	output, err := c.ec2.CreateTags(request)

	if err != nil {
		klog.Errorf("Error occurred trying to tag resources, %v", err)
		return err
	}

	klog.Infof("Done calling create-tags to EC2: %v", output)

	return nil
}

func (c *Cloud) UntagResource(resourceId string, tags map[string]string) error {
	request := &ec2.DeleteTagsInput{
		Resources: []*string{aws.String(resourceId)},
		Tags:      buildAwsTags(tags),
	}

	output, err := c.ec2.DeleteTags(request)

	if err != nil {
		klog.Errorf("Error occurred trying to untag resources, %v", err)
		return err
	}

	klog.Infof("Done calling delete-tags to EC2: %v", output)

	return nil
}

func buildAwsTags(tags map[string]string) []*ec2.Tag {
	var awsTags []*ec2.Tag
	for k, v := range tags {
		newTag := &ec2.Tag{
			Key:   aws.String(k),
			Value: aws.String(v),
		}
		awsTags = append(awsTags, newTag)
	}

	return awsTags
}
<commit_msg>addressed verify-lint errors<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package aws

import (
	"fmt"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"k8s.io/klog/v2"

	"k8s.io/apimachinery/pkg/util/wait"
)

// TagNameKubernetesClusterPrefix is the tag name we use to differentiate multiple
// logically independent clusters running in the same AZ.
// The tag key = TagNameKubernetesClusterPrefix + clusterID
// The tag value is an ownership value
const TagNameKubernetesClusterPrefix = "kubernetes.io/cluster/"

// TagNameKubernetesClusterLegacy is the legacy tag name we use to differentiate multiple
// logically independent clusters running in the same AZ.  The problem with it was that it
// did not allow shared resources.
const TagNameKubernetesClusterLegacy = "KubernetesCluster"

// ResourceLifecycle is the cluster lifecycle state used in tagging
type ResourceLifecycle string

const (
	// ResourceLifecycleOwned is the value we use when tagging resources to indicate
	// that the resource is considered owned and managed by the cluster,
	// and in particular that the lifecycle is tied to the lifecycle of the cluster.
	ResourceLifecycleOwned = "owned"
	// ResourceLifecycleShared is the value we use when tagging resources to indicate
	// that the resource is shared between multiple clusters, and should not be destroyed
	// if the cluster is destroyed.
	ResourceLifecycleShared = "shared"
)

type awsTagging struct {
	// ClusterID is our cluster identifier: we tag AWS resources with this value,
	// and thus we can run two independent clusters in the same VPC or subnets.
	// This gives us similar functionality to GCE projects.
	ClusterID string

	// usesLegacyTags is true if we are using the legacy TagNameKubernetesClusterLegacy tags
	usesLegacyTags bool
}

func (t *awsTagging) init(legacyClusterID string, clusterID string) error {
	if legacyClusterID != "" {
		if clusterID != "" && legacyClusterID != clusterID {
			return fmt.Errorf("clusterID tags did not match: %q vs %q", clusterID, legacyClusterID)
		}
		t.usesLegacyTags = true
		clusterID = legacyClusterID
	}

	t.ClusterID = clusterID

	if clusterID != "" {
		klog.Infof("AWS cloud filtering on ClusterID: %v", clusterID)
	} else {
		return fmt.Errorf("AWS cloud failed to find ClusterID")
	}

	return nil
}

// Extracts a clusterID from the given tags, if one is present
// If no clusterID is found, returns "", nil
// If multiple (different) clusterIDs are found, returns an error
func (t *awsTagging) initFromTags(tags []*ec2.Tag) error {
	legacyClusterID, newClusterID, err := findClusterIDs(tags)
	if err != nil {
		return err
	}

	if legacyClusterID == "" && newClusterID == "" {
		klog.Errorf("Tag %q nor %q not found; Kubernetes may behave unexpectedly.", TagNameKubernetesClusterLegacy, TagNameKubernetesClusterPrefix+"...")
	}

	return t.init(legacyClusterID, newClusterID)
}

// Extracts the legacy & new cluster ids from the given tags, if they are present
// If duplicate tags are found, returns an error
func findClusterIDs(tags []*ec2.Tag) (string, string, error) {
	legacyClusterID := ""
	newClusterID := ""

	for _, tag := range tags {
		tagKey := aws.StringValue(tag.Key)
		if strings.HasPrefix(tagKey, TagNameKubernetesClusterPrefix) {
			id := strings.TrimPrefix(tagKey, TagNameKubernetesClusterPrefix)
			if newClusterID != "" {
				return "", "", fmt.Errorf("Found multiple cluster tags with prefix %s (%q and %q)", TagNameKubernetesClusterPrefix, newClusterID, id)
			}
			newClusterID = id
		}

		if tagKey == TagNameKubernetesClusterLegacy {
			id := aws.StringValue(tag.Value)
			if legacyClusterID != "" {
				return "", "", fmt.Errorf("Found multiple %s tags (%q and %q)", TagNameKubernetesClusterLegacy, legacyClusterID, id)
			}
			legacyClusterID = id
		}
	}

	return legacyClusterID, newClusterID, nil
}

func (t *awsTagging) clusterTagKey() string {
	return TagNameKubernetesClusterPrefix + t.ClusterID
}

func (t *awsTagging) hasClusterTag(tags []*ec2.Tag) bool {
	// if the clusterID is not configured -- we consider all instances.
	if len(t.ClusterID) == 0 {
		return true
	}
	clusterTagKey := t.clusterTagKey()
	for _, tag := range tags {
		tagKey := aws.StringValue(tag.Key)
		// For 1.6, we continue to recognize the legacy tags, for the 1.5 -> 1.6 upgrade
		// Note that we want to continue traversing tag list if we see a legacy tag with value != ClusterID
		if (tagKey == TagNameKubernetesClusterLegacy) && (aws.StringValue(tag.Value) == t.ClusterID) {
			return true
		}
		if tagKey == clusterTagKey {
			return true
		}
	}
	return false
}

func (t *awsTagging) hasNoClusterPrefixTag(tags []*ec2.Tag) bool {
	for _, tag := range tags {
		if strings.HasPrefix(aws.StringValue(tag.Key), TagNameKubernetesClusterPrefix) {
			return false
		}
	}
	return true
}

// Ensure that a resource has the correct tags
// If it has no tags, we assume that this was a problem caused by an error in between creation and tagging,
// and we add the tags.  If it has a different cluster's tags, that is an error.
func (t *awsTagging) readRepairClusterTags(client EC2, resourceID string, lifecycle ResourceLifecycle, additionalTags map[string]string, observedTags []*ec2.Tag) error {
	actualTagMap := make(map[string]string)
	for _, tag := range observedTags {
		actualTagMap[aws.StringValue(tag.Key)] = aws.StringValue(tag.Value)
	}

	expectedTags := t.buildTags(lifecycle, additionalTags)

	addTags := make(map[string]string)
	for k, expected := range expectedTags {
		actual := actualTagMap[k]
		if actual == expected {
			continue
		}
		if actual == "" {
			klog.Warningf("Resource %q was missing expected cluster tag %q.  Will add (with value %q)", resourceID, k, expected)
			addTags[k] = expected
		} else {
			return fmt.Errorf("resource %q has tag belonging to another cluster: %q=%q (expected %q)", resourceID, k, actual, expected)
		}
	}

	if len(addTags) == 0 {
		return nil
	}

	if err := t.createTags(client, resourceID, lifecycle, addTags); err != nil {
		return fmt.Errorf("error adding missing tags to resource %q: %q", resourceID, err)
	}

	return nil
}

// createTags calls EC2 CreateTags, but adds retry-on-failure logic
// We retry mainly because if we create an object, we cannot tag it until it is "fully created" (eventual consistency)
// The error code varies though (depending on what we are tagging), so we simply retry on all errors
func (t *awsTagging) createTags(client EC2, resourceID string, lifecycle ResourceLifecycle, additionalTags map[string]string) error {
	tags := t.buildTags(lifecycle, additionalTags)

	if tags == nil || len(tags) == 0 {
		return nil
	}

	var awsTags []*ec2.Tag
	for k, v := range tags {
		tag := &ec2.Tag{
			Key:   aws.String(k),
			Value: aws.String(v),
		}
		awsTags = append(awsTags, tag)
	}

	backoff := wait.Backoff{
		Duration: createTagInitialDelay,
		Factor:   createTagFactor,
		Steps:    createTagSteps,
	}
	request := &ec2.CreateTagsInput{}
	request.Resources = []*string{&resourceID}
	request.Tags = awsTags

	var lastErr error
	err := wait.ExponentialBackoff(backoff, func() (bool, error) {
		_, err := client.CreateTags(request)
		if err == nil {
			return true, nil
		}

		// We could check that the error is retryable, but the error code changes based on what we are tagging
		// SecurityGroup: InvalidGroup.NotFound
		klog.V(2).Infof("Failed to create tags; will retry.  Error was %q", err)
		lastErr = err
		return false, nil
	})
	if err == wait.ErrWaitTimeout {
		// return real CreateTags error instead of timeout
		err = lastErr
	}
	return err
}

// Add additional filters, to match on our tags
// This lets us run multiple k8s clusters in a single EC2 AZ
func (t *awsTagging) addFilters(filters []*ec2.Filter) []*ec2.Filter {
	// if there are no clusterID configured - no filtering by special tag names
	// should be applied to revert to legacy behaviour.
	if len(t.ClusterID) == 0 {
		if len(filters) == 0 {
			// We can't pass a zero-length Filters to AWS (it's an error)
			// So if we end up with no filters; just return nil
			return nil
		}
		return filters
	}

	f := newEc2Filter("tag-key", t.clusterTagKey())
	filters = append(filters, f)
	return filters
}

// Add additional filters, to match on our tags. This uses the tag for legacy
// 1.5 -> 1.6 clusters and exists for backwards compatibility
//
// This lets us run multiple k8s clusters in a single EC2 AZ
func (t *awsTagging) addLegacyFilters(filters []*ec2.Filter) []*ec2.Filter {
	// if there are no clusterID configured - no filtering by special tag names
	// should be applied to revert to legacy behaviour.
	if len(t.ClusterID) == 0 {
		if len(filters) == 0 {
			// We can't pass a zero-length Filters to AWS (it's an error)
			// So if we end up with no filters; just return nil
			return nil
		}
		return filters
	}

	f := newEc2Filter(fmt.Sprintf("tag:%s", TagNameKubernetesClusterLegacy), t.ClusterID)

	// We can't pass a zero-length Filters to AWS (it's an error)
	// So if we end up with no filters; we need to return nil
	filters = append(filters, f)
	return filters
}

func (t *awsTagging) buildTags(lifecycle ResourceLifecycle, additionalTags map[string]string) map[string]string {
	tags := make(map[string]string)
	for k, v := range additionalTags {
		tags[k] = v
	}

	// no clusterID is a sign of misconfigured cluster, but we can't be tagging the resources with empty
	// strings
	if len(t.ClusterID) == 0 {
		return tags
	}

	// We only create legacy tags if we are using legacy tags, i.e. if we have seen a legacy tag on our instance
	if t.usesLegacyTags {
		tags[TagNameKubernetesClusterLegacy] = t.ClusterID
	}
	tags[t.clusterTagKey()] = string(lifecycle)

	return tags
}

func (t *awsTagging) clusterID() string {
	return t.ClusterID
}

// TagResource calls EC2 and tag the resource associated to resourceID
// with the supplied tags
func (c *Cloud) TagResource(resourceID string, tags map[string]string) error {
	request := &ec2.CreateTagsInput{
		Resources: []*string{aws.String(resourceID)},
		Tags:      buildAwsTags(tags),
	}

	output, err := c.ec2.CreateTags(request)

	if err != nil {
		klog.Errorf("Error occurred trying to tag resources, %v", err)
		return err
	}

	klog.Infof("Done calling create-tags to EC2: %v", output)

	return nil
}

// UntagResource calls EC2 and tag the resource associated to resourceID
// with the supplied tags
func (c *Cloud) UntagResource(resourceID string, tags map[string]string) error {
	request := &ec2.DeleteTagsInput{
		Resources: []*string{aws.String(resourceID)},
		Tags:      buildAwsTags(tags),
	}

	output, err := c.ec2.DeleteTags(request)

	if err != nil {
		klog.Errorf("Error occurred trying to untag resources, %v", err)
		return err
	}

	klog.Infof("Done calling delete-tags to EC2: %v", output)

	return nil
}

func buildAwsTags(tags map[string]string) []*ec2.Tag {
	var awsTags []*ec2.Tag
	for k, v := range tags {
		newTag := &ec2.Tag{
			Key:   aws.String(k),
			Value: aws.String(v),
		}
		awsTags = append(awsTags, newTag)
	}

	return awsTags
}
<|endoftext|>


<commit_before>// https://github.com/abiosoft/ishell/blob/master/example/main.go
package main

import (
//	"errors"
//	"fmt"
//	"context"
	"os"
//	"strings"
//	"time"

	"github.com/abiosoft/ishell"

//	"github.com/aws/aws-sdk-go/aws"
//	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
//	"github.com/aws/aws-sdk-go/service/s3/s3manager"

)

func main () {
	pwd := "/"
	var list map [string]string
	sess := session.Must(session.NewSessionWithOptions(session.Options{
		SharedConfigState: session.SharedConfigEnable,
		}))
	svc := s3.New(sess)

	shell := ishell.New()

	//display info
	shell.Println("S3 Shell")

	shell.AddCmd(&ishell.Cmd{
		Name: "ls",
		Help: "list objects",
		Func: func(c *ishell.Context){
			list = ls (c, svc, &pwd, sess)
			for r,v := range list {
				if len(v) > 0 {
					c.Print(v + "\t")
				}
				c.Println(r)
			}
		},
	})

	shell.AddCmd(&ishell.Cmd{
		Name: "desc",
		Help: "describe object's metadata",
		Func: func (c *ishell.Context) {
			obj := ""
			if len(c.Args) > 0 {
				obj = c.Args[0]
			}
			if len(obj) == 0 {
				c.Println("Please specify object name.")
				return
			}

			describe (c, svc, &pwd, obj)
		},
	})

	shell.AddCmd(&ishell.Cmd{
		Name: "pwd",
		Help: "print current directory",
		Func: func(c *ishell.Context){
			printdir (c, &pwd)
		},
	})

	shell.AddCmd(&ishell.Cmd{
		Name: "cd",
		Help: "change directory",
		Func: func(c *ishell.Context){
			cd(c, svc, &pwd, sess)
		},
	})

	if len(os.Args) > 1 && os.Args[1] == "exit" {
		shell.Process(os.Args[2:]...)
	} else {
		shell.Run()
		shell.Close()
	}

}
<commit_msg>fixed cr function to shange session, not only service.  Added get function.  Next step: move cr and get to functions.go file.<commit_after>// https://github.com/abiosoft/ishell/blob/master/example/main.go
package main

import (
//	"errors"
//	"fmt"
//	"context"
	"os"
	"strings"
//	"time"

	"github.com/abiosoft/ishell"

	"github.com/aws/aws-sdk-go/aws"
//	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"

)

func main () {
	pwd := "/"
	var list map [string]string
//	var uploader s3manager.Uploader
	var downloader *s3manager.Downloader
	sess := session.Must(session.NewSessionWithOptions(session.Options{
		SharedConfigState: session.SharedConfigEnable,
		}))
	svc := s3.New(sess)

	shell := ishell.New()

	//display info
	shell.Println("S3 Shell")

	shell.AddCmd(&ishell.Cmd{
		Name: "get",
		Help: "download an object",
		Func: func(c *ishell.Context){
			key := c.Args[0]
			bucket := strings.SplitAfter(pwd, "/")[1]
			if strings.LastIndex(bucket, "/") == len(bucket) -1 {
                 		bucket = bucket[:len(bucket) - 1]
       			}
        		prefix := strings.SplitAfter(pwd, bucket)[1]
        		if strings.Index(prefix, "/") == 0 {
                		prefix = prefix[1:]
       			}
        		if strings.LastIndex(prefix, "/") != len(prefix) - 1 {
                		prefix = prefix + "/"
        		}
			fullKey := prefix + key
			downloader = s3manager.NewDownloader(sess)
			f, err := os.Create(key)
			if err != nil {
				c.Println("Can't create local file " + key)
				c.Println("%v", err)
				return
			}
			n, err := downloader.Download(f, &s3.GetObjectInput{
				Bucket: &bucket,
				Key:  &fullKey,
			})
			if err != nil {
				c.Printf("Failed to download file %v\n", key)
				c.Printf("%v\n", err)
				return
			}
			c.Printf("Downloaded %v bytes\n", n)

		},
	})

	shell.AddCmd(&ishell.Cmd{
		Name: "cr",
		Help: "change region",
		Func: func(c *ishell.Context){
			if len(c.Args) > 0 {
				sess = session.Must(session.NewSessionWithOptions(session.Options{
                SharedConfigState: session.SharedConfigEnable,
		Config: aws.Config{Region: &c.Args[0] },
                }))
				svc = s3.New(sess)
				//svc = s3.New(sess, aws.NewConfig().WithRegion(c.Args[0]))
			}
		        if len(c.Args) == 0 {
				c.Println("Please specify region name.")
				return
			}
		},
	})

	shell.AddCmd(&ishell.Cmd{
		Name: "ls",
		Help: "list objects",
		Func: func(c *ishell.Context){
			list = ls (c, svc, &pwd, sess)
			for r,v := range list {
				if len(v) > 0 {
					c.Print(v + "\t")
				}
				c.Println(r)
			}
		},
	})

	shell.AddCmd(&ishell.Cmd{
		Name: "desc",
		Help: "describe object's metadata",
		Func: func (c *ishell.Context) {
			obj := ""
			if len(c.Args) > 0 {
				obj = c.Args[0]
			}
			if len(obj) == 0 {
				c.Println("Please specify object name.")
				return
			}

			describe (c, svc, &pwd, obj)
		},
	})

	shell.AddCmd(&ishell.Cmd{
		Name: "pwd",
		Help: "print current directory",
		Func: func(c *ishell.Context){
			printdir (c, &pwd)
		},
	})

	shell.AddCmd(&ishell.Cmd{
		Name: "cd",
		Help: "change directory",
		Func: func(c *ishell.Context){
			cd(c, svc, &pwd, sess)
		},
	})

	if len(os.Args) > 1 && os.Args[1] == "exit" {
		shell.Process(os.Args[2:]...)
	} else {
		shell.Run()
		shell.Close()
	}

}
<|endoftext|>


<commit_before>package fetch

import (
	"testing"
	"time"

	"github.com/slotix/dataflowkit/splash"
	"github.com/spf13/viper"
	"github.com/stretchr/testify/assert"
)

func Test_server(t *testing.T) {
	//start fetch server
	fetchServer := "127.0.0.1:8000"
	serverCfg := Config{
		Host:         fetchServer, //"localhost:5000",
		ReadTimeout:  5 * time.Second,
		WriteTimeout: 5 * time.Second,
	}
	viper.Set("SKIP_STORAGE_MW", true)
	htmlServer := Start(serverCfg)

	//create HTTPClient to send requests.
	svc, err := NewHTTPClient(fetchServer)
	if err != nil {
		logger.Error(err)
	}

	//send request to base fetcher endpoint
	req := BaseFetcherRequest{
		//URL: "http://" + addr,
		URL: "http://example.com",
	}
	html, err := svc.Fetch(req)
	if err != nil {
		logger.Error(err)
	}
	assert.NoError(t, err, "error is nil")
	assert.NotNil(t, html)
	//data, err := ioutil.ReadAll(html)
	//assert.NoError(t, err, "Expected no error")
	//assert.Equal(t, indexContent, data, "Expected Hello World")

	//send request to splash fetcher endpoint
	sReq := splash.Request{
		//URL: "http://" + addr,
		URL: "http://example.com",
	}
	r, err := svc.Fetch(sReq)
	if err != nil {
		logger.Error(err)
	}
	//data, err = ioutil.ReadAll(r)
	assert.NoError(t, err, "Expected no error")
	assert.NotNil(t, r)
	//t.Log(string(data))
	//assert.Equal(t, indexContent, data, "Expected Hello World")

	// //Test forbidden by robots
	// req = BaseFetcherRequest{
	// 	URL: "https://github.com",
	// }
	// html, err = svc.Fetch(req)
	// if err != nil {
	// 	logger.Error(err)
	// }
	// assert.NoError(t, err, "error is nil")

	//Stop fetch server
	htmlServer.Stop()
}
<commit_msg>add more tests<commit_after>package fetch

import (
	"io/ioutil"
	"testing"
	"time"

	"github.com/slotix/dataflowkit/splash"
	"github.com/spf13/viper"
	"github.com/stretchr/testify/assert"
)

func Test_server_Base(t *testing.T) {
	//start fetch server
	viper.Set("DFK_FETCH", "127.0.0.1:8000")
	fetchServer := viper.GetString("DFK_FETCH")
	serverCfg := Config{
		Host:         fetchServer,
		ReadTimeout:  5 * time.Second,
		WriteTimeout: 5 * time.Second,
	}
	//viper.Set("SKIP_STORAGE_MW", true)
	htmlServer := Start(serverCfg)

	//create HTTPClient to send requests.
	svc, err := NewHTTPClient(fetchServer)
	if err != nil {
		logger.Error(err)
	}
	////////

	//send request to base fetcher endpoint
	req := BaseFetcherRequest{
		URL: "http://" + addr,
		//URL: "http://example.com",
		//URL: "http://github.com",
	}
	data, err := svc.Fetch(req)
	if err != nil {
		logger.Error(err)
	}
	assert.NoError(t, err, "error is nil")
	//	assert.NotNil(t, html)
	html, err := ioutil.ReadAll(data)
	assert.NoError(t, err, "Expected no error")
	assert.Equal(t, indexContent, html, "Expected Hello World")

	//Test invalid Response Status codes.
	urls := []string{
		"http://" + addr + "/status/404",
		"http://" + addr + "/status/400",
		"http://" + addr + "/status/401",
		//"http://" + addr + "/status/unknown",
		"http://" + addr + "/status/403",
		"http://" + addr + "/status/500",
		"http://" + addr + "/status/502",
		"http://" + addr + "/status/504",
		"http://" + addr + "/status/600",
		"http://google",
		"google.com",
	}
	for _, url := range urls {
		req := BaseFetcherRequest{
			URL: url,
		}
		_, err = svc.Fetch(req)
		t.Log(err)
		assert.Error(t, err)
	}

	htmlServer.Stop()
}

func Test_server_Splash(t *testing.T) {
	//start fetch server
	viper.Set("DFK_FETCH", "127.0.0.1:8000")
	fetchServer := viper.GetString("DFK_FETCH")
	serverCfg := Config{
		Host:         fetchServer, //"localhost:5000",
		ReadTimeout:  5 * time.Second,
		WriteTimeout: 5 * time.Second,
	}
	viper.Set("SKIP_STORAGE_MW", true)
	htmlServer := Start(serverCfg)

	//create HTTPClient to send requests.
	svc, err := NewHTTPClient(fetchServer)
	if err != nil {
		logger.Error(err)
	}
	////////
	//send request to splash fetcher endpoint
	sReq := splash.Request{
		//URL: "http://" + addr,
		URL: "http://example.com",
	}
	r, err := svc.Response(sReq)
	if err != nil {
		logger.Error(err)
	}
	//data, err = ioutil.ReadAll(r)
	assert.NoError(t, err, "Expected no error")
	assert.NotNil(t, r)
	//t.Log(string(data))
	//assert.Equal(t, indexContent, data, "Expected Hello World")

	data, err := svc.Fetch(sReq)
	if err != nil {
		logger.Error(err)
	}
	//data, err = ioutil.ReadAll(r)
	assert.NoError(t, err, "Expected no error")
	assert.NotNil(t, data)

	htmlServer.Stop()
}
<|endoftext|>