<commit_before>package main

import (
	"context"
	"expvar"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/aws/aws-sdk-go/service/kinesis"

	alog "github.com/apex/log"
	"github.com/apex/log/handlers/text"

	consumer "github.com/harlow/kinesis-consumer"
	checkpoint "github.com/harlow/kinesis-consumer/checkpoint/ddb"
)

// kick off a server for exposing scan metrics
func init() {
	sock, err := net.Listen("tcp", "localhost:8080")
	if err != nil {
		log.Printf("net listen error: %v", err)
	}
	go func() {
		fmt.Println("Metrics available at http://localhost:8080/debug/vars")
		http.Serve(sock, nil)
	}()
}

// A myLogger provides a minimalistic logger satisfying the Logger interface.
type myLogger struct {
	logger alog.Logger
}

// Log logs the parameters to the stdlib logger. See log.Println.
func (l *myLogger) Log(args ...interface{}) {
	l.logger.Infof("producer: %v", args...)
}

func main() {
	// Wrap myLogger around  apex logger
	log := &myLogger{
		logger: alog.Logger{
			Handler: text.New(os.Stdout),
			Level:   alog.DebugLevel,
		},
	}

	var (
		app    = flag.String("app", "", "App name")
		stream = flag.String("stream", "", "Stream name")
		table  = flag.String("table", "", "Checkpoint table name")
	)
	flag.Parse()

	// Following will overwrite the default dynamodb client
	// Older versions of aws sdk does not picking up aws config properly.
	// You probably need to update aws sdk verison. Tested the following with 1.13.59
	myDynamoDbClient := dynamodb.New(
		session.New(aws.NewConfig()), &aws.Config{
			Region: aws.String("us-west-2"),
		},
	)

	// ddb checkpoint
	ck, err := checkpoint.New(*app, *table, checkpoint.WithDynamoClient(myDynamoDbClient), checkpoint.WithRetryer(&MyRetryer{}))
	if err != nil {
		log.Log("checkpoint error: %v", err)
	}

	var counter = expvar.NewMap("counters")

	// The following 2 lines will overwrite the default kinesis client
	ksis := kinesis.New(
		session.New(aws.NewConfig()), &aws.Config{
			Region: aws.String("us-west-2"),
		},
	)

	// consumer
	c, err := consumer.New(
		*stream,
		consumer.WithCheckpoint(ck),
		consumer.WithLogger(log),
		consumer.WithCounter(counter),
		consumer.WithClient(ksis),
	)
	if err != nil {
		log.Log("consumer error: %v", err)
	}

	// use cancel func to signal shutdown
	ctx, cancel := context.WithCancel(context.Background())

	// trap SIGINT, wait to trigger shutdown
	signals := make(chan os.Signal, 1)
	signal.Notify(signals, os.Interrupt)

	go func() {
		<-signals
		cancel()
	}()

	// scan stream
	err = c.Scan(ctx, func(r *consumer.Record) consumer.ScanStatus {
		fmt.Println(string(r.Data))

		// continue scanning
		return consumer.ScanStatus{}
	})
	if err != nil {
		log.Log("scan error: %v", err)
	}

	if err := ck.Shutdown(); err != nil {
		log.Log("checkpoint shutdown error: %v", err)
	}
}

// MyRetryer used for checkpointing
type MyRetryer struct {
	checkpoint.Retryer
}

// ShouldRetry implements custom logic for when a checkpont should retry
func (r *MyRetryer) ShouldRetry(err error) bool {
	if awsErr, ok := err.(awserr.Error); ok {
		switch awsErr.Code() {
		case dynamodb.ErrCodeProvisionedThroughputExceededException, dynamodb.ErrCodeLimitExceededException:
			return true
		default:
			return false
		}
	}
	return false
}
<commit_msg>Update example to use new AWS Session<commit_after>package main

import (
	"context"
	"expvar"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"

	alog "github.com/apex/log"
	"github.com/apex/log/handlers/text"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/aws/aws-sdk-go/service/kinesis"
	consumer "github.com/harlow/kinesis-consumer"
	checkpoint "github.com/harlow/kinesis-consumer/checkpoint/ddb"
)

// kick off a server for exposing scan metrics
func init() {
	sock, err := net.Listen("tcp", "localhost:8080")
	if err != nil {
		log.Printf("net listen error: %v", err)
	}
	go func() {
		fmt.Println("Metrics available at http://localhost:8080/debug/vars")
		http.Serve(sock, nil)
	}()
}

// A myLogger provides a minimalistic logger satisfying the Logger interface.
type myLogger struct {
	logger alog.Logger
}

// Log logs the parameters to the stdlib logger. See log.Println.
func (l *myLogger) Log(args ...interface{}) {
	l.logger.Infof("producer: %v", args...)
}

func main() {
	// Wrap myLogger around  apex logger
	log := &myLogger{
		logger: alog.Logger{
			Handler: text.New(os.Stdout),
			Level:   alog.DebugLevel,
		},
	}

	var (
		app    = flag.String("app", "", "App name")
		stream = flag.String("stream", "", "Stream name")
		table  = flag.String("table", "", "Checkpoint table name")
	)
	flag.Parse()

	sess, err := session.NewSession(aws.NewConfig())
	if err != nil {
		log.Log("new session error: %v", err)
	}

	// New Kinesis and DynamoDB clients (if you need custom config)
	myKsis := kinesis.New(sess)
	myDdbClient := dynamodb.New(sess)

	// ddb checkpoint
	ck, err := checkpoint.New(*app, *table, checkpoint.WithDynamoClient(myDdbClient), checkpoint.WithRetryer(&MyRetryer{}))
	if err != nil {
		log.Log("checkpoint error: %v", err)
	}

	// expvar counter
	var counter = expvar.NewMap("counters")

	// consumer
	c, err := consumer.New(
		*stream,
		consumer.WithCheckpoint(ck),
		consumer.WithLogger(log),
		consumer.WithCounter(counter),
		consumer.WithClient(myKsis),
	)
	if err != nil {
		log.Log("consumer error: %v", err)
	}

	// use cancel func to signal shutdown
	ctx, cancel := context.WithCancel(context.Background())

	// trap SIGINT, wait to trigger shutdown
	signals := make(chan os.Signal, 1)
	signal.Notify(signals, os.Interrupt)

	go func() {
		<-signals
		cancel()
	}()

	// scan stream
	err = c.Scan(ctx, func(r *consumer.Record) consumer.ScanStatus {
		fmt.Println(string(r.Data))

		// continue scanning
		return consumer.ScanStatus{}
	})
	if err != nil {
		log.Log("scan error: %v", err)
	}

	if err := ck.Shutdown(); err != nil {
		log.Log("checkpoint shutdown error: %v", err)
	}
}

// MyRetryer used for checkpointing
type MyRetryer struct {
	checkpoint.Retryer
}

// ShouldRetry implements custom logic for when a checkpont should retry
func (r *MyRetryer) ShouldRetry(err error) bool {
	if awsErr, ok := err.(awserr.Error); ok {
		switch awsErr.Code() {
		case dynamodb.ErrCodeProvisionedThroughputExceededException, dynamodb.ErrCodeLimitExceededException:
			return true
		default:
			return false
		}
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2015 Robert S. Gerus. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package bot

import (
	"encoding/json"
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/cloudflare/gokabinet/kt"

	cfg "github.com/arachnist/gorepost/config"
	"github.com/arachnist/gorepost/irc"
)

var k *kt.Conn

type seenRecord struct {
	Network string
	Target  string
	Action  string
	Time    time.Time
	Text    string
}

func seenrecord(output chan irc.Message, msg irc.Message) {
	v := seenRecord{
		Network: msg.Context["Network"],
		Target:  msg.Params[0],
		Action:  msg.Command,
		Time:    time.Now(),
		Text:    msg.Trailing,
	}

	b, err := json.Marshal(v)
	if err != nil {
		log.Println("Context:", msg.Context, "json marshal of seen record failed:", err)
		return
	}

	err = k.Set("seen/"+msg.Prefix.Name, b)
	if err != nil {
		log.Println("Context:", msg.Context, "error recording seen record:", err)
	}
}

func seen(output chan irc.Message, msg irc.Message) {
	var v seenRecord
	var r string

	args := strings.Split(msg.Trailing, " ")

	if args[0] != ":seen" {
		return
	}
	if len(args) < 2 {
		return
	}

	b, err := k.GetBytes("seen/" + args[1])
	if err == kt.ErrNotFound {
		output <- reply(msg, cfg.LookupString(msg.Context, "NotSeenMessage"))
		return
	} else if err != nil {
		output <- reply(msg, fmt.Sprint("error getting record for", args[1], err))
		return
	}

	err = json.Unmarshal(b, &v)
	if err != nil {
		output <- reply(msg, fmt.Sprint("error unmarshaling record for", args[1], err))
		return
	}

	r = fmt.Sprintf("Last seen %s on %s/%s at %v ", args[1], v.Network, v.Target, v.Time.Round(time.Minute))

	switch v.Action {
	case "JOIN":
		r += "joining"
	case "PART":
		r += fmt.Sprint("leaving: ", v.Text)
	case "QUIT":
		r += fmt.Sprint("quitting with reasson: ", v.Text)
	case "PRIVMSG":
		r += fmt.Sprint("saying: ", v.Text)
	}

	output <- reply(msg, r)
}

func init() {
	var err error
	var ktHost = "127.0.0.1"
	var ktPort = 1337

	log.Println("SEEN: connecting to KT")
	k, err = kt.NewConn(ktHost, ktPort, 4, 2*time.Second)
	if err != nil {
		log.Println("error connecting to kyoto tycoon", err)
		return
	}

	log.Println("Registering callbacks")
	addCallback("PRIVMSG", "seen", seen)
	addCallback("PRIVMSG", "seenrecord", seenrecord)
	addCallback("JOIN", "seenrecord", seenrecord)
	addCallback("PART", "seenrecord", seenrecord)
	addCallback("QUIT", "seenrecord", seenrecord)
	addCallback("NOTICE", "seenrecord", seenrecord)
}
<commit_msg>Fix crash<commit_after>// Copyright 2015 Robert S. Gerus. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package bot

import (
	"encoding/json"
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/cloudflare/gokabinet/kt"

	cfg "github.com/arachnist/gorepost/config"
	"github.com/arachnist/gorepost/irc"
)

var k *kt.Conn

type seenRecord struct {
	Network string
	Target  string
	Action  string
	Time    time.Time
	Text    string
}

func seenrecord(output chan irc.Message, msg irc.Message) {
	if msg.Params == nil {
		return
	}

	v := seenRecord{
		Network: msg.Context["Network"],
		Target:  msg.Params[0],
		Action:  msg.Command,
		Time:    time.Now(),
		Text:    msg.Trailing,
	}

	b, err := json.Marshal(v)
	if err != nil {
		log.Println("Context:", msg.Context, "json marshal of seen record failed:", err)
		return
	}

	err = k.Set("seen/"+msg.Prefix.Name, b)
	if err != nil {
		log.Println("Context:", msg.Context, "error recording seen record:", err)
	}
}

func seen(output chan irc.Message, msg irc.Message) {
	var v seenRecord
	var r string

	args := strings.Split(msg.Trailing, " ")

	if args[0] != ":seen" {
		return
	}
	if len(args) < 2 {
		return
	}

	b, err := k.GetBytes("seen/" + args[1])
	if err == kt.ErrNotFound {
		output <- reply(msg, cfg.LookupString(msg.Context, "NotSeenMessage"))
		return
	} else if err != nil {
		output <- reply(msg, fmt.Sprint("error getting record for", args[1], err))
		return
	}

	err = json.Unmarshal(b, &v)
	if err != nil {
		output <- reply(msg, fmt.Sprint("error unmarshaling record for", args[1], err))
		return
	}

	r = fmt.Sprintf("Last seen %s on %s/%s at %v ", args[1], v.Network, v.Target, v.Time.Round(time.Minute))

	switch v.Action {
	case "JOIN":
		r += "joining"
	case "PART":
		r += fmt.Sprint("leaving: ", v.Text)
	case "QUIT":
		r += fmt.Sprint("quitting with reasson: ", v.Text)
	case "PRIVMSG":
		r += fmt.Sprint("saying: ", v.Text)
	}

	output <- reply(msg, r)
}

func init() {
	var err error
	var ktHost = "127.0.0.1"
	var ktPort = 1337

	log.Println("SEEN: connecting to KT")
	k, err = kt.NewConn(ktHost, ktPort, 4, 2*time.Second)
	if err != nil {
		log.Println("error connecting to kyoto tycoon", err)
		return
	}

	log.Println("Registering callbacks")
	addCallback("PRIVMSG", "seen", seen)
	addCallback("PRIVMSG", "seenrecord", seenrecord)
	addCallback("JOIN", "seenrecord", seenrecord)
	addCallback("PART", "seenrecord", seenrecord)
	addCallback("QUIT", "seenrecord", seenrecord)
	addCallback("NOTICE", "seenrecord", seenrecord)
}
<|endoftext|>


<commit_before>package logs

import (
	"context"
	"io"
	"os"

	"github.com/moby/buildkit/client"
	"github.com/moby/buildkit/identity"
	"github.com/moby/buildkit/util/progress"
	"github.com/pkg/errors"
)

func NewLogStreams(ctx context.Context, printOutput bool) (io.WriteCloser, io.WriteCloser) {
	return newStreamWriter(ctx, 1, printOutput), newStreamWriter(ctx, 2, printOutput)
}

func newStreamWriter(ctx context.Context, stream int, printOutput bool) io.WriteCloser {
	pw, _, _ := progress.FromContext(ctx)
	return &streamWriter{
		pw:          pw,
		stream:      stream,
		printOutput: printOutput,
	}
}

type streamWriter struct {
	pw          progress.Writer
	stream      int
	printOutput bool
}

func (sw *streamWriter) Write(dt []byte) (int, error) {
	sw.pw.Write(identity.NewID(), client.VertexLog{
		Stream: sw.stream,
		Data:   append([]byte{}, dt...),
	})
	if sw.printOutput {
		switch sw.stream {
		case 1:
			return os.Stdout.Write(dt)
		case 2:
			return os.Stderr.Write(dt)
		default:
			return 0, errors.Errorf("invalid stream %d", sw.stream)
		}
	}
	return len(dt), nil
}

func (sw *streamWriter) Close() error {
	return sw.pw.Close()
}
<commit_msg>progress: add log limits and clipping<commit_after>package logs

import (
	"context"
	"io"
	"math"
	"os"
	"strconv"
	"sync"
	"time"

	"github.com/moby/buildkit/client"
	"github.com/moby/buildkit/identity"
	"github.com/moby/buildkit/util/progress"
	"github.com/pkg/errors"
)

var defaultMaxLogSize = 1024 * 1024
var defaultMaxLogSpeed = 100 * 1024 // per second

var configCheckOnce sync.Once

func NewLogStreams(ctx context.Context, printOutput bool) (io.WriteCloser, io.WriteCloser) {
	return newStreamWriter(ctx, 1, printOutput), newStreamWriter(ctx, 2, printOutput)
}

func newStreamWriter(ctx context.Context, stream int, printOutput bool) io.WriteCloser {
	pw, _, _ := progress.FromContext(ctx)
	return &streamWriter{
		pw:          pw,
		stream:      stream,
		printOutput: printOutput,
		created:     time.Now(),
	}
}

type streamWriter struct {
	pw          progress.Writer
	stream      int
	printOutput bool
	created     time.Time
	size        int
	clipping    bool
}

func (sw *streamWriter) checkLimit(n int) int {
	configCheckOnce.Do(func() {
		maxLogSize, err := strconv.ParseInt(os.Getenv("BUILDKIT_STEP_LOG_MAX_SIZE"), 10, 32)
		if err == nil {
			defaultMaxLogSize = int(maxLogSize)
		}
		maxLogSpeed, err := strconv.ParseInt(os.Getenv("BUILDKIT_STEP_LOG_MAX_SPEED"), 10, 32)
		if err == nil {
			defaultMaxLogSpeed = int(maxLogSpeed)
		}
	})

	oldSize := sw.size
	sw.size += n

	maxSize := -1
	if defaultMaxLogSpeed != -1 {
		maxSize = int(math.Ceil(time.Since(sw.created).Seconds())) * defaultMaxLogSpeed
	}
	if maxSize > defaultMaxLogSize {
		maxSize = defaultMaxLogSize
	}
	if maxSize < oldSize {
		return 0
	}

	if maxSize != -1 {
		if sw.size > maxSize {
			return maxSize - oldSize
		}
	}
	return n
}

func (sw *streamWriter) Write(dt []byte) (int, error) {
	oldSize := len(dt)
	dt = append([]byte{}, dt[:sw.checkLimit(len(dt))]...)

	if sw.clipping && oldSize == len(dt) {
		sw.clipping = false
	}
	if !sw.clipping && oldSize != len(dt) {
		dt = append(dt, []byte("\n[output clipped, log limit reached]\n")...)
		sw.clipping = true
	}

	if len(dt) != 0 {
		sw.pw.Write(identity.NewID(), client.VertexLog{
			Stream: sw.stream,
			Data:   dt,
		})
		if sw.printOutput {
			switch sw.stream {
			case 1:
				return os.Stdout.Write(dt)
			case 2:
				return os.Stderr.Write(dt)
			default:
				return 0, errors.Errorf("invalid stream %d", sw.stream)
			}
		}
	}
	return oldSize, nil
}

func (sw *streamWriter) Close() error {
	return sw.pw.Close()
}
<|endoftext|>


<commit_before>package hook

import (
	"context"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"

	gocf "github.com/mweagle/go-cloudformation"

	"github.com/aws/aws-sdk-go/aws/session"
	sparta "github.com/mweagle/Sparta"
	"github.com/mweagle/Sparta/system"
	"github.com/rs/zerolog"
)

// PostBuildUPXCompressHook returns a WorkflowHookHandler that handles
// UPX compressing the `go` binary containing our functions
func PostBuildUPXCompressHook(dockerImageName string) sparta.WorkflowHookHandler {
	upxHook := func(ctx context.Context,
		serviceName string,
		S3Bucket gocf.Stringable,
		buildID string,
		awsSession *session.Session,
		noop bool,
		logger *zerolog.Logger) (context.Context, error) {
		logger.Info().
			Str("DockerImage", dockerImageName).
			Msg("Compressing binary with UPX")

		outputDir := ctx.Value(sparta.ContextKeyBuildOutputDir).(string)
		outputBinaryName := ctx.Value(sparta.ContextKeyBuildBinaryName).(string)
		absPath, _ := filepath.Abs(outputDir)
		outputBinaryPath := filepath.Join(absPath, outputBinaryName)

		outputCompressedBinaryName := fmt.Sprintf("%s.upx", outputBinaryName)
		outputCompressedBinaryPath := filepath.Join(absPath, outputCompressedBinaryName)
		_, statErr := os.Stat(outputCompressedBinaryPath)
		if !os.IsNotExist(statErr) {
			removeErr := os.Remove(outputCompressedBinaryPath)
			if removeErr != nil {
				logger.Warn().
					Err(removeErr).
					Msg("Failed to delete existing file")
			}
		}

		// Run the UPX packer against the binary with a new name,
		// then overwrite the exisitng one...
		commandLineArgs := []string{
			"run",
			"--rm",
			"-w",
			"/var/tmp",
			"-v",
			fmt.Sprintf("%s:/var/tmp", absPath),
			"hairyhenderson/upx",
			"--best",
			"--lzma",
			"-o",
			outputCompressedBinaryName,
			outputBinaryName,
		}
		dockerCompressCmd := exec.Command("docker", commandLineArgs...)
		dockerCompressCmdErr := system.RunOSCommand(dockerCompressCmd, logger)
		if dockerCompressCmdErr != nil {
			return ctx, dockerCompressCmdErr
		}
		return ctx, os.Rename(outputCompressedBinaryPath, outputBinaryPath)
	}
	return sparta.WorkflowHookFunc(upxHook)
}
<commit_msg>Update misspellings<commit_after>package hook

import (
	"context"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"

	gocf "github.com/mweagle/go-cloudformation"

	"github.com/aws/aws-sdk-go/aws/session"
	sparta "github.com/mweagle/Sparta"
	"github.com/mweagle/Sparta/system"
	"github.com/rs/zerolog"
)

// PostBuildUPXCompressHook returns a WorkflowHookHandler that handles
// UPX compressing the `go` binary containing our functions
func PostBuildUPXCompressHook(dockerImageName string) sparta.WorkflowHookHandler {
	upxHook := func(ctx context.Context,
		serviceName string,
		S3Bucket gocf.Stringable,
		buildID string,
		awsSession *session.Session,
		noop bool,
		logger *zerolog.Logger) (context.Context, error) {
		logger.Info().
			Str("DockerImage", dockerImageName).
			Msg("Compressing binary with UPX")

		outputDir := ctx.Value(sparta.ContextKeyBuildOutputDir).(string)
		outputBinaryName := ctx.Value(sparta.ContextKeyBuildBinaryName).(string)
		absPath, _ := filepath.Abs(outputDir)
		outputBinaryPath := filepath.Join(absPath, outputBinaryName)

		outputCompressedBinaryName := fmt.Sprintf("%s.upx", outputBinaryName)
		outputCompressedBinaryPath := filepath.Join(absPath, outputCompressedBinaryName)
		_, statErr := os.Stat(outputCompressedBinaryPath)
		if !os.IsNotExist(statErr) {
			removeErr := os.Remove(outputCompressedBinaryPath)
			if removeErr != nil {
				logger.Warn().
					Err(removeErr).
					Msg("Failed to delete existing file")
			}
		}

		// Run the UPX packer against the binary with a new name,
		// then overwrite the existing one...
		commandLineArgs := []string{
			"run",
			"--rm",
			"-w",
			"/var/tmp",
			"-v",
			fmt.Sprintf("%s:/var/tmp", absPath),
			"hairyhenderson/upx",
			"--best",
			"--lzma",
			"-o",
			outputCompressedBinaryName,
			outputBinaryName,
		}
		dockerCompressCmd := exec.Command("docker", commandLineArgs...)
		dockerCompressCmdErr := system.RunOSCommand(dockerCompressCmd, logger)
		if dockerCompressCmdErr != nil {
			return ctx, dockerCompressCmdErr
		}
		return ctx, os.Rename(outputCompressedBinaryPath, outputBinaryPath)
	}
	return sparta.WorkflowHookFunc(upxHook)
}
<|endoftext|>


<commit_before>package semver

import (
	"errors"
	"fmt"
	"strconv"
	"strings"
)

const (
	numbers  string = "0123456789"
	alphas          = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-"
	alphanum        = alphas + numbers
)

// SpecVersion is the latest fully supported spec version of semver
var SpecVersion = Version{
	Major: 2,
	Minor: 0,
	Patch: 0,
}

// Version represents a semver compatible version
type Version struct {
	Major uint64
	Minor uint64
	Patch uint64
	Pre   []PRVersion
	Build []string //No Precedence
}

// Version to string
func (v Version) String() string {
	b := make([]byte, 0, 5)
	b = strconv.AppendUint(b, v.Major, 10)
	b = append(b, '.')
	b = strconv.AppendUint(b, v.Minor, 10)
	b = append(b, '.')
	b = strconv.AppendUint(b, v.Patch, 10)

	if len(v.Pre) > 0 {
		b = append(b, '-')
		b = append(b, v.Pre[0].String()...)

		for _, pre := range v.Pre[1:] {
			b = append(b, '.')
			b = append(b, pre.String()...)
		}
	}

	if len(v.Build) > 0 {
		b = append(b, '+')
		b = append(b, v.Build[0]...)

		for _, build := range v.Build[1:] {
			b = append(b, '.')
			b = append(b, build...)
		}
	}

	return string(b)
}

// Equals checks if v is equal to o.
func (v Version) Equals(o Version) bool {
	return (v.Compare(o) == 0)
}

// EQ checks if v is equal to o.
func (v Version) EQ(o Version) bool {
	return (v.Compare(o) == 0)
}

// NE checks if v is not equal to o.
func (v Version) NE(o Version) bool {
	return (v.Compare(o) != 0)
}

// GT checks if v is greater than o.
func (v Version) GT(o Version) bool {
	return (v.Compare(o) == 1)
}

// GTE checks if v is greater than or equal to o.
func (v Version) GTE(o Version) bool {
	return (v.Compare(o) >= 0)
}

// GE checks if v is greater than or equal to o.
func (v Version) GE(o Version) bool {
	return (v.Compare(o) >= 0)
}

// LT checks if v is less than o.
func (v Version) LT(o Version) bool {
	return (v.Compare(o) == -1)
}

// LTE checks if v is less than or equal to o.
func (v Version) LTE(o Version) bool {
	return (v.Compare(o) <= 0)
}

// LE checks if v is less than or equal to o.
func (v Version) LE(o Version) bool {
	return (v.Compare(o) <= 0)
}

// Compare compares Versions v to o:
// -1 == v is less than o
// 0 == v is equal to o
// 1 == v is greater than o
func (v Version) Compare(o Version) int {
	if v.Major != o.Major {
		if v.Major > o.Major {
			return 1
		}
		return -1
	}
	if v.Minor != o.Minor {
		if v.Minor > o.Minor {
			return 1
		}
		return -1
	}
	if v.Patch != o.Patch {
		if v.Patch > o.Patch {
			return 1
		}
		return -1
	}

	// Quick comparison if a version has no prerelease versions
	if len(v.Pre) == 0 && len(o.Pre) == 0 {
		return 0
	} else if len(v.Pre) == 0 && len(o.Pre) > 0 {
		return 1
	} else if len(v.Pre) > 0 && len(o.Pre) == 0 {
		return -1
	}

	i := 0
	for ; i < len(v.Pre) && i < len(o.Pre); i++ {
		if comp := v.Pre[i].Compare(o.Pre[i]); comp == 0 {
			continue
		} else if comp == 1 {
			return 1
		} else {
			return -1
		}
	}

	// If all pr versions are the equal but one has further prversion, this one greater
	if i == len(v.Pre) && i == len(o.Pre) {
		return 0
	} else if i == len(v.Pre) && i < len(o.Pre) {
		return -1
	} else {
		return 1
	}

}

// IncrementPatch increments the patch version
func (v *Version) IncrementPatch() error {
	v.Patch++
	return nil
}

// IncrementMinor increments the minor version
func (v *Version) IncrementMinor() error {
	v.Minor++
	v.Patch = 0
	return nil
}

// IncrementMajor increments the major version
func (v *Version) IncrementMajor() error {
	v.Major++
	v.Minor = 0
	v.Patch = 0
	return nil
}

// Validate validates v and returns error in case
func (v Version) Validate() error {
	// Major, Minor, Patch already validated using uint64

	for _, pre := range v.Pre {
		if !pre.IsNum { //Numeric prerelease versions already uint64
			if len(pre.VersionStr) == 0 {
				return fmt.Errorf("Prerelease can not be empty %q", pre.VersionStr)
			}
			if !containsOnly(pre.VersionStr, alphanum) {
				return fmt.Errorf("Invalid character(s) found in prerelease %q", pre.VersionStr)
			}
		}
	}

	for _, build := range v.Build {
		if len(build) == 0 {
			return fmt.Errorf("Build meta data can not be empty %q", build)
		}
		if !containsOnly(build, alphanum) {
			return fmt.Errorf("Invalid character(s) found in build meta data %q", build)
		}
	}

	return nil
}

// New is an alias for Parse and returns a pointer, parses version string and returns a validated Version or error
func New(s string) (vp *Version, err error) {
	v, err := Parse(s)
	vp = &v
	return
}

// Make is an alias for Parse, parses version string and returns a validated Version or error
func Make(s string) (Version, error) {
	return Parse(s)
}

// ParseTolerant allows for certain version specifications that do not strictly adhere to semver
// specs to be parsed by this library. It does so by normalizing versions before passing them to
// Parse(). It currently trims spaces, removes a "v" prefix, adds a 0 patch number to versions
// with only major and minor components specified, and removes leading 0s.
func ParseTolerant(s string) (Version, error) {
	s = strings.TrimSpace(s)
	s = strings.TrimPrefix(s, "v")

	// Split into major.minor.(patch+pr+meta)
	parts := strings.SplitN(s, ".", 3)
	// Remove leading zeros.
	for i, p := range parts {
		if len(p) > 1 {
			p = strings.TrimLeft(p, "0")
			if len(p) == 0 || !strings.ContainsAny(p[0:1], "0123456789") {
				p = "0" + p
			}
			parts[i] = p
		}
	}
	// Fill up shortened versions.
	if len(parts) < 3 {
		if strings.ContainsAny(parts[len(parts)-1], "+-") {
			return Version{}, errors.New("Short version cannot contain PreRelease/Build meta data")
		}
		for len(parts) < 3 {
			parts = append(parts, "0")
		}
	}
	s = strings.Join(parts, ".")

	return Parse(s)
}

// Parse parses version string and returns a validated Version or error
func Parse(s string) (Version, error) {
	if len(s) == 0 {
		return Version{}, errors.New("Version string empty")
	}

	// Split into major.minor.(patch+pr+meta)
	parts := strings.SplitN(s, ".", 3)
	if len(parts) != 3 {
		return Version{}, errors.New("No Major.Minor.Patch elements found")
	}

	// Major
	if !containsOnly(parts[0], numbers) {
		return Version{}, fmt.Errorf("Invalid character(s) found in major number %q", parts[0])
	}
	if hasLeadingZeroes(parts[0]) {
		return Version{}, fmt.Errorf("Major number must not contain leading zeroes %q", parts[0])
	}
	major, err := strconv.ParseUint(parts[0], 10, 64)
	if err != nil {
		return Version{}, err
	}

	// Minor
	if !containsOnly(parts[1], numbers) {
		return Version{}, fmt.Errorf("Invalid character(s) found in minor number %q", parts[1])
	}
	if hasLeadingZeroes(parts[1]) {
		return Version{}, fmt.Errorf("Minor number must not contain leading zeroes %q", parts[1])
	}
	minor, err := strconv.ParseUint(parts[1], 10, 64)
	if err != nil {
		return Version{}, err
	}

	v := Version{}
	v.Major = major
	v.Minor = minor

	var build, prerelease []string
	patchStr := parts[2]

	if buildIndex := strings.IndexRune(patchStr, '+'); buildIndex != -1 {
		build = strings.Split(patchStr[buildIndex+1:], ".")
		patchStr = patchStr[:buildIndex]
	}

	if preIndex := strings.IndexRune(patchStr, '-'); preIndex != -1 {
		prerelease = strings.Split(patchStr[preIndex+1:], ".")
		patchStr = patchStr[:preIndex]
	}

	if !containsOnly(patchStr, numbers) {
		return Version{}, fmt.Errorf("Invalid character(s) found in patch number %q", patchStr)
	}
	if hasLeadingZeroes(patchStr) {
		return Version{}, fmt.Errorf("Patch number must not contain leading zeroes %q", patchStr)
	}
	patch, err := strconv.ParseUint(patchStr, 10, 64)
	if err != nil {
		return Version{}, err
	}

	v.Patch = patch

	// Prerelease
	for _, prstr := range prerelease {
		parsedPR, err := NewPRVersion(prstr)
		if err != nil {
			return Version{}, err
		}
		v.Pre = append(v.Pre, parsedPR)
	}

	// Build meta data
	for _, str := range build {
		if len(str) == 0 {
			return Version{}, errors.New("Build meta data is empty")
		}
		if !containsOnly(str, alphanum) {
			return Version{}, fmt.Errorf("Invalid character(s) found in build meta data %q", str)
		}
		v.Build = append(v.Build, str)
	}

	return v, nil
}

// MustParse is like Parse but panics if the version cannot be parsed.
func MustParse(s string) Version {
	v, err := Parse(s)
	if err != nil {
		panic(`semver: Parse(` + s + `): ` + err.Error())
	}
	return v
}

// PRVersion represents a PreRelease Version
type PRVersion struct {
	VersionStr string
	VersionNum uint64
	IsNum      bool
}

// NewPRVersion creates a new valid prerelease version
func NewPRVersion(s string) (PRVersion, error) {
	if len(s) == 0 {
		return PRVersion{}, errors.New("Prerelease is empty")
	}
	v := PRVersion{}
	if containsOnly(s, numbers) {
		if hasLeadingZeroes(s) {
			return PRVersion{}, fmt.Errorf("Numeric PreRelease version must not contain leading zeroes %q", s)
		}
		num, err := strconv.ParseUint(s, 10, 64)

		// Might never be hit, but just in case
		if err != nil {
			return PRVersion{}, err
		}
		v.VersionNum = num
		v.IsNum = true
	} else if containsOnly(s, alphanum) {
		v.VersionStr = s
		v.IsNum = false
	} else {
		return PRVersion{}, fmt.Errorf("Invalid character(s) found in prerelease %q", s)
	}
	return v, nil
}

// IsNumeric checks if prerelease-version is numeric
func (v PRVersion) IsNumeric() bool {
	return v.IsNum
}

// Compare compares two PreRelease Versions v and o:
// -1 == v is less than o
// 0 == v is equal to o
// 1 == v is greater than o
func (v PRVersion) Compare(o PRVersion) int {
	if v.IsNum && !o.IsNum {
		return -1
	} else if !v.IsNum && o.IsNum {
		return 1
	} else if v.IsNum && o.IsNum {
		if v.VersionNum == o.VersionNum {
			return 0
		} else if v.VersionNum > o.VersionNum {
			return 1
		} else {
			return -1
		}
	} else { // both are Alphas
		if v.VersionStr == o.VersionStr {
			return 0
		} else if v.VersionStr > o.VersionStr {
			return 1
		} else {
			return -1
		}
	}
}

// PreRelease version to string
func (v PRVersion) String() string {
	if v.IsNum {
		return strconv.FormatUint(v.VersionNum, 10)
	}
	return v.VersionStr
}

func containsOnly(s string, set string) bool {
	return strings.IndexFunc(s, func(r rune) bool {
		return !strings.ContainsRune(set, r)
	}) == -1
}

func hasLeadingZeroes(s string) bool {
	return len(s) > 1 && s[0] == '0'
}

// NewBuildVersion creates a new valid build version
func NewBuildVersion(s string) (string, error) {
	if len(s) == 0 {
		return "", errors.New("Buildversion is empty")
	}
	if !containsOnly(s, alphanum) {
		return "", fmt.Errorf("Invalid character(s) found in build meta data %q", s)
	}
	return s, nil
}
<commit_msg>maintaining consistency with other func format<commit_after>package semver

import (
	"errors"
	"fmt"
	"strconv"
	"strings"
)

const (
	numbers  string = "0123456789"
	alphas          = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-"
	alphanum        = alphas + numbers
)

// SpecVersion is the latest fully supported spec version of semver
var SpecVersion = Version{
	Major: 2,
	Minor: 0,
	Patch: 0,
}

// Version represents a semver compatible version
type Version struct {
	Major uint64
	Minor uint64
	Patch uint64
	Pre   []PRVersion
	Build []string //No Precedence
}

// Version to string
func (v Version) String() string {
	b := make([]byte, 0, 5)
	b = strconv.AppendUint(b, v.Major, 10)
	b = append(b, '.')
	b = strconv.AppendUint(b, v.Minor, 10)
	b = append(b, '.')
	b = strconv.AppendUint(b, v.Patch, 10)

	if len(v.Pre) > 0 {
		b = append(b, '-')
		b = append(b, v.Pre[0].String()...)

		for _, pre := range v.Pre[1:] {
			b = append(b, '.')
			b = append(b, pre.String()...)
		}
	}

	if len(v.Build) > 0 {
		b = append(b, '+')
		b = append(b, v.Build[0]...)

		for _, build := range v.Build[1:] {
			b = append(b, '.')
			b = append(b, build...)
		}
	}

	return string(b)
}

// Equals checks if v is equal to o.
func (v Version) Equals(o Version) bool {
	return (v.Compare(o) == 0)
}

// EQ checks if v is equal to o.
func (v Version) EQ(o Version) bool {
	return (v.Compare(o) == 0)
}

// NE checks if v is not equal to o.
func (v Version) NE(o Version) bool {
	return (v.Compare(o) != 0)
}

// GT checks if v is greater than o.
func (v Version) GT(o Version) bool {
	return (v.Compare(o) == 1)
}

// GTE checks if v is greater than or equal to o.
func (v Version) GTE(o Version) bool {
	return (v.Compare(o) >= 0)
}

// GE checks if v is greater than or equal to o.
func (v Version) GE(o Version) bool {
	return (v.Compare(o) >= 0)
}

// LT checks if v is less than o.
func (v Version) LT(o Version) bool {
	return (v.Compare(o) == -1)
}

// LTE checks if v is less than or equal to o.
func (v Version) LTE(o Version) bool {
	return (v.Compare(o) <= 0)
}

// LE checks if v is less than or equal to o.
func (v Version) LE(o Version) bool {
	return (v.Compare(o) <= 0)
}

// Compare compares Versions v to o:
// -1 == v is less than o
// 0 == v is equal to o
// 1 == v is greater than o
func (v Version) Compare(o Version) int {
	if v.Major != o.Major {
		if v.Major > o.Major {
			return 1
		}
		return -1
	}
	if v.Minor != o.Minor {
		if v.Minor > o.Minor {
			return 1
		}
		return -1
	}
	if v.Patch != o.Patch {
		if v.Patch > o.Patch {
			return 1
		}
		return -1
	}

	// Quick comparison if a version has no prerelease versions
	if len(v.Pre) == 0 && len(o.Pre) == 0 {
		return 0
	} else if len(v.Pre) == 0 && len(o.Pre) > 0 {
		return 1
	} else if len(v.Pre) > 0 && len(o.Pre) == 0 {
		return -1
	}

	i := 0
	for ; i < len(v.Pre) && i < len(o.Pre); i++ {
		if comp := v.Pre[i].Compare(o.Pre[i]); comp == 0 {
			continue
		} else if comp == 1 {
			return 1
		} else {
			return -1
		}
	}

	// If all pr versions are the equal but one has further prversion, this one greater
	if i == len(v.Pre) && i == len(o.Pre) {
		return 0
	} else if i == len(v.Pre) && i < len(o.Pre) {
		return -1
	} else {
		return 1
	}

}

// IncrementPatch increments the patch version
func (v *Version) IncrementPatch() error {
	v.Patch++
	return nil
}

// IncrementMinor increments the minor version
func (v *Version) IncrementMinor() error {
	v.Minor++
	v.Patch = 0
	return nil
}

// IncrementMajor increments the major version
func (v *Version) IncrementMajor() error {
	v.Major++
	v.Minor = 0
	v.Patch = 0
	return nil
}

// Validate validates v and returns error in case
func (v Version) Validate() error {
	// Major, Minor, Patch already validated using uint64

	for _, pre := range v.Pre {
		if !pre.IsNum { //Numeric prerelease versions already uint64
			if len(pre.VersionStr) == 0 {
				return fmt.Errorf("Prerelease can not be empty %q", pre.VersionStr)
			}
			if !containsOnly(pre.VersionStr, alphanum) {
				return fmt.Errorf("Invalid character(s) found in prerelease %q", pre.VersionStr)
			}
		}
	}

	for _, build := range v.Build {
		if len(build) == 0 {
			return fmt.Errorf("Build meta data can not be empty %q", build)
		}
		if !containsOnly(build, alphanum) {
			return fmt.Errorf("Invalid character(s) found in build meta data %q", build)
		}
	}

	return nil
}

// New is an alias for Parse and returns a pointer, parses version string and returns a validated Version or error
func New(s string) (*Version, error) {
	v, err := Parse(s)
	vp := &v
	return vp, err
}

// Make is an alias for Parse, parses version string and returns a validated Version or error
func Make(s string) (Version, error) {
	return Parse(s)
}

// ParseTolerant allows for certain version specifications that do not strictly adhere to semver
// specs to be parsed by this library. It does so by normalizing versions before passing them to
// Parse(). It currently trims spaces, removes a "v" prefix, adds a 0 patch number to versions
// with only major and minor components specified, and removes leading 0s.
func ParseTolerant(s string) (Version, error) {
	s = strings.TrimSpace(s)
	s = strings.TrimPrefix(s, "v")

	// Split into major.minor.(patch+pr+meta)
	parts := strings.SplitN(s, ".", 3)
	// Remove leading zeros.
	for i, p := range parts {
		if len(p) > 1 {
			p = strings.TrimLeft(p, "0")
			if len(p) == 0 || !strings.ContainsAny(p[0:1], "0123456789") {
				p = "0" + p
			}
			parts[i] = p
		}
	}
	// Fill up shortened versions.
	if len(parts) < 3 {
		if strings.ContainsAny(parts[len(parts)-1], "+-") {
			return Version{}, errors.New("Short version cannot contain PreRelease/Build meta data")
		}
		for len(parts) < 3 {
			parts = append(parts, "0")
		}
	}
	s = strings.Join(parts, ".")

	return Parse(s)
}

// Parse parses version string and returns a validated Version or error
func Parse(s string) (Version, error) {
	if len(s) == 0 {
		return Version{}, errors.New("Version string empty")
	}

	// Split into major.minor.(patch+pr+meta)
	parts := strings.SplitN(s, ".", 3)
	if len(parts) != 3 {
		return Version{}, errors.New("No Major.Minor.Patch elements found")
	}

	// Major
	if !containsOnly(parts[0], numbers) {
		return Version{}, fmt.Errorf("Invalid character(s) found in major number %q", parts[0])
	}
	if hasLeadingZeroes(parts[0]) {
		return Version{}, fmt.Errorf("Major number must not contain leading zeroes %q", parts[0])
	}
	major, err := strconv.ParseUint(parts[0], 10, 64)
	if err != nil {
		return Version{}, err
	}

	// Minor
	if !containsOnly(parts[1], numbers) {
		return Version{}, fmt.Errorf("Invalid character(s) found in minor number %q", parts[1])
	}
	if hasLeadingZeroes(parts[1]) {
		return Version{}, fmt.Errorf("Minor number must not contain leading zeroes %q", parts[1])
	}
	minor, err := strconv.ParseUint(parts[1], 10, 64)
	if err != nil {
		return Version{}, err
	}

	v := Version{}
	v.Major = major
	v.Minor = minor

	var build, prerelease []string
	patchStr := parts[2]

	if buildIndex := strings.IndexRune(patchStr, '+'); buildIndex != -1 {
		build = strings.Split(patchStr[buildIndex+1:], ".")
		patchStr = patchStr[:buildIndex]
	}

	if preIndex := strings.IndexRune(patchStr, '-'); preIndex != -1 {
		prerelease = strings.Split(patchStr[preIndex+1:], ".")
		patchStr = patchStr[:preIndex]
	}

	if !containsOnly(patchStr, numbers) {
		return Version{}, fmt.Errorf("Invalid character(s) found in patch number %q", patchStr)
	}
	if hasLeadingZeroes(patchStr) {
		return Version{}, fmt.Errorf("Patch number must not contain leading zeroes %q", patchStr)
	}
	patch, err := strconv.ParseUint(patchStr, 10, 64)
	if err != nil {
		return Version{}, err
	}

	v.Patch = patch

	// Prerelease
	for _, prstr := range prerelease {
		parsedPR, err := NewPRVersion(prstr)
		if err != nil {
			return Version{}, err
		}
		v.Pre = append(v.Pre, parsedPR)
	}

	// Build meta data
	for _, str := range build {
		if len(str) == 0 {
			return Version{}, errors.New("Build meta data is empty")
		}
		if !containsOnly(str, alphanum) {
			return Version{}, fmt.Errorf("Invalid character(s) found in build meta data %q", str)
		}
		v.Build = append(v.Build, str)
	}

	return v, nil
}

// MustParse is like Parse but panics if the version cannot be parsed.
func MustParse(s string) Version {
	v, err := Parse(s)
	if err != nil {
		panic(`semver: Parse(` + s + `): ` + err.Error())
	}
	return v
}

// PRVersion represents a PreRelease Version
type PRVersion struct {
	VersionStr string
	VersionNum uint64
	IsNum      bool
}

// NewPRVersion creates a new valid prerelease version
func NewPRVersion(s string) (PRVersion, error) {
	if len(s) == 0 {
		return PRVersion{}, errors.New("Prerelease is empty")
	}
	v := PRVersion{}
	if containsOnly(s, numbers) {
		if hasLeadingZeroes(s) {
			return PRVersion{}, fmt.Errorf("Numeric PreRelease version must not contain leading zeroes %q", s)
		}
		num, err := strconv.ParseUint(s, 10, 64)

		// Might never be hit, but just in case
		if err != nil {
			return PRVersion{}, err
		}
		v.VersionNum = num
		v.IsNum = true
	} else if containsOnly(s, alphanum) {
		v.VersionStr = s
		v.IsNum = false
	} else {
		return PRVersion{}, fmt.Errorf("Invalid character(s) found in prerelease %q", s)
	}
	return v, nil
}

// IsNumeric checks if prerelease-version is numeric
func (v PRVersion) IsNumeric() bool {
	return v.IsNum
}

// Compare compares two PreRelease Versions v and o:
// -1 == v is less than o
// 0 == v is equal to o
// 1 == v is greater than o
func (v PRVersion) Compare(o PRVersion) int {
	if v.IsNum && !o.IsNum {
		return -1
	} else if !v.IsNum && o.IsNum {
		return 1
	} else if v.IsNum && o.IsNum {
		if v.VersionNum == o.VersionNum {
			return 0
		} else if v.VersionNum > o.VersionNum {
			return 1
		} else {
			return -1
		}
	} else { // both are Alphas
		if v.VersionStr == o.VersionStr {
			return 0
		} else if v.VersionStr > o.VersionStr {
			return 1
		} else {
			return -1
		}
	}
}

// PreRelease version to string
func (v PRVersion) String() string {
	if v.IsNum {
		return strconv.FormatUint(v.VersionNum, 10)
	}
	return v.VersionStr
}

func containsOnly(s string, set string) bool {
	return strings.IndexFunc(s, func(r rune) bool {
		return !strings.ContainsRune(set, r)
	}) == -1
}

func hasLeadingZeroes(s string) bool {
	return len(s) > 1 && s[0] == '0'
}

// NewBuildVersion creates a new valid build version
func NewBuildVersion(s string) (string, error) {
	if len(s) == 0 {
		return "", errors.New("Buildversion is empty")
	}
	if !containsOnly(s, alphanum) {
		return "", fmt.Errorf("Invalid character(s) found in build meta data %q", s)
	}
	return s, nil
}
<|endoftext|>


<commit_before>package hostess_test

import (
	"fmt"
	"runtime"
	"strings"
	"testing"

	"github.com/cbednarski/hostess"
)

const ipv4Pass = `
127.0.0.1
127.0.1.1
10.200.30.50
99.99.99.99
999.999.999.999
0.1.1.0
`

const ipv4Fail = `
1234.1.1.1
123.5.6
12.12
76.76.67.67.45
`

const ipv6 = ``
const domain = "localhost"
const ip = "127.0.0.1"
const enabled = true

func Diff(expected, actual string) string {
	return fmt.Sprintf(`
---- Expected ----
%s
----- Actual -----
%s
`, expected, actual)
}

func TestGetHostsPath(t *testing.T) {
	path := hostess.GetHostsPath()
	const expected = "/etc/hosts"
	if path != expected {
		t.Error("Hosts path should be " + expected)
	}
}

func TestFormatHostfile(t *testing.T) {
	// The sort order here is a bit weird.
	// 1. We want localhost entries at the top
	// 2. The rest are sorted by IP as STRINGS, not numeric values, so 10
	//    precedes 8
	const expected = `127.0.0.1 localhost devsite
127.0.1.1 ip-10-37-12-18
# 8.8.8.8 google.com
10.37.12.18 devsite.com m.devsite.com
`

	hostfile := hostess.NewHostfile()
	hostfile.Path = "./hosts"
	hostfile.Hosts.Add(hostess.NewHostname("localhost", "127.0.0.1", true))
	hostfile.Hosts.Add(hostess.NewHostname("ip-10-37-12-18", "127.0.1.1", true))
	hostfile.Hosts.Add(hostess.NewHostname("devsite", "127.0.0.1", true))
	hostfile.Hosts.Add(hostess.NewHostname("google.com", "8.8.8.8", false))
	hostfile.Hosts.Add(hostess.NewHostname("devsite.com", "10.37.12.18", true))
	hostfile.Hosts.Add(hostess.NewHostname("m.devsite.com", "10.37.12.18", true))
	f := string(hostfile.Format())
	if f != expected {
		t.Errorf("Hostfile output is not formatted correctly: %s", Diff(expected, f))
	}
}

func TestTrimWS(t *testing.T) {
	const expected = `  candy

	`
	actual := hostess.TrimWS(expected)
	if actual != "candy" {
		t.Errorf("Output was not trimmed correctly: %s", Diff(expected, actual))
	}
}

func TestParseLine(t *testing.T) {
	var hosts hostess.Hostlist

	// Blank line
	hosts = hostess.ParseLine("")
	if len(hosts) > 0 {
		t.Error("Expected to find zero hostnames")
	}

	// Comment
	hosts = hostess.ParseLine("# The following lines are desirable for IPv6 capable hosts")
	if len(hosts) > 0 {
		t.Error("Expected to find zero hostnames")
	}

	// Single word comment
	hosts = hostess.ParseLine("#blah")
	if len(hosts) > 0 {
		t.Error("Expected to find zero hostnames")
	}

	hosts = hostess.ParseLine("#66.33.99.11              test.domain.com")
	if !hosts.Contains(hostess.NewHostname("test.domain.com", "66.33.99.11", false)) ||
		len(hosts) != 1 {
		t.Error("Expected to find test.domain.com (disabled)")
	}

	hosts = hostess.ParseLine("#  66.33.99.11	test.domain.com	domain.com")
	if !hosts.Contains(hostess.NewHostname("test.domain.com", "66.33.99.11", false)) ||
		!hosts.Contains(hostess.NewHostname("domain.com", "66.33.99.11", false)) ||
		len(hosts) != 2 {
		t.Error("Expected to find domain.com and test.domain.com (disabled)")
		t.Errorf("Found %s", hosts)
	}

	// Not Commented stuff
	hosts = hostess.ParseLine("255.255.255.255 broadcasthost test.domain.com	domain.com")
	if !hosts.Contains(hostess.NewHostname("broadcasthost", "255.255.255.255", true)) ||
		!hosts.Contains(hostess.NewHostname("test.domain.com", "255.255.255.255", true)) ||
		!hosts.Contains(hostess.NewHostname("domain.com", "255.255.255.255", true)) ||
		len(hosts) != 3 {
		t.Error("Expected to find broadcasthost, domain.com, and test.domain.com (enabled)")
	}

	// Ipv6 stuff
	hosts = hostess.ParseLine("::1             localhost")
	if !hosts.Contains(hostess.NewHostname("localhost", "::1", true)) ||
		len(hosts) != 1 {
		t.Error("Expected to find localhost ipv6 (enabled)")
	}

	hosts = hostess.ParseLine("ff02::1 ip6-allnodes")
	if !hosts.Contains(hostess.NewHostname("ip6-allnodes", "ff02::1", true)) ||
		len(hosts) != 1 {
		t.Error("Expected to find ip6-allnodes ipv6 (enabled)")
	}
}

func TestLoadHostfile(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Skip("Default host file is empty on Windows")
	}
	hostfile := hostess.NewHostfile()
	hostfile.Read()
	if !strings.Contains(string(hostfile.GetData()), domain) {
		t.Errorf("Expected to find %s", domain)
	}

	hostfile.Parse()
	hostname := hostess.NewHostname(domain, ip, enabled)
	found := hostfile.Hosts.Contains(hostname)
	if !found {
		t.Errorf("Expected to find %#v", hostname)
	}
}
<commit_msg>Updated hosts path test for windows<commit_after>package hostess_test

import (
	"fmt"
	"runtime"
	"strings"
	"testing"

	"github.com/cbednarski/hostess"
)

const ipv4Pass = `
127.0.0.1
127.0.1.1
10.200.30.50
99.99.99.99
999.999.999.999
0.1.1.0
`

const ipv4Fail = `
1234.1.1.1
123.5.6
12.12
76.76.67.67.45
`

const ipv6 = ``
const domain = "localhost"
const ip = "127.0.0.1"
const enabled = true

func Diff(expected, actual string) string {
	return fmt.Sprintf(`
---- Expected ----
%s
----- Actual -----
%s
`, expected, actual)
}

func TestGetHostsPath(t *testing.T) {
	path := hostess.GetHostsPath()
	var expected string
	if runtime.GOOS == "windows" {
		expected = "C:\\Windows\\System32\\drivers\\etc\\hosts"
	} else {
		expected = "/etc/hosts"
	}
	if path != expected {
		t.Error("Hosts path should be " + expected)
	}
}

func TestFormatHostfile(t *testing.T) {
	// The sort order here is a bit weird.
	// 1. We want localhost entries at the top
	// 2. The rest are sorted by IP as STRINGS, not numeric values, so 10
	//    precedes 8
	const expected = `127.0.0.1 localhost devsite
127.0.1.1 ip-10-37-12-18
# 8.8.8.8 google.com
10.37.12.18 devsite.com m.devsite.com
`

	hostfile := hostess.NewHostfile()
	hostfile.Path = "./hosts"
	hostfile.Hosts.Add(hostess.NewHostname("localhost", "127.0.0.1", true))
	hostfile.Hosts.Add(hostess.NewHostname("ip-10-37-12-18", "127.0.1.1", true))
	hostfile.Hosts.Add(hostess.NewHostname("devsite", "127.0.0.1", true))
	hostfile.Hosts.Add(hostess.NewHostname("google.com", "8.8.8.8", false))
	hostfile.Hosts.Add(hostess.NewHostname("devsite.com", "10.37.12.18", true))
	hostfile.Hosts.Add(hostess.NewHostname("m.devsite.com", "10.37.12.18", true))
	f := string(hostfile.Format())
	if f != expected {
		t.Errorf("Hostfile output is not formatted correctly: %s", Diff(expected, f))
	}
}

func TestTrimWS(t *testing.T) {
	const expected = `  candy

	`
	actual := hostess.TrimWS(expected)
	if actual != "candy" {
		t.Errorf("Output was not trimmed correctly: %s", Diff(expected, actual))
	}
}

func TestParseLine(t *testing.T) {
	var hosts hostess.Hostlist

	// Blank line
	hosts = hostess.ParseLine("")
	if len(hosts) > 0 {
		t.Error("Expected to find zero hostnames")
	}

	// Comment
	hosts = hostess.ParseLine("# The following lines are desirable for IPv6 capable hosts")
	if len(hosts) > 0 {
		t.Error("Expected to find zero hostnames")
	}

	// Single word comment
	hosts = hostess.ParseLine("#blah")
	if len(hosts) > 0 {
		t.Error("Expected to find zero hostnames")
	}

	hosts = hostess.ParseLine("#66.33.99.11              test.domain.com")
	if !hosts.Contains(hostess.NewHostname("test.domain.com", "66.33.99.11", false)) ||
		len(hosts) != 1 {
		t.Error("Expected to find test.domain.com (disabled)")
	}

	hosts = hostess.ParseLine("#  66.33.99.11	test.domain.com	domain.com")
	if !hosts.Contains(hostess.NewHostname("test.domain.com", "66.33.99.11", false)) ||
		!hosts.Contains(hostess.NewHostname("domain.com", "66.33.99.11", false)) ||
		len(hosts) != 2 {
		t.Error("Expected to find domain.com and test.domain.com (disabled)")
		t.Errorf("Found %s", hosts)
	}

	// Not Commented stuff
	hosts = hostess.ParseLine("255.255.255.255 broadcasthost test.domain.com	domain.com")
	if !hosts.Contains(hostess.NewHostname("broadcasthost", "255.255.255.255", true)) ||
		!hosts.Contains(hostess.NewHostname("test.domain.com", "255.255.255.255", true)) ||
		!hosts.Contains(hostess.NewHostname("domain.com", "255.255.255.255", true)) ||
		len(hosts) != 3 {
		t.Error("Expected to find broadcasthost, domain.com, and test.domain.com (enabled)")
	}

	// Ipv6 stuff
	hosts = hostess.ParseLine("::1             localhost")
	if !hosts.Contains(hostess.NewHostname("localhost", "::1", true)) ||
		len(hosts) != 1 {
		t.Error("Expected to find localhost ipv6 (enabled)")
	}

	hosts = hostess.ParseLine("ff02::1 ip6-allnodes")
	if !hosts.Contains(hostess.NewHostname("ip6-allnodes", "ff02::1", true)) ||
		len(hosts) != 1 {
		t.Error("Expected to find ip6-allnodes ipv6 (enabled)")
	}
}

func TestLoadHostfile(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Skip("Default host file is empty on Windows")
	}
	hostfile := hostess.NewHostfile()
	hostfile.Read()
	if !strings.Contains(string(hostfile.GetData()), domain) {
		t.Errorf("Expected to find %s", domain)
	}

	hostfile.Parse()
	hostname := hostess.NewHostname(domain, ip, enabled)
	found := hostfile.Hosts.Contains(hostname)
	if !found {
		t.Errorf("Expected to find %#v", hostname)
	}
}
<|endoftext|>


<commit_before>// Copyright 2017-2021 Jeff Foley. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package http

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/cookiejar"
	"net/http/httputil"
	"net/url"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/OWASP/Amass/v3/filter"
	amassnet "github.com/OWASP/Amass/v3/net"
	"github.com/OWASP/Amass/v3/net/dns"
	"github.com/PuerkitoBio/goquery"
	"github.com/caffix/stringset"
	"github.com/geziyor/geziyor"
	"github.com/geziyor/geziyor/client"
)

const (
	// Accept is the default HTTP Accept header value used by Amass.
	Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"

	// AcceptLang is the default HTTP Accept-Language header value used by Amass.
	AcceptLang = "en-US,en;q=0.8"

	httpTimeout      = 30 * time.Second
	handshakeTimeout = 5 * time.Second
)

var (
  // UserAgent is the default user agent used by Amass during HTTP requests.
	UserAgent       string
	subRE           = dns.AnySubdomainRegex()
	crawlRE         = regexp.MustCompile(`\.\w{2,6}($|\?|#)`)
	crawlFileEnds   = []string{"html", "do", "action", "cgi"}
	crawlFileStarts = []string{"js", "htm", "as", "php", "inc"}
	nameStripRE     = regexp.MustCompile(`^u[0-9a-f]{4}|20|22|25|2b|2f|3d|3a|40`)
)

// DefaultClient is the same HTTP client used by the package methods.
var DefaultClient *http.Client

// BasicAuth contains the data used for HTTP basic authentication.
type BasicAuth struct {
	Username string
	Password string
}

func init() {
	jar, _ := cookiejar.New(nil)
	DefaultClient = &http.Client{
		Timeout: httpTimeout,
		Transport: &http.Transport{
			Proxy:                 http.ProxyFromEnvironment,
			DialContext:           amassnet.DialContext,
			MaxIdleConns:          200,
			MaxConnsPerHost:       50,
			IdleConnTimeout:       90 * time.Second,
			TLSHandshakeTimeout:   handshakeTimeout,
			ExpectContinueTimeout: 10 * time.Second,
			TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
		},
		Jar: jar,
	}

	switch runtime.GOOS {
	case "windows":
		UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.164 Safari/537.36"
	case "darwin":
		UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.164 Safari/537.36"
	default:
		UserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.164 Safari/537.36"
	}
}

// CopyCookies copies cookies from one domain to another. Some of our data
// sources rely on shared auth tokens and this avoids sending extra requests
// to have the site reissue cookies for the other domains.
func CopyCookies(src string, dest string) {
	srcURL, _ := url.Parse(src)
	destURL, _ := url.Parse(dest)
	DefaultClient.Jar.SetCookies(destURL, DefaultClient.Jar.Cookies(srcURL))
}

// CheckCookie checks if a cookie exists in the cookie jar for a given host
func CheckCookie(urlString string, cookieName string) bool {
	cookieURL, _ := url.Parse(urlString)
	found := false
	for _, cookie := range DefaultClient.Jar.Cookies(cookieURL) {
		if cookie.Name == cookieName {
			found = true
			break
		}
	}
	return found
}

// RequestWebPage returns a string containing the entire response for the provided URL when successful.
func RequestWebPage(ctx context.Context, u string, body io.Reader, hvals map[string]string, auth *BasicAuth) (string, error) {
	method := "GET"
	if body != nil {
		method = "POST"
	}
	req, err := http.NewRequestWithContext(ctx, method, u, body)
	if err != nil {
		return "", err
	}
	if auth != nil && auth.Username != "" && auth.Password != "" {
		req.SetBasicAuth(auth.Username, auth.Password)
	}
	req.Header.Set("User-Agent", UserAgent)
	req.Header.Set("Accept", Accept)
	req.Header.Set("Accept-Language", AcceptLang)

	for k, v := range hvals {
		req.Header.Set(k, v)
	}

	resp, err := DefaultClient.Do(req)
	if err != nil {
		return "", err
	}

	in, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()

	if resp.StatusCode < 200 || resp.StatusCode >= 400 {
		err = errors.New(resp.Status)
	}
	return string(in), err
}

// Crawl will spider the web page at the URL argument looking for DNS names within the scope argument.
func Crawl(ctx context.Context, u string, scope []string, max int, f filter.Filter) ([]string, error) {
	select {
	case <-ctx.Done():
		return nil, fmt.Errorf("The context expired")
	default:
	}

	newScope := append([]string{}, scope...)

	target := subRE.FindString(u)
	if target != "" {
		var found bool
		for _, domain := range newScope {
			if target == domain {
				found = true
				break
			}
		}
		if !found {
			newScope = append(newScope, target)
		}
	}

	if f == nil {
		f = filter.NewStringFilter()
	}

	var count int
	var m sync.Mutex
	results := stringset.New()
	g := geziyor.NewGeziyor(&geziyor.Options{
		AllowedDomains:        newScope,
		StartURLs:             []string{u},
		Timeout:               5 * time.Minute,
		RobotsTxtDisabled:     true,
		UserAgent:             UserAgent,
		LogDisabled:           true,
		ConcurrentRequests:    5,
		RequestDelay:          750 * time.Millisecond,
		RequestDelayRandomize: true,
		ParseFunc: func(g *geziyor.Geziyor, r *client.Response) {
			resp, err := httputil.DumpResponse(interface{}(r).(*http.Response), true)
			if err == nil {
				for _, n := range subRE.FindAllString(string(resp), -1) {
					if name := CleanName(n); whichDomain(name, scope) != "" {
						m.Lock()
						results.Insert(name)
						m.Unlock()
					}
				}
			}

			processURL := func(u *url.URL) {
				// Attempt to save the name in our results
				m.Lock()
				results.Insert(u.Hostname())
				m.Unlock()

				if s := crawlFilterURLs(u, f); s != "" {
					// Be sure the crawl has not exceeded the maximum links to be followed
					m.Lock()
					count++
					current := count
					m.Unlock()
					if max <= 0 || current < max {
						g.Get(s, g.Opt.ParseFunc)
					}
				}
			}

			r.HTMLDoc.Find("a").Each(func(i int, s *goquery.Selection) {
				if href, ok := s.Attr("href"); ok {
					if u, err := r.JoinURL(href); err == nil && whichDomain(u.Hostname(), newScope) != "" {
						processURL(u)
					}
				}
			})

			r.HTMLDoc.Find("script").Each(func(i int, s *goquery.Selection) {
				if src, ok := s.Attr("src"); ok {
					if u, err := r.JoinURL(src); err == nil && whichDomain(u.Hostname(), newScope) != "" {
						processURL(u)
					}
				}
			})
		},
	})
	options := &client.Options{
		MaxBodySize:    100 * 1024 * 1024, // 100MB
		RetryTimes:     2,
		RetryHTTPCodes: []int{408, 500, 502, 503, 504, 522, 524},
	}
	g.Client = client.NewClient(options)
	g.Client.Client = http.DefaultClient

	done := make(chan struct{}, 2)
	go func() {
		g.Start()
		done <- struct{}{}
	}()

	var err error
	select {
	case <-ctx.Done():
		err = fmt.Errorf("The context expired during the crawl of %s", u)
	case <-done:
		if len(results.Slice()) == 0 {
			err = fmt.Errorf("No DNS names were discovered during the crawl of %s", u)
		}
	}

	return results.Slice(), err
}

func crawlFilterURLs(p *url.URL, f filter.Filter) string {
	// Check that the URL has an appropriate scheme for scraping
	if !p.IsAbs() || (p.Scheme != "http" && p.Scheme != "https") {
		return ""
	}
	// If the URL path has a file extension, check that it's of interest
	if ext := crawlRE.FindString(p.Path); ext != "" {
		ext = strings.TrimRight(ext, "?#")

		var found bool
		for _, s := range crawlFileStarts {
			if strings.HasPrefix(ext, "." + s) {
				found = true
				break
			}
		}
		for _, e := range crawlFileEnds {
			if strings.HasSuffix(ext, e) {
				found = true
				break
			}
		}
		if !found {
			return ""
		}
	}
	// Remove fragments and check if we've seen this URL before
	p.Fragment = ""
	p.RawFragment = ""
	if f.Duplicate(p.String()) {
		return ""
	}
	return p.String()
}

func whichDomain(name string, scope []string) string {
	n := strings.TrimSpace(name)

	for _, d := range scope {
		if strings.HasSuffix(n, d) {
			// fork made me do it :>
			nlen := len(n)
			dlen := len(d)
			// Check for exact match first to guard against out of bound index
			if nlen == dlen || n[nlen-dlen-1] == '.' {
				return d
			}
		}
	}
	return ""
}

// PullCertificateNames attempts to pull a cert from one or more ports on an IP.
func PullCertificateNames(ctx context.Context, addr string, ports []int) []string {
	var names []string

	// Check hosts for certificates that contain subdomain names
	for _, port := range ports {
		select {
		case <-ctx.Done():
			return names
		default:
		}

		// Set the maximum time allowed for making the connection
		tCtx, cancel := context.WithTimeout(ctx, handshakeTimeout)
		defer cancel()
		// Obtain the connection
		conn, err := amassnet.DialContext(tCtx, "tcp", net.JoinHostPort(addr, strconv.Itoa(port)))
		if err != nil {
			continue
		}
		defer conn.Close()

		c := tls.Client(conn, &tls.Config{InsecureSkipVerify: true})
		// Attempt to acquire the certificate chain
		errChan := make(chan error, 2)
		go func() {
			errChan <- c.Handshake()
		}()

		t := time.NewTimer(handshakeTimeout)
		select {
		case <-t.C:
			err = errors.New("Handshake timeout")
		case e := <-errChan:
			err = e
		}
		t.Stop()

		if err != nil {
			continue
		}
		// Get the correct certificate in the chain
		certChain := c.ConnectionState().PeerCertificates
		cert := certChain[0]
		// Create the new requests from names found within the cert
		names = append(names, namesFromCert(cert)...)
	}

	return names
}

func namesFromCert(cert *x509.Certificate) []string {
	var cn string

	for _, name := range cert.Subject.Names {
		oid := name.Type
		if len(oid) == 4 && oid[0] == 2 && oid[1] == 5 && oid[2] == 4 {
			if oid[3] == 3 {
				cn = fmt.Sprintf("%s", name.Value)
				break
			}
		}
	}

	subdomains := stringset.New()
	// Add the subject common name to the list of subdomain names
	commonName := dns.RemoveAsteriskLabel(cn)
	if commonName != "" {
		subdomains.Insert(commonName)
	}
	// Add the cert DNS names to the list of subdomain names
	for _, name := range cert.DNSNames {
		n := dns.RemoveAsteriskLabel(name)
		if n != "" {
			subdomains.Insert(n)
		}
	}
	return subdomains.Slice()
}

// ClientCountryCode returns the country code for the public-facing IP address for the host of the process.
func ClientCountryCode(ctx context.Context) string {
	headers := map[string]string{"Content-Type": "application/json"}

	page, err := RequestWebPage(ctx, "https://ipapi.co/json", nil, headers, nil)
	if err != nil {
		return ""
	}

	// Extract the country code from the REST API results
	var ipinfo struct {
		CountryCode string `json:"country"`
	}

	if err := json.Unmarshal([]byte(page), &ipinfo); err != nil {
		return ""
	}
	return strings.ToLower(ipinfo.CountryCode)
}

// CleanName will clean up the names scraped from the web.
func CleanName(name string) string {
	var err error

	name, err = strconv.Unquote("\"" + strings.TrimSpace(name) + "\"")
	if err == nil {
		name = subRE.FindString(name)
	}

	name = strings.ToLower(name)
	for {
		name = strings.Trim(name, "-.")

		if i := nameStripRE.FindStringIndex(name); i != nil {
			name = name[i[1]:]
		} else {
			break
		}
	}

	return name
}
<commit_msg>Added more strip regex<commit_after>// Copyright 2017-2021 Jeff Foley. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package http

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/cookiejar"
	"net/http/httputil"
	"net/url"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/OWASP/Amass/v3/filter"
	amassnet "github.com/OWASP/Amass/v3/net"
	"github.com/OWASP/Amass/v3/net/dns"
	"github.com/PuerkitoBio/goquery"
	"github.com/caffix/stringset"
	"github.com/geziyor/geziyor"
	"github.com/geziyor/geziyor/client"
)

const (
	// Accept is the default HTTP Accept header value used by Amass.
	Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"

	// AcceptLang is the default HTTP Accept-Language header value used by Amass.
	AcceptLang = "en-US,en;q=0.8"

	httpTimeout      = 30 * time.Second
	handshakeTimeout = 5 * time.Second
)

var (
  // UserAgent is the default user agent used by Amass during HTTP requests.
	UserAgent       string
	subRE           = dns.AnySubdomainRegex()
	crawlRE         = regexp.MustCompile(`\.\w{2,6}($|\?|#)`)
	crawlFileEnds   = []string{"html", "do", "action", "cgi"}
	crawlFileStarts = []string{"js", "htm", "as", "php", "inc"}
	nameStripRE     = regexp.MustCompile(`^u[0-9a-f]{4}|20|22|25|27|2b|2f|3d|3a|40`)
)

// DefaultClient is the same HTTP client used by the package methods.
var DefaultClient *http.Client

// BasicAuth contains the data used for HTTP basic authentication.
type BasicAuth struct {
	Username string
	Password string
}

func init() {
	jar, _ := cookiejar.New(nil)
	DefaultClient = &http.Client{
		Timeout: httpTimeout,
		Transport: &http.Transport{
			Proxy:                 http.ProxyFromEnvironment,
			DialContext:           amassnet.DialContext,
			MaxIdleConns:          200,
			MaxConnsPerHost:       50,
			IdleConnTimeout:       90 * time.Second,
			TLSHandshakeTimeout:   handshakeTimeout,
			ExpectContinueTimeout: 10 * time.Second,
			TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
		},
		Jar: jar,
	}

	switch runtime.GOOS {
	case "windows":
		UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.164 Safari/537.36"
	case "darwin":
		UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.164 Safari/537.36"
	default:
		UserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.164 Safari/537.36"
	}
}

// CopyCookies copies cookies from one domain to another. Some of our data
// sources rely on shared auth tokens and this avoids sending extra requests
// to have the site reissue cookies for the other domains.
func CopyCookies(src string, dest string) {
	srcURL, _ := url.Parse(src)
	destURL, _ := url.Parse(dest)
	DefaultClient.Jar.SetCookies(destURL, DefaultClient.Jar.Cookies(srcURL))
}

// CheckCookie checks if a cookie exists in the cookie jar for a given host
func CheckCookie(urlString string, cookieName string) bool {
	cookieURL, _ := url.Parse(urlString)
	found := false
	for _, cookie := range DefaultClient.Jar.Cookies(cookieURL) {
		if cookie.Name == cookieName {
			found = true
			break
		}
	}
	return found
}

// RequestWebPage returns a string containing the entire response for the provided URL when successful.
func RequestWebPage(ctx context.Context, u string, body io.Reader, hvals map[string]string, auth *BasicAuth) (string, error) {
	method := "GET"
	if body != nil {
		method = "POST"
	}
	req, err := http.NewRequestWithContext(ctx, method, u, body)
	if err != nil {
		return "", err
	}
	if auth != nil && auth.Username != "" && auth.Password != "" {
		req.SetBasicAuth(auth.Username, auth.Password)
	}
	req.Header.Set("User-Agent", UserAgent)
	req.Header.Set("Accept", Accept)
	req.Header.Set("Accept-Language", AcceptLang)

	for k, v := range hvals {
		req.Header.Set(k, v)
	}

	resp, err := DefaultClient.Do(req)
	if err != nil {
		return "", err
	}

	in, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()

	if resp.StatusCode < 200 || resp.StatusCode >= 400 {
		err = errors.New(resp.Status)
	}
	return string(in), err
}

// Crawl will spider the web page at the URL argument looking for DNS names within the scope argument.
func Crawl(ctx context.Context, u string, scope []string, max int, f filter.Filter) ([]string, error) {
	select {
	case <-ctx.Done():
		return nil, fmt.Errorf("The context expired")
	default:
	}

	newScope := append([]string{}, scope...)

	target := subRE.FindString(u)
	if target != "" {
		var found bool
		for _, domain := range newScope {
			if target == domain {
				found = true
				break
			}
		}
		if !found {
			newScope = append(newScope, target)
		}
	}

	if f == nil {
		f = filter.NewStringFilter()
	}

	var count int
	var m sync.Mutex
	results := stringset.New()
	g := geziyor.NewGeziyor(&geziyor.Options{
		AllowedDomains:        newScope,
		StartURLs:             []string{u},
		Timeout:               5 * time.Minute,
		RobotsTxtDisabled:     true,
		UserAgent:             UserAgent,
		LogDisabled:           true,
		ConcurrentRequests:    5,
		RequestDelay:          750 * time.Millisecond,
		RequestDelayRandomize: true,
		ParseFunc: func(g *geziyor.Geziyor, r *client.Response) {
			resp, err := httputil.DumpResponse(interface{}(r).(*http.Response), true)
			if err == nil {
				for _, n := range subRE.FindAllString(string(resp), -1) {
					if name := CleanName(n); whichDomain(name, scope) != "" {
						m.Lock()
						results.Insert(name)
						m.Unlock()
					}
				}
			}

			processURL := func(u *url.URL) {
				// Attempt to save the name in our results
				m.Lock()
				results.Insert(u.Hostname())
				m.Unlock()

				if s := crawlFilterURLs(u, f); s != "" {
					// Be sure the crawl has not exceeded the maximum links to be followed
					m.Lock()
					count++
					current := count
					m.Unlock()
					if max <= 0 || current < max {
						g.Get(s, g.Opt.ParseFunc)
					}
				}
			}

			r.HTMLDoc.Find("a").Each(func(i int, s *goquery.Selection) {
				if href, ok := s.Attr("href"); ok {
					if u, err := r.JoinURL(href); err == nil && whichDomain(u.Hostname(), newScope) != "" {
						processURL(u)
					}
				}
			})

			r.HTMLDoc.Find("script").Each(func(i int, s *goquery.Selection) {
				if src, ok := s.Attr("src"); ok {
					if u, err := r.JoinURL(src); err == nil && whichDomain(u.Hostname(), newScope) != "" {
						processURL(u)
					}
				}
			})
		},
	})
	options := &client.Options{
		MaxBodySize:    100 * 1024 * 1024, // 100MB
		RetryTimes:     2,
		RetryHTTPCodes: []int{408, 500, 502, 503, 504, 522, 524},
	}
	g.Client = client.NewClient(options)
	g.Client.Client = http.DefaultClient

	done := make(chan struct{}, 2)
	go func() {
		g.Start()
		done <- struct{}{}
	}()

	var err error
	select {
	case <-ctx.Done():
		err = fmt.Errorf("The context expired during the crawl of %s", u)
	case <-done:
		if len(results.Slice()) == 0 {
			err = fmt.Errorf("No DNS names were discovered during the crawl of %s", u)
		}
	}

	return results.Slice(), err
}

func crawlFilterURLs(p *url.URL, f filter.Filter) string {
	// Check that the URL has an appropriate scheme for scraping
	if !p.IsAbs() || (p.Scheme != "http" && p.Scheme != "https") {
		return ""
	}
	// If the URL path has a file extension, check that it's of interest
	if ext := crawlRE.FindString(p.Path); ext != "" {
		ext = strings.TrimRight(ext, "?#")

		var found bool
		for _, s := range crawlFileStarts {
			if strings.HasPrefix(ext, "." + s) {
				found = true
				break
			}
		}
		for _, e := range crawlFileEnds {
			if strings.HasSuffix(ext, e) {
				found = true
				break
			}
		}
		if !found {
			return ""
		}
	}
	// Remove fragments and check if we've seen this URL before
	p.Fragment = ""
	p.RawFragment = ""
	if f.Duplicate(p.String()) {
		return ""
	}
	return p.String()
}

func whichDomain(name string, scope []string) string {
	n := strings.TrimSpace(name)

	for _, d := range scope {
		if strings.HasSuffix(n, d) {
			// fork made me do it :>
			nlen := len(n)
			dlen := len(d)
			// Check for exact match first to guard against out of bound index
			if nlen == dlen || n[nlen-dlen-1] == '.' {
				return d
			}
		}
	}
	return ""
}

// PullCertificateNames attempts to pull a cert from one or more ports on an IP.
func PullCertificateNames(ctx context.Context, addr string, ports []int) []string {
	var names []string

	// Check hosts for certificates that contain subdomain names
	for _, port := range ports {
		select {
		case <-ctx.Done():
			return names
		default:
		}

		// Set the maximum time allowed for making the connection
		tCtx, cancel := context.WithTimeout(ctx, handshakeTimeout)
		defer cancel()
		// Obtain the connection
		conn, err := amassnet.DialContext(tCtx, "tcp", net.JoinHostPort(addr, strconv.Itoa(port)))
		if err != nil {
			continue
		}
		defer conn.Close()

		c := tls.Client(conn, &tls.Config{InsecureSkipVerify: true})
		// Attempt to acquire the certificate chain
		errChan := make(chan error, 2)
		go func() {
			errChan <- c.Handshake()
		}()

		t := time.NewTimer(handshakeTimeout)
		select {
		case <-t.C:
			err = errors.New("Handshake timeout")
		case e := <-errChan:
			err = e
		}
		t.Stop()

		if err != nil {
			continue
		}
		// Get the correct certificate in the chain
		certChain := c.ConnectionState().PeerCertificates
		cert := certChain[0]
		// Create the new requests from names found within the cert
		names = append(names, namesFromCert(cert)...)
	}

	return names
}

func namesFromCert(cert *x509.Certificate) []string {
	var cn string

	for _, name := range cert.Subject.Names {
		oid := name.Type
		if len(oid) == 4 && oid[0] == 2 && oid[1] == 5 && oid[2] == 4 {
			if oid[3] == 3 {
				cn = fmt.Sprintf("%s", name.Value)
				break
			}
		}
	}

	subdomains := stringset.New()
	// Add the subject common name to the list of subdomain names
	commonName := dns.RemoveAsteriskLabel(cn)
	if commonName != "" {
		subdomains.Insert(commonName)
	}
	// Add the cert DNS names to the list of subdomain names
	for _, name := range cert.DNSNames {
		n := dns.RemoveAsteriskLabel(name)
		if n != "" {
			subdomains.Insert(n)
		}
	}
	return subdomains.Slice()
}

// ClientCountryCode returns the country code for the public-facing IP address for the host of the process.
func ClientCountryCode(ctx context.Context) string {
	headers := map[string]string{"Content-Type": "application/json"}

	page, err := RequestWebPage(ctx, "https://ipapi.co/json", nil, headers, nil)
	if err != nil {
		return ""
	}

	// Extract the country code from the REST API results
	var ipinfo struct {
		CountryCode string `json:"country"`
	}

	if err := json.Unmarshal([]byte(page), &ipinfo); err != nil {
		return ""
	}
	return strings.ToLower(ipinfo.CountryCode)
}

// CleanName will clean up the names scraped from the web.
func CleanName(name string) string {
	var err error

	name, err = strconv.Unquote("\"" + strings.TrimSpace(name) + "\"")
	if err == nil {
		name = subRE.FindString(name)
	}

	name = strings.ToLower(name)
	for {
		name = strings.Trim(name, "-.")

		if i := nameStripRE.FindStringIndex(name); i != nil {
			name = name[i[1]:]
		} else {
			break
		}
	}

	return name
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package watch

import (
	"context"
	"errors"
	"fmt"
	"io"
	"net/http"
	"time"

	"github.com/davecgh/go-spew/spew"

	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/net"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/tools/cache"
	"k8s.io/klog"
)

// resourceVersionGetter is an interface used to get resource version from events.
// We can't reuse an interface from meta otherwise it would be a cyclic dependency and we need just this one method
type resourceVersionGetter interface {
	GetResourceVersion() string
}

// RetryWatcher will make sure that in case the underlying watcher is closed (e.g. due to API timeout or etcd timeout)
// it will get restarted from the last point without the consumer even knowing about it.
// RetryWatcher does that by inspecting events and keeping track of resourceVersion.
// Especially useful when using watch.UntilWithoutRetry where premature termination is causing issues and flakes.
// Please note that this is not resilient to etcd cache not having the resource version anymore - you would need to
// use Informers for that.
type RetryWatcher struct {
	lastResourceVersion string
	watcherClient       cache.Watcher
	resultChan          chan watch.Event
	stopChan            chan struct{}
	doneChan            chan struct{}
	minRestartDelay     time.Duration
}

// NewRetryWatcher creates a new RetryWatcher.
// It will make sure that watches gets restarted in case of recoverable errors.
// The initialResourceVersion will be given to watch method when first called.
func NewRetryWatcher(initialResourceVersion string, watcherClient cache.Watcher) (*RetryWatcher, error) {
	return newRetryWatcher(initialResourceVersion, watcherClient, 1*time.Second)
}

func newRetryWatcher(initialResourceVersion string, watcherClient cache.Watcher, minRestartDelay time.Duration) (*RetryWatcher, error) {
	switch initialResourceVersion {
	case "", "0":
		// TODO: revisit this if we ever get WATCH v2 where it means start "now"
		//       without doing the synthetic list of objects at the beginning (see #74022)
		return nil, fmt.Errorf("initial RV %q is not supported due to issues with underlying WATCH", initialResourceVersion)
	default:
		break
	}

	rw := &RetryWatcher{
		lastResourceVersion: initialResourceVersion,
		watcherClient:       watcherClient,
		stopChan:            make(chan struct{}),
		doneChan:            make(chan struct{}),
		resultChan:          make(chan watch.Event, 0),
		minRestartDelay:     minRestartDelay,
	}

	go rw.receive()
	return rw, nil
}

func (rw *RetryWatcher) send(event watch.Event) bool {
	// Writing to an unbuffered channel is blocking operation
	// and we need to check if stop wasn't requested while doing so.
	select {
	case rw.resultChan <- event:
		return true
	case <-rw.stopChan:
		return false
	}
}

// doReceive returns true when it is done, false otherwise.
// If it is not done the second return value holds the time to wait before calling it again.
func (rw *RetryWatcher) doReceive() (bool, time.Duration) {
	watcher, err := rw.watcherClient.Watch(metav1.ListOptions{
		ResourceVersion: rw.lastResourceVersion,
	})
	// We are very unlikely to hit EOF here since we are just establishing the call,
	// but it may happen that the apiserver is just shutting down (e.g. being restarted)
	// This is consistent with how it is handled for informers
	switch err {
	case nil:
		break

	case io.EOF:
		// watch closed normally
		return false, 0

	case io.ErrUnexpectedEOF:
		klog.V(1).Infof("Watch closed with unexpected EOF: %v", err)
		return false, 0

	default:
		msg := "Watch failed: %v"
		if net.IsProbableEOF(err) {
			klog.V(5).Infof(msg, err)
			// Retry
			return false, 0
		}

		klog.Errorf(msg, err)
		// Retry
		return false, 0
	}

	if watcher == nil {
		klog.Error("Watch returned nil watcher")
		// Retry
		return false, 0
	}

	ch := watcher.ResultChan()
	defer watcher.Stop()

	for {
		select {
		case <-rw.stopChan:
			klog.V(4).Info("Stopping RetryWatcher.")
			return true, 0
		case event, ok := <-ch:
			if !ok {
				klog.V(4).Infof("Failed to get event! Re-creating the watcher. Last RV: %s", rw.lastResourceVersion)
				return false, 0
			}

			// We need to inspect the event and get ResourceVersion out of it
			switch event.Type {
			case watch.Added, watch.Modified, watch.Deleted:
				metaObject, ok := event.Object.(resourceVersionGetter)
				if !ok {
					_ = rw.send(watch.Event{
						Type:   watch.Error,
						Object: &apierrors.NewInternalError(errors.New("retryWatcher: doesn't support resourceVersion")).ErrStatus,
					})
					// We have to abort here because this might cause lastResourceVersion inconsistency by skipping a potential RV with valid data!
					return true, 0
				}

				resourceVersion := metaObject.GetResourceVersion()
				if resourceVersion == "" {
					_ = rw.send(watch.Event{
						Type:   watch.Error,
						Object: &apierrors.NewInternalError(fmt.Errorf("retryWatcher: object %#v doesn't support resourceVersion", event.Object)).ErrStatus,
					})
					// We have to abort here because this might cause lastResourceVersion inconsistency by skipping a potential RV with valid data!
					return true, 0
				}

				// All is fine; send the event and update lastResourceVersion
				ok = rw.send(event)
				if !ok {
					return true, 0
				}
				rw.lastResourceVersion = resourceVersion

				continue

			case watch.Error:
				status, ok := event.Object.(*metav1.Status)
				if !ok {
					klog.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
					// Retry unknown errors
					return false, 0
				}

				statusDelay := time.Duration(0)
				if status.Details != nil {
					statusDelay = time.Duration(status.Details.RetryAfterSeconds) * time.Second
				}

				switch status.Code {
				case http.StatusGone:
					// Never retry RV too old errors
					_ = rw.send(event)
					return true, 0

				case http.StatusGatewayTimeout, http.StatusInternalServerError:
					// Retry
					return false, statusDelay

				default:
					// We retry by default. RetryWatcher is meant to proceed unless it is certain
					// that it can't. If we are not certain, we proceed with retry and leave it
					// up to the user to timeout if needed.

					// Log here so we have a record of hitting the unexpected error
					// and we can whitelist some error codes if we missed any that are expected.
					klog.V(5).Info(spew.Sprintf("Retrying after unexpected error: %#+v", event.Object))

					// Retry
					return false, statusDelay
				}

			default:
				klog.Errorf("Failed to recognize Event type %q", event.Type)
				_ = rw.send(watch.Event{
					Type:   watch.Error,
					Object: &apierrors.NewInternalError(fmt.Errorf("retryWatcher failed to recognize Event type %q", event.Type)).ErrStatus,
				})
				// We are unable to restart the watch and have to stop the loop or this might cause lastResourceVersion inconsistency by skipping a potential RV with valid data!
				return true, 0
			}
		}
	}
}

// receive reads the result from a watcher, restarting it if necessary.
func (rw *RetryWatcher) receive() {
	defer close(rw.doneChan)
	defer close(rw.resultChan)

	klog.V(4).Info("Starting RetryWatcher.")
	defer klog.V(4).Info("Stopping RetryWatcher.")

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	go func() {
		select {
		case <-rw.stopChan:
			cancel()
			return
		case <-ctx.Done():
			return
		}
	}()

	// We use non sliding until so we don't introduce delays on happy path when WATCH call
	// timeouts or gets closed and we need to reestablish it while also avoiding hot loops.
	wait.NonSlidingUntilWithContext(ctx, func(ctx context.Context) {
		done, retryAfter := rw.doReceive()
		if done {
			cancel()
			return
		}

		time.Sleep(retryAfter)

		klog.V(4).Infof("Restarting RetryWatcher at RV=%q", rw.lastResourceVersion)
	}, rw.minRestartDelay)
}

// ResultChan implements Interface.
func (rw *RetryWatcher) ResultChan() <-chan watch.Event {
	return rw.resultChan
}

// Stop implements Interface.
func (rw *RetryWatcher) Stop() {
	close(rw.stopChan)
}

// Done allows the caller to be notified when Retry watcher stops.
func (rw *RetryWatcher) Done() <-chan struct{} {
	return rw.doneChan
}
<commit_msg>Handle unstructured status in RetryWatcher<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package watch

import (
	"context"
	"errors"
	"fmt"
	"io"
	"net/http"
	"time"

	"github.com/davecgh/go-spew/spew"

	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/net"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/tools/cache"
	"k8s.io/klog"
)

// resourceVersionGetter is an interface used to get resource version from events.
// We can't reuse an interface from meta otherwise it would be a cyclic dependency and we need just this one method
type resourceVersionGetter interface {
	GetResourceVersion() string
}

// RetryWatcher will make sure that in case the underlying watcher is closed (e.g. due to API timeout or etcd timeout)
// it will get restarted from the last point without the consumer even knowing about it.
// RetryWatcher does that by inspecting events and keeping track of resourceVersion.
// Especially useful when using watch.UntilWithoutRetry where premature termination is causing issues and flakes.
// Please note that this is not resilient to etcd cache not having the resource version anymore - you would need to
// use Informers for that.
type RetryWatcher struct {
	lastResourceVersion string
	watcherClient       cache.Watcher
	resultChan          chan watch.Event
	stopChan            chan struct{}
	doneChan            chan struct{}
	minRestartDelay     time.Duration
}

// NewRetryWatcher creates a new RetryWatcher.
// It will make sure that watches gets restarted in case of recoverable errors.
// The initialResourceVersion will be given to watch method when first called.
func NewRetryWatcher(initialResourceVersion string, watcherClient cache.Watcher) (*RetryWatcher, error) {
	return newRetryWatcher(initialResourceVersion, watcherClient, 1*time.Second)
}

func newRetryWatcher(initialResourceVersion string, watcherClient cache.Watcher, minRestartDelay time.Duration) (*RetryWatcher, error) {
	switch initialResourceVersion {
	case "", "0":
		// TODO: revisit this if we ever get WATCH v2 where it means start "now"
		//       without doing the synthetic list of objects at the beginning (see #74022)
		return nil, fmt.Errorf("initial RV %q is not supported due to issues with underlying WATCH", initialResourceVersion)
	default:
		break
	}

	rw := &RetryWatcher{
		lastResourceVersion: initialResourceVersion,
		watcherClient:       watcherClient,
		stopChan:            make(chan struct{}),
		doneChan:            make(chan struct{}),
		resultChan:          make(chan watch.Event, 0),
		minRestartDelay:     minRestartDelay,
	}

	go rw.receive()
	return rw, nil
}

func (rw *RetryWatcher) send(event watch.Event) bool {
	// Writing to an unbuffered channel is blocking operation
	// and we need to check if stop wasn't requested while doing so.
	select {
	case rw.resultChan <- event:
		return true
	case <-rw.stopChan:
		return false
	}
}

// doReceive returns true when it is done, false otherwise.
// If it is not done the second return value holds the time to wait before calling it again.
func (rw *RetryWatcher) doReceive() (bool, time.Duration) {
	watcher, err := rw.watcherClient.Watch(metav1.ListOptions{
		ResourceVersion: rw.lastResourceVersion,
	})
	// We are very unlikely to hit EOF here since we are just establishing the call,
	// but it may happen that the apiserver is just shutting down (e.g. being restarted)
	// This is consistent with how it is handled for informers
	switch err {
	case nil:
		break

	case io.EOF:
		// watch closed normally
		return false, 0

	case io.ErrUnexpectedEOF:
		klog.V(1).Infof("Watch closed with unexpected EOF: %v", err)
		return false, 0

	default:
		msg := "Watch failed: %v"
		if net.IsProbableEOF(err) {
			klog.V(5).Infof(msg, err)
			// Retry
			return false, 0
		}

		klog.Errorf(msg, err)
		// Retry
		return false, 0
	}

	if watcher == nil {
		klog.Error("Watch returned nil watcher")
		// Retry
		return false, 0
	}

	ch := watcher.ResultChan()
	defer watcher.Stop()

	for {
		select {
		case <-rw.stopChan:
			klog.V(4).Info("Stopping RetryWatcher.")
			return true, 0
		case event, ok := <-ch:
			if !ok {
				klog.V(4).Infof("Failed to get event! Re-creating the watcher. Last RV: %s", rw.lastResourceVersion)
				return false, 0
			}

			// We need to inspect the event and get ResourceVersion out of it
			switch event.Type {
			case watch.Added, watch.Modified, watch.Deleted:
				metaObject, ok := event.Object.(resourceVersionGetter)
				if !ok {
					_ = rw.send(watch.Event{
						Type:   watch.Error,
						Object: &apierrors.NewInternalError(errors.New("retryWatcher: doesn't support resourceVersion")).ErrStatus,
					})
					// We have to abort here because this might cause lastResourceVersion inconsistency by skipping a potential RV with valid data!
					return true, 0
				}

				resourceVersion := metaObject.GetResourceVersion()
				if resourceVersion == "" {
					_ = rw.send(watch.Event{
						Type:   watch.Error,
						Object: &apierrors.NewInternalError(fmt.Errorf("retryWatcher: object %#v doesn't support resourceVersion", event.Object)).ErrStatus,
					})
					// We have to abort here because this might cause lastResourceVersion inconsistency by skipping a potential RV with valid data!
					return true, 0
				}

				// All is fine; send the event and update lastResourceVersion
				ok = rw.send(event)
				if !ok {
					return true, 0
				}
				rw.lastResourceVersion = resourceVersion

				continue

			case watch.Error:
				// This round trip allows us to handle unstructured status
				errObject := apierrors.FromObject(event.Object)
				statusErr, ok := errObject.(*apierrors.StatusError)
				if !ok {
					klog.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
					// Retry unknown errors
					return false, 0
				}

				status := statusErr.ErrStatus

				statusDelay := time.Duration(0)
				if status.Details != nil {
					statusDelay = time.Duration(status.Details.RetryAfterSeconds) * time.Second
				}

				switch status.Code {
				case http.StatusGone:
					// Never retry RV too old errors
					_ = rw.send(event)
					return true, 0

				case http.StatusGatewayTimeout, http.StatusInternalServerError:
					// Retry
					return false, statusDelay

				default:
					// We retry by default. RetryWatcher is meant to proceed unless it is certain
					// that it can't. If we are not certain, we proceed with retry and leave it
					// up to the user to timeout if needed.

					// Log here so we have a record of hitting the unexpected error
					// and we can whitelist some error codes if we missed any that are expected.
					klog.V(5).Info(spew.Sprintf("Retrying after unexpected error: %#+v", event.Object))

					// Retry
					return false, statusDelay
				}

			default:
				klog.Errorf("Failed to recognize Event type %q", event.Type)
				_ = rw.send(watch.Event{
					Type:   watch.Error,
					Object: &apierrors.NewInternalError(fmt.Errorf("retryWatcher failed to recognize Event type %q", event.Type)).ErrStatus,
				})
				// We are unable to restart the watch and have to stop the loop or this might cause lastResourceVersion inconsistency by skipping a potential RV with valid data!
				return true, 0
			}
		}
	}
}

// receive reads the result from a watcher, restarting it if necessary.
func (rw *RetryWatcher) receive() {
	defer close(rw.doneChan)
	defer close(rw.resultChan)

	klog.V(4).Info("Starting RetryWatcher.")
	defer klog.V(4).Info("Stopping RetryWatcher.")

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	go func() {
		select {
		case <-rw.stopChan:
			cancel()
			return
		case <-ctx.Done():
			return
		}
	}()

	// We use non sliding until so we don't introduce delays on happy path when WATCH call
	// timeouts or gets closed and we need to reestablish it while also avoiding hot loops.
	wait.NonSlidingUntilWithContext(ctx, func(ctx context.Context) {
		done, retryAfter := rw.doReceive()
		if done {
			cancel()
			return
		}

		time.Sleep(retryAfter)

		klog.V(4).Infof("Restarting RetryWatcher at RV=%q", rw.lastResourceVersion)
	}, rw.minRestartDelay)
}

// ResultChan implements Interface.
func (rw *RetryWatcher) ResultChan() <-chan watch.Event {
	return rw.resultChan
}

// Stop implements Interface.
func (rw *RetryWatcher) Stop() {
	close(rw.stopChan)
}

// Done allows the caller to be notified when Retry watcher stops.
func (rw *RetryWatcher) Done() <-chan struct{} {
	return rw.doneChan
}
<|endoftext|>


<commit_before>package memstr

import (
	"errors"
	"regexp"
	"strconv"
)

var (
	sizeMap = map[string]int64{
		"k": 1000,
		"K": 1024,
		"m": 1000000,
		"M": 1048576,
		"g": 1000000000,
		"G": 1073741824,
		"t": 1000000000000,
		"T": 1099511627776,
	}

	// ErrBadFormat means the string was incorreclty formatted.
	ErrBadFormat = errors.New("bad format")

	// ErrInvalidSizeString means the string was formatted correctly, but the size was invalid
	ErrInvalidSizeString = errors.New("invalid size string")

	memMatcher = regexp.MustCompile("([0-9]+)([kKmMgGtT]*)")
)

// Parse given a string, parse the size of memory it represents ie "1K"= 1024 bytes
func Parse(s string) (int64, error) {
	match := memMatcher.FindStringSubmatch(s)
	if len(match) < 2 {
		return 0, ErrBadFormat
	}
	i, err := strconv.ParseInt(match[1], 10, 64)
	if err != nil {
		return 0, err
	}

	// if we did not match a string size i.e. k,m,G then assume the multiple is 1
	var n int64
	if len(match) < 3 {
		n = 1
	} else {
		var ok bool
		// get the multiple from the size string
		n, ok = sizeMap[match[2]]
		if !ok {
			n = 1
		}
	}
	return n * i, nil
}

// CompareMemory given two size strings, compare them.
// given CompareMemory(x,y)
// 1 : x > y
// 0 : x == y
// -1 : x < y
// -2 : there was an error
func CompareMemory(x, y string) (int, error) {
	var xi, yi int64
	var err error
	xi, err = Parse(x)
	yi, err = Parse(y)

	if err != nil {
		return -2, err
	}
	if xi > yi {
		return 1, nil
	}
	if xi == yi {
		return 0, nil
	}
	if xi < yi {
		return -1, nil
	}
	return -2, nil
}
<commit_msg>support floating point numbers<commit_after>package memstr

import (
	"errors"
	"regexp"
	"strconv"
)

var (
	sizeMap = map[string]int64{
		"k": 1000,
		"K": 1024,
		"m": 1000000,
		"M": 1048576,
		"g": 1000000000,
		"G": 1073741824,
		"t": 1000000000000,
		"T": 1099511627776,
	}

	// ErrBadFormat means the string was incorreclty formatted.
	ErrBadFormat = errors.New("bad format")

	// ErrInvalidSizeString means the string was formatted correctly, but the size was invalid
	ErrInvalidSizeString = errors.New("invalid size string")

	memMatcher = regexp.MustCompile(`([0-9]+\.*[0-9]*)([kKmMgGtT]*)`)
)

// Parse given a string, parse the size of memory it represents ie "1K"= 1024 bytes
func Parse(s string) (int64, error) {
	match := memMatcher.FindStringSubmatch(s)
	if len(match) < 2 {
		return 0, ErrBadFormat
	}
	i, err := strconv.ParseFloat(match[1], 64)
	if err != nil {
		return 0, err
	}

	// if we did not match a string size i.e. k,m,G then assume the multiple is 1
	var n int64
	if len(match) < 3 {
		n = 1
	} else {
		var ok bool
		// get the multiple from the size string
		n, ok = sizeMap[match[2]]
		if !ok {
			n = 1
		}
	}
	return int64(float64(n) * i), nil
}

// CompareMemory given two size strings, compare them.
// given CompareMemory(x,y)
// 1 : x > y
// 0 : x == y
// -1 : x < y
// -2 : there was an error
func CompareMemory(x, y string) (int, error) {
	var xi, yi int64
	var err error
	xi, err = Parse(x)
	yi, err = Parse(y)

	if err != nil {
		return -2, err
	}
	if xi > yi {
		return 1, nil
	}
	if xi == yi {
		return 0, nil
	}
	if xi < yi {
		return -1, nil
	}
	return -2, nil
}
<|endoftext|>