<commit_before>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"os"
	"os/signal"
	"strconv"
	"strings"

	"github.com/Shopify/sarama"
	"github.com/companieshouse/chs.go/avro"
	"github.com/companieshouse/chs.go/avro/schema"
	"github.com/companieshouse/chs.go/kafka/producer"
	"github.com/companieshouse/kafka-management-tool/schemas"
)

// MandatoryFlags lists all flags and their types
type MandatoryFlags struct {
	Broker         string
	Topic          string
	Schema         string
	SchemaRegistry string
	Partition      int64
	Offset         string
	JsonOut        int64
}

// Arguments struct will hold the offset, consumer and producer ready for
// passing into the main process method, this allows setup to be done
// before any processing
type Arguments struct {
	OffsetArray []int64
	Consumer    sarama.Consumer
	Producer    *producer.Producer
}

// flags is the global variable for the command line flags
var flags MandatoryFlags

// init maps all flag inputs to the flags package variable
func init() {
	flag.StringVar(&flags.Broker, "broker", "", "Broker address")
	flag.StringVar(&flags.Topic, "topic", "", "Topic name")
	flag.StringVar(&flags.Schema, "schema", "", "Schema name")
	flag.StringVar(&flags.SchemaRegistry, "schema-registry", "", "Schema Registry")
	flag.Int64Var(&flags.Partition, "partition", 0, "Partition (default: 0)")
	flag.StringVar(&flags.Offset, "offset", "", "Offset number, can be single offset number i.e. 10 or a range separated by the - operator i.e. 10-20")
	flag.Int64Var(&flags.JsonOut, "json-out", 0, "Print deserialized JSON message (default: 0)")
}

func main() {
	flag.Parse()

	// check if all mandatory flags have been provided
	if err := validateFlags(createFlagMap()); err != nil {
		panic(err)
	}

	// Print flags and parameters of the tool
	printFlags()

	//create consumer
	consumer, err := sarama.NewConsumer([]string{flags.Broker}, nil)
	if err != nil {
		fmt.Println("error creating consumer")
		panic(err)
	}

	//create new producer
	p, err := producer.New(&producer.Config{Acks: &producer.WaitForAll, BrokerAddrs: []string{flags.Broker}})
	if err != nil {
		fmt.Println("error creating producer")
		panic(err)
	}

	// pass arguments to struct
	arguments := Arguments{
		OffsetArray: createOffsetArray(flags.Offset),
		Consumer:    consumer,
		Producer:    p,
	}

	// process the messages on the topic
	processMessages(arguments)
}

// processMessages iterates through the topic messages and
// republishes them and outputting them if the json-out flag
// is set to true
func processMessages(argu Arguments) {

	// create messages chan
	messages := make(chan *sarama.ConsumerMessage)
	go consumePartition(argu.Consumer, flags.Topic, int32(flags.Partition), messages, argu.OffsetArray)

	// create signals chan
	signals := make(chan os.Signal, 1)
	signal.Notify(signals, os.Interrupt)

	consumed := 0
ConsumerLoop:
	for {
		select {
		case msg := <-messages:

			// make messageBytes channel
			messageBytes := make(chan []byte)

			// if jsonOutPtr is set to 1, then output JSON and republish
			// otherwise just republish message
			if flags.JsonOut == 1 {
				go outputJSON(msg, messageBytes)
			} else {
				go func() {
					messageBytes <- msg.Value
				}()
			}

			// create producer message
			producerMessage := &producer.Message{
				Value: <-messageBytes,
				Topic: flags.Topic,
			}

			// republish message
			fmt.Printf("Republishing message for offset: %d \n", msg.Offset)
			partition, offset, err := argu.Producer.Send(producerMessage)
			if err != nil {
				fmt.Printf("error republishing message with offset: %d, \n", msg.Offset)
				panic(err)
			}
			fmt.Printf("Successfully republished message with offset %v to topic: %v using partition: %v new offset: %v \n", msg.Offset, flags.Topic, partition, offset)
			fmt.Println("---------------------")

			consumed++

			// check if we have hit the end of the array of messages
			// if so, exit the tool by breaking the ConsumerLoop
			if consumed == len(argu.OffsetArray) {
				break ConsumerLoop
			}
		case <-signals:
			fmt.Println("break")
			break ConsumerLoop
		}
	}
}

// outputJSON prints the deserialised json message to the console for
// easier reading if the json-out flag is set to 1
func outputJSON(msg *sarama.ConsumerMessage, messageBytes chan []byte) {
	schemaStruct := schemas.IdentifySchema(flags.Topic)

	if schemaStruct == nil {
		fmt.Println("no schema identified")
		os.Exit(1)
	}

	var err error

	// Get schema from schema registry and use it to create avro consumer
	fmt.Printf("Getting schema for topic: %v \n", flags.Topic)
	schema, err := schema.Get(flags.SchemaRegistry, flags.Schema)
	if err != nil {
		fmt.Println("error getting schema")
		panic(err)
	}
	fmt.Printf("Retrieved schema for topic: %v \n", flags.Topic)

	consumerAvro := &avro.Schema{
		Definition: schema,
	}

	// Unmarshal message value and assign it to schemaStruct
	fmt.Printf("Unmarshalling message for offset: %v \n", msg.Offset)
	if err = consumerAvro.Unmarshal(msg.Value, schemaStruct); err != nil {
		fmt.Printf("error unmarshalling avro message for offset: %v \n", msg.Offset)
		panic(err)
	}
	fmt.Printf("Message successfully unmarshalled for offset: %v \n", msg.Offset)

	// Print the unmarshalled message out to the terminal
	fmt.Println("Unmarshalled message:")
	data, err := json.Marshal(schemaStruct)
	if err != nil {
		fmt.Printf("error marshalling JSON message for offset: %v \n", msg.Offset)
		panic(err)
	}
	fmt.Println(string(data))

	// create avro producer
	producerAvro := &avro.Schema{
		Definition: schema,
	}

	// Marshall schemaStruct that contains message value ready for republishing
	fmt.Printf("Marshalling message for offset: %v \n", msg.Offset)
	message, err := producerAvro.Marshal(schemaStruct)
	if err != nil {
		fmt.Printf("error marshalling avro message for offset: %v \n", msg.Offset)
		panic(err)
	}
	fmt.Printf("Message successfully marshalled for offset: %v \n", msg.Offset)

	// output the marshalled message into the messageBytes chan
	messageBytes <- message
}

// createFlagMap creates a map of flags to be used for validation
// to check if the correct flags have been specified
func createFlagMap() map[string]string {
	flagsMap := make(map[string]string)
	flagsMap["broker"] = flags.Broker
	flagsMap["topic"] = flags.Topic
	flagsMap["schema"] = flags.Schema
	flagsMap["schema-registry"] = flags.SchemaRegistry
	flagsMap["partition"] = string(flags.Partition)
	flagsMap["offset"] = flags.Offset
	flagsMap["json-out"] = string(flags.JsonOut)
	return flagsMap
}

// Validates the flags to make sure all mandatory flags have been supplied
// throw an error if not the case
func validateFlags(flags map[string]string) error {
	var error int8
	for k := range flags {
		if flags[k] == "" {
			fmt.Printf("Value not supplied for: %v \n", k)
			error = 1
		}
	}

	if error == 1 {
		return errors.New("flag validation failed")
	}
	return nil
}

// printFlags prints the flags and parameters of the tool
// to the console
func printFlags() {
	fmt.Println("---------------------")
	fmt.Println("Parameters:")
	flag.VisitAll(func(f *flag.Flag) {
		fmt.Printf("%v: %v\n", f.Name, f.Value)
	})
	fmt.Println("---------------------")
}

// consumePartition iterates over and each message in the topic related to the offset number
// in the offsetArray and outputs the message into the 'out' chan
func consumePartition(consumer sarama.Consumer, topic string, partition int32, out chan *sarama.ConsumerMessage, offsetArray []int64) {
	for _, offset := range offsetArray {
		partitionConsumer, err := consumer.ConsumePartition(topic, partition, offset)
		if err != nil {
			panic(err)
		}

		msg := <-partitionConsumer.Messages()
		err = partitionConsumer.Close()
		if err != nil {
			panic(err)
		}
		out <- msg
	}
}

// createOffsetArray creates the offset array depending on whether a
// range or a single value is entered as an argument to the tool
func createOffsetArray(offset string) []int64 {
	arraySize := make([]int64, 0)
	if strings.ContainsAny(offset, "-") {
		slice := strings.Split(offset, "-")
		minRange, err := strconv.ParseInt(slice[0], 10, 64)
		if err != nil {
			panic(err)
		}

		maxRange, err := strconv.ParseInt(slice[1], 10, 64)
		if err != nil {
			panic(err)
		}

		if minRange > maxRange {
			fmt.Printf("min range cannot be greater than max range")
			os.Exit(1)
		}

		for value := minRange; value <= maxRange; value++ {
			arraySize = append(arraySize, value)
		}
	} else {
		value, err := strconv.ParseInt(offset, 10, 64)
		if err != nil {
			panic(err)
		}
		arraySize = append(arraySize, value)
	}
	return arraySize
}
<commit_msg>capitalise json<commit_after>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"os"
	"os/signal"
	"strconv"
	"strings"

	"github.com/Shopify/sarama"
	"github.com/companieshouse/chs.go/avro"
	"github.com/companieshouse/chs.go/avro/schema"
	"github.com/companieshouse/chs.go/kafka/producer"
	"github.com/companieshouse/kafka-management-tool/schemas"
)

// MandatoryFlags lists all flags and their types
type MandatoryFlags struct {
	Broker         string
	Topic          string
	Schema         string
	SchemaRegistry string
	Partition      int64
	Offset         string
	JSONOut        int64
}

// Arguments struct will hold the offset, consumer and producer ready for
// passing into the main process method, this allows setup to be done
// before any processing
type Arguments struct {
	OffsetArray []int64
	Consumer    sarama.Consumer
	Producer    *producer.Producer
}

// flags is the global variable for the command line flags
var flags MandatoryFlags

// init maps all flag inputs to the flags package variable
func init() {
	flag.StringVar(&flags.Broker, "broker", "", "Broker address")
	flag.StringVar(&flags.Topic, "topic", "", "Topic name")
	flag.StringVar(&flags.Schema, "schema", "", "Schema name")
	flag.StringVar(&flags.SchemaRegistry, "schema-registry", "", "Schema Registry")
	flag.Int64Var(&flags.Partition, "partition", 0, "Partition (default: 0)")
	flag.StringVar(&flags.Offset, "offset", "", "Offset number, can be single offset number i.e. 10 or a range separated by the - operator i.e. 10-20")
	flag.Int64Var(&flags.JSONOut, "json-out", 0, "Print deserialized JSON message (default: 0)")
}

func main() {
	flag.Parse()

	// check if all mandatory flags have been provided
	if err := validateFlags(createFlagMap()); err != nil {
		panic(err)
	}

	// Print flags and parameters of the tool
	printFlags()

	//create consumer
	consumer, err := sarama.NewConsumer([]string{flags.Broker}, nil)
	if err != nil {
		fmt.Println("error creating consumer")
		panic(err)
	}

	//create new producer
	p, err := producer.New(&producer.Config{Acks: &producer.WaitForAll, BrokerAddrs: []string{flags.Broker}})
	if err != nil {
		fmt.Println("error creating producer")
		panic(err)
	}

	// pass arguments to struct
	arguments := Arguments{
		OffsetArray: createOffsetArray(flags.Offset),
		Consumer:    consumer,
		Producer:    p,
	}

	// process the messages on the topic
	processMessages(arguments)
}

// processMessages iterates through the topic messages and
// republishes them and outputting them if the json-out flag
// is set to true
func processMessages(argu Arguments) {

	// create messages chan
	messages := make(chan *sarama.ConsumerMessage)
	go consumePartition(argu.Consumer, flags.Topic, int32(flags.Partition), messages, argu.OffsetArray)

	// create signals chan
	signals := make(chan os.Signal, 1)
	signal.Notify(signals, os.Interrupt)

	consumed := 0
ConsumerLoop:
	for {
		select {
		case msg := <-messages:

			// make messageBytes channel
			messageBytes := make(chan []byte)

			// if jsonOutPtr is set to 1, then output JSON and republish
			// otherwise just republish message
			if flags.JSONOut == 1 {
				go outputJSON(msg, messageBytes)
			} else {
				go func() {
					messageBytes <- msg.Value
				}()
			}

			// create producer message
			producerMessage := &producer.Message{
				Value: <-messageBytes,
				Topic: flags.Topic,
			}

			// republish message
			fmt.Printf("Republishing message for offset: %d \n", msg.Offset)
			partition, offset, err := argu.Producer.Send(producerMessage)
			if err != nil {
				fmt.Printf("error republishing message with offset: %d, \n", msg.Offset)
				panic(err)
			}
			fmt.Printf("Successfully republished message with offset %v to topic: %v using partition: %v new offset: %v \n", msg.Offset, flags.Topic, partition, offset)
			fmt.Println("---------------------")

			consumed++

			// check if we have hit the end of the array of messages
			// if so, exit the tool by breaking the ConsumerLoop
			if consumed == len(argu.OffsetArray) {
				break ConsumerLoop
			}
		case <-signals:
			fmt.Println("break")
			break ConsumerLoop
		}
	}
}

// outputJSON prints the deserialised json message to the console for
// easier reading if the json-out flag is set to 1
func outputJSON(msg *sarama.ConsumerMessage, messageBytes chan []byte) {
	schemaStruct := schemas.IdentifySchema(flags.Topic)

	if schemaStruct == nil {
		fmt.Println("no schema identified")
		os.Exit(1)
	}

	var err error

	// Get schema from schema registry and use it to create avro consumer
	fmt.Printf("Getting schema for topic: %v \n", flags.Topic)
	schema, err := schema.Get(flags.SchemaRegistry, flags.Schema)
	if err != nil {
		fmt.Println("error getting schema")
		panic(err)
	}
	fmt.Printf("Retrieved schema for topic: %v \n", flags.Topic)

	consumerAvro := &avro.Schema{
		Definition: schema,
	}

	// Unmarshal message value and assign it to schemaStruct
	fmt.Printf("Unmarshalling message for offset: %v \n", msg.Offset)
	if err = consumerAvro.Unmarshal(msg.Value, schemaStruct); err != nil {
		fmt.Printf("error unmarshalling avro message for offset: %v \n", msg.Offset)
		panic(err)
	}
	fmt.Printf("Message successfully unmarshalled for offset: %v \n", msg.Offset)

	// Print the unmarshalled message out to the terminal
	fmt.Println("Unmarshalled message:")
	data, err := json.Marshal(schemaStruct)
	if err != nil {
		fmt.Printf("error marshalling JSON message for offset: %v \n", msg.Offset)
		panic(err)
	}
	fmt.Println(string(data))

	// create avro producer
	producerAvro := &avro.Schema{
		Definition: schema,
	}

	// Marshall schemaStruct that contains message value ready for republishing
	fmt.Printf("Marshalling message for offset: %v \n", msg.Offset)
	message, err := producerAvro.Marshal(schemaStruct)
	if err != nil {
		fmt.Printf("error marshalling avro message for offset: %v \n", msg.Offset)
		panic(err)
	}
	fmt.Printf("Message successfully marshalled for offset: %v \n", msg.Offset)

	// output the marshalled message into the messageBytes chan
	messageBytes <- message
}

// createFlagMap creates a map of flags to be used for validation
// to check if the correct flags have been specified
func createFlagMap() map[string]string {
	flagsMap := make(map[string]string)
	flagsMap["broker"] = flags.Broker
	flagsMap["topic"] = flags.Topic
	flagsMap["schema"] = flags.Schema
	flagsMap["schema-registry"] = flags.SchemaRegistry
	flagsMap["partition"] = string(flags.Partition)
	flagsMap["offset"] = flags.Offset
	flagsMap["json-out"] = string(flags.JSONOut)
	return flagsMap
}

// Validates the flags to make sure all mandatory flags have been supplied
// throw an error if not the case
func validateFlags(flags map[string]string) error {
	var error int8
	for k := range flags {
		if flags[k] == "" {
			fmt.Printf("Value not supplied for: %v \n", k)
			error = 1
		}
	}

	if error == 1 {
		return errors.New("flag validation failed")
	}
	return nil
}

// printFlags prints the flags and parameters of the tool
// to the console
func printFlags() {
	fmt.Println("---------------------")
	fmt.Println("Parameters:")
	flag.VisitAll(func(f *flag.Flag) {
		fmt.Printf("%v: %v\n", f.Name, f.Value)
	})
	fmt.Println("---------------------")
}

// consumePartition iterates over and each message in the topic related to the offset number
// in the offsetArray and outputs the message into the 'out' chan
func consumePartition(consumer sarama.Consumer, topic string, partition int32, out chan *sarama.ConsumerMessage, offsetArray []int64) {
	for _, offset := range offsetArray {
		partitionConsumer, err := consumer.ConsumePartition(topic, partition, offset)
		if err != nil {
			panic(err)
		}

		msg := <-partitionConsumer.Messages()
		err = partitionConsumer.Close()
		if err != nil {
			panic(err)
		}
		out <- msg
	}
}

// createOffsetArray creates the offset array depending on whether a
// range or a single value is entered as an argument to the tool
func createOffsetArray(offset string) []int64 {
	arraySize := make([]int64, 0)
	if strings.ContainsAny(offset, "-") {
		slice := strings.Split(offset, "-")
		minRange, err := strconv.ParseInt(slice[0], 10, 64)
		if err != nil {
			panic(err)
		}

		maxRange, err := strconv.ParseInt(slice[1], 10, 64)
		if err != nil {
			panic(err)
		}

		if minRange > maxRange {
			fmt.Printf("min range cannot be greater than max range")
			os.Exit(1)
		}

		for value := minRange; value <= maxRange; value++ {
			arraySize = append(arraySize, value)
		}
	} else {
		value, err := strconv.ParseInt(offset, 10, 64)
		if err != nil {
			panic(err)
		}
		arraySize = append(arraySize, value)
	}
	return arraySize
}
<|endoftext|>


<commit_before>package response

import (
	"encoding/json"
	"net/http"
)

// AsJSON wraps handler and parse payload to json response
func AsJSON(next http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")

		ctx := contextWithResponse(r.Context())

		next.ServeHTTP(w, r.WithContext(ctx))

		if response, ok := fromContext(ctx); ok {
			switch t := response.payload.(type) {
			case HTTPError:
				w.WriteHeader(t.Code)
			case *HTTPError:
				w.WriteHeader(t.Code)
			}

			encoder := json.NewEncoder(w)
			encoder.SetEscapeHTML(true)
			encoder.SetIndent("", "")

			err := encoder.Encode(response.payload)
			if err != nil {
				w.WriteHeader(http.StatusInternalServerError)
				json.NewEncoder(w).Encode(HTTPError{
					Code:    http.StatusInternalServerError,
					Error:   err,
					Message: http.StatusText(http.StatusInternalServerError),
				})
				return
			}

			if f, ok := w.(http.Flusher); ok {
				f.Flush()
			} else {
				// Write nil in case of setting http.StatusOK header if header not set
				w.Write(nil)
			}
		}
	}

	return http.HandlerFunc(fn)
}
<commit_msg>If payload is nil return empty response instead of null<commit_after>package response

import (
  "encoding/json"
  "net/http"
  "fmt"
)

// AsJSON wraps handler and parse payload to json response
func AsJSON(next http.Handler) http.Handler {
  fn := func(w http.ResponseWriter, r *http.Request) {
    w.Header().Set("Content-Type", "application/json")

    ctx := contextWithResponse(r.Context())

    next.ServeHTTP(w, r.WithContext(ctx))

    if response, ok := fromContext(ctx); ok {
      switch t := response.payload.(type) {
      case HTTPError:
        w.WriteHeader(t.Code)
      case *HTTPError:
        w.WriteHeader(t.Code)
      }

      encoder := json.NewEncoder(w)
      encoder.SetEscapeHTML(true)
      encoder.SetIndent("", "")

      fmt.Println(response.payload)

      if response.payload != nil {
        err := encoder.Encode(response.payload)

        if err != nil {
          w.WriteHeader(http.StatusInternalServerError)
          encoder.Encode(HTTPError{
            Code:    http.StatusInternalServerError,
            Error:   err,
            Message: http.StatusText(http.StatusInternalServerError),
          })

          return
        }
      }

      if f, ok := w.(http.Flusher); ok {
        f.Flush()
      } else {
        // Write nil in case of setting http.StatusOK header if header not set
        w.Write(nil)
      }
    }
  }

  return http.HandlerFunc(fn)
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package prober

import (
	"math/rand"
	"time"

	"github.com/golang/glog"
	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/runtime"
	podutil "k8s.io/kubernetes/pkg/api/v1/pod"
	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
	"k8s.io/kubernetes/pkg/kubelet/prober/results"
	"k8s.io/kubernetes/pkg/kubelet/util/format"
)

// worker handles the periodic probing of its assigned container. Each worker has a go-routine
// associated with it which runs the probe loop until the container permanently terminates, or the
// stop channel is closed. The worker uses the probe Manager's statusManager to get up-to-date
// container IDs.
type worker struct {
	// Channel for stopping the probe.
	stopCh chan struct{}

	// The pod containing this probe (read-only)
	pod *v1.Pod

	// The container to probe (read-only)
	container v1.Container

	// Describes the probe configuration (read-only)
	spec *v1.Probe

	// The type of the worker.
	probeType probeType

	// The probe value during the initial delay.
	initialValue results.Result

	// Where to store this workers results.
	resultsManager results.Manager
	probeManager   *manager

	// The last known container ID for this worker.
	containerID kubecontainer.ContainerID
	// The last probe result for this worker.
	lastResult results.Result
	// How many times in a row the probe has returned the same result.
	resultRun int

	// If set, skip probing.
	onHold bool
}

// Creates and starts a new probe worker.
func newWorker(
	m *manager,
	probeType probeType,
	pod *v1.Pod,
	container v1.Container) *worker {

	w := &worker{
		stopCh:       make(chan struct{}, 1), // Buffer so stop() can be non-blocking.
		pod:          pod,
		container:    container,
		probeType:    probeType,
		probeManager: m,
	}

	switch probeType {
	case readiness:
		w.spec = container.ReadinessProbe
		w.resultsManager = m.readinessManager
		w.initialValue = results.Failure
	case liveness:
		w.spec = container.LivenessProbe
		w.resultsManager = m.livenessManager
		w.initialValue = results.Success
	}

	return w
}

// run periodically probes the container.
func (w *worker) run() {
	probeTickerPeriod := time.Duration(w.spec.PeriodSeconds) * time.Second
	probeTicker := time.NewTicker(probeTickerPeriod)

	defer func() {
		// Clean up.
		probeTicker.Stop()
		if !w.containerID.IsEmpty() {
			w.resultsManager.Remove(w.containerID)
		}

		w.probeManager.removeWorker(w.pod.UID, w.container.Name, w.probeType)
	}()

	// If kubelet restarted the probes could be started in rapid succession.
	// Let the worker wait for a random portion of tickerPeriod before probing.
	time.Sleep(time.Duration(rand.Float64() * float64(probeTickerPeriod)))

probeLoop:
	for w.doProbe() {
		// Wait for next probe tick.
		select {
		case <-w.stopCh:
			break probeLoop
		case <-probeTicker.C:
			// continue
		}
	}
}

// stop stops the probe worker. The worker handles cleanup and removes itself from its manager.
// It is safe to call stop multiple times.
func (w *worker) stop() {
	select {
	case w.stopCh <- struct{}{}:
	default: // Non-blocking.
	}
}

// doProbe probes the container once and records the result.
// Returns whether the worker should continue.
func (w *worker) doProbe() (keepGoing bool) {
	defer func() { recover() }() // Actually eat panics (HandleCrash takes care of logging)
	defer runtime.HandleCrash(func(_ interface{}) { keepGoing = true })

	status, ok := w.probeManager.statusManager.GetPodStatus(w.pod.UID)
	if !ok {
		// Either the pod has not been created yet, or it was already deleted.
		glog.V(3).Infof("No status for pod: %v", format.Pod(w.pod))
		return true
	}

	// Worker should terminate if pod is terminated.
	if status.Phase == v1.PodFailed || status.Phase == v1.PodSucceeded {
		glog.V(3).Infof("Pod %v %v, exiting probe worker",
			format.Pod(w.pod), status.Phase)
		return false
	}

	c, ok := podutil.GetContainerStatus(status.ContainerStatuses, w.container.Name)
	if !ok || len(c.ContainerID) == 0 {
		// Either the container has not been created yet, or it was deleted.
		glog.V(3).Infof("Probe target container not found: %v - %v",
			format.Pod(w.pod), w.container.Name)
		return true // Wait for more information.
	}

	if w.containerID.String() != c.ContainerID {
		if !w.containerID.IsEmpty() {
			w.resultsManager.Remove(w.containerID)
		}
		w.containerID = kubecontainer.ParseContainerID(c.ContainerID)
		w.resultsManager.Set(w.containerID, w.initialValue, w.pod)
		// We've got a new container; resume probing.
		w.onHold = false
	}

	if w.onHold {
		// Worker is on hold until there is a new container.
		return true
	}

	if c.State.Running == nil {
		glog.V(3).Infof("Non-running container probed: %v - %v",
			format.Pod(w.pod), w.container.Name)
		if !w.containerID.IsEmpty() {
			w.resultsManager.Set(w.containerID, results.Failure, w.pod)
		}
		// Abort if the container will not be restarted.
		return c.State.Terminated == nil ||
			w.pod.Spec.RestartPolicy != v1.RestartPolicyNever
	}

	if int32(time.Since(c.State.Running.StartedAt.Time).Seconds()) < w.spec.InitialDelaySeconds {
		return true
	}

	// TODO: in order for exec probes to correctly handle downward API env, we must be able to reconstruct
	// the full container environment here, OR we must make a call to the CRI in order to get those environment
	// values from the running container.
	result, err := w.probeManager.prober.probe(w.probeType, w.pod, status, w.container, w.containerID)
	if err != nil {
		// Prober error, throw away the result.
		return true
	}

	if w.lastResult == result {
		w.resultRun++
	} else {
		w.lastResult = result
		w.resultRun = 1
	}

	if (result == results.Failure && w.resultRun < int(w.spec.FailureThreshold)) ||
		(result == results.Success && w.resultRun < int(w.spec.SuccessThreshold)) {
		// Success or failure is below threshold - leave the probe state unchanged.
		return true
	}

	w.resultsManager.Set(w.containerID, result, w.pod)

	if w.probeType == liveness && result == results.Failure {
		// The container fails a liveness check, it will need to be restarted.
		// Stop probing until we see a new container ID. This is to reduce the
		// chance of hitting #21751, where running `docker exec` when a
		// container is being stopped may lead to corrupted container state.
		w.onHold = true
		w.resultRun = 1
	}

	return true
}
<commit_msg>fix prober ticking shift for kubelet restarted cases<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package prober

import (
	"math/rand"
	"time"

	"github.com/golang/glog"
	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/runtime"
	podutil "k8s.io/kubernetes/pkg/api/v1/pod"
	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
	"k8s.io/kubernetes/pkg/kubelet/prober/results"
	"k8s.io/kubernetes/pkg/kubelet/util/format"
)

// worker handles the periodic probing of its assigned container. Each worker has a go-routine
// associated with it which runs the probe loop until the container permanently terminates, or the
// stop channel is closed. The worker uses the probe Manager's statusManager to get up-to-date
// container IDs.
type worker struct {
	// Channel for stopping the probe.
	stopCh chan struct{}

	// The pod containing this probe (read-only)
	pod *v1.Pod

	// The container to probe (read-only)
	container v1.Container

	// Describes the probe configuration (read-only)
	spec *v1.Probe

	// The type of the worker.
	probeType probeType

	// The probe value during the initial delay.
	initialValue results.Result

	// Where to store this workers results.
	resultsManager results.Manager
	probeManager   *manager

	// The last known container ID for this worker.
	containerID kubecontainer.ContainerID
	// The last probe result for this worker.
	lastResult results.Result
	// How many times in a row the probe has returned the same result.
	resultRun int

	// If set, skip probing.
	onHold bool
}

// Creates and starts a new probe worker.
func newWorker(
	m *manager,
	probeType probeType,
	pod *v1.Pod,
	container v1.Container) *worker {

	w := &worker{
		stopCh:       make(chan struct{}, 1), // Buffer so stop() can be non-blocking.
		pod:          pod,
		container:    container,
		probeType:    probeType,
		probeManager: m,
	}

	switch probeType {
	case readiness:
		w.spec = container.ReadinessProbe
		w.resultsManager = m.readinessManager
		w.initialValue = results.Failure
	case liveness:
		w.spec = container.LivenessProbe
		w.resultsManager = m.livenessManager
		w.initialValue = results.Success
	}

	return w
}

// run periodically probes the container.
func (w *worker) run() {
	probeTickerPeriod := time.Duration(w.spec.PeriodSeconds) * time.Second

	// If kubelet restarted the probes could be started in rapid succession.
	// Let the worker wait for a random portion of tickerPeriod before probing.
	time.Sleep(time.Duration(rand.Float64() * float64(probeTickerPeriod)))

	probeTicker := time.NewTicker(probeTickerPeriod)

	defer func() {
		// Clean up.
		probeTicker.Stop()
		if !w.containerID.IsEmpty() {
			w.resultsManager.Remove(w.containerID)
		}

		w.probeManager.removeWorker(w.pod.UID, w.container.Name, w.probeType)
	}()

probeLoop:
	for w.doProbe() {
		// Wait for next probe tick.
		select {
		case <-w.stopCh:
			break probeLoop
		case <-probeTicker.C:
			// continue
		}
	}
}

// stop stops the probe worker. The worker handles cleanup and removes itself from its manager.
// It is safe to call stop multiple times.
func (w *worker) stop() {
	select {
	case w.stopCh <- struct{}{}:
	default: // Non-blocking.
	}
}

// doProbe probes the container once and records the result.
// Returns whether the worker should continue.
func (w *worker) doProbe() (keepGoing bool) {
	defer func() { recover() }() // Actually eat panics (HandleCrash takes care of logging)
	defer runtime.HandleCrash(func(_ interface{}) { keepGoing = true })

	status, ok := w.probeManager.statusManager.GetPodStatus(w.pod.UID)
	if !ok {
		// Either the pod has not been created yet, or it was already deleted.
		glog.V(3).Infof("No status for pod: %v", format.Pod(w.pod))
		return true
	}

	// Worker should terminate if pod is terminated.
	if status.Phase == v1.PodFailed || status.Phase == v1.PodSucceeded {
		glog.V(3).Infof("Pod %v %v, exiting probe worker",
			format.Pod(w.pod), status.Phase)
		return false
	}

	c, ok := podutil.GetContainerStatus(status.ContainerStatuses, w.container.Name)
	if !ok || len(c.ContainerID) == 0 {
		// Either the container has not been created yet, or it was deleted.
		glog.V(3).Infof("Probe target container not found: %v - %v",
			format.Pod(w.pod), w.container.Name)
		return true // Wait for more information.
	}

	if w.containerID.String() != c.ContainerID {
		if !w.containerID.IsEmpty() {
			w.resultsManager.Remove(w.containerID)
		}
		w.containerID = kubecontainer.ParseContainerID(c.ContainerID)
		w.resultsManager.Set(w.containerID, w.initialValue, w.pod)
		// We've got a new container; resume probing.
		w.onHold = false
	}

	if w.onHold {
		// Worker is on hold until there is a new container.
		return true
	}

	if c.State.Running == nil {
		glog.V(3).Infof("Non-running container probed: %v - %v",
			format.Pod(w.pod), w.container.Name)
		if !w.containerID.IsEmpty() {
			w.resultsManager.Set(w.containerID, results.Failure, w.pod)
		}
		// Abort if the container will not be restarted.
		return c.State.Terminated == nil ||
			w.pod.Spec.RestartPolicy != v1.RestartPolicyNever
	}

	if int32(time.Since(c.State.Running.StartedAt.Time).Seconds()) < w.spec.InitialDelaySeconds {
		return true
	}

	// TODO: in order for exec probes to correctly handle downward API env, we must be able to reconstruct
	// the full container environment here, OR we must make a call to the CRI in order to get those environment
	// values from the running container.
	result, err := w.probeManager.prober.probe(w.probeType, w.pod, status, w.container, w.containerID)
	if err != nil {
		// Prober error, throw away the result.
		return true
	}

	if w.lastResult == result {
		w.resultRun++
	} else {
		w.lastResult = result
		w.resultRun = 1
	}

	if (result == results.Failure && w.resultRun < int(w.spec.FailureThreshold)) ||
		(result == results.Success && w.resultRun < int(w.spec.SuccessThreshold)) {
		// Success or failure is below threshold - leave the probe state unchanged.
		return true
	}

	w.resultsManager.Set(w.containerID, result, w.pod)

	if w.probeType == liveness && result == results.Failure {
		// The container fails a liveness check, it will need to be restarted.
		// Stop probing until we see a new container ID. This is to reduce the
		// chance of hitting #21751, where running `docker exec` when a
		// container is being stopped may lead to corrupted container state.
		w.onHold = true
		w.resultRun = 1
	}

	return true
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package manifest

import (
	"bufio"
	"context"
	"log"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"

	"knative.dev/pkg/injection/clients/dynamicclient"
	"knative.dev/reconciler-test/pkg/environment"
)

func InstallLocalYaml(ctx context.Context, base map[string]interface{}) (Manifest, error) {
	env := environment.FromContext(ctx)
	cfg := env.TemplateConfig(base)

	pwd, _ := os.Getwd()
	log.Println("PWD: ", pwd)
	_, filename, _, _ := runtime.Caller(1)
	log.Println("FILENAME: ", filename)

	yamls, err := ParseTemplates(path.Dir(filename), env.Images(), cfg)
	if err != nil {
		return nil, err
	}

	dynamicClient := dynamicclient.Get(ctx)

	manifest, err := NewYamlManifest(yamls, false, dynamicClient)
	if err != nil {
		return nil, err
	}

	// Apply yaml.
	if err := manifest.ApplyAll(); err != nil {
		return manifest, err
	}

	// Save the refs.
	env.Reference(manifest.References()...)

	// Temp
	refs := manifest.References()
	log.Println("Created:")
	for _, ref := range refs {
		log.Println(ref)
	}
	return manifest, nil
}

func ImagesLocalYaml() []string {
	pwd, _ := os.Getwd()
	log.Println("PWD: ", pwd)
	_, filename, _, _ := runtime.Caller(1)
	log.Println("FILENAME: ", filename)

	var images []string

	_ = filepath.Walk(path.Dir(filename), func(root string, info os.FileInfo, err error) error {
		if info == nil || info.IsDir() {
			return nil
		}
		if strings.HasSuffix(info.Name(), "yaml") {
			file, err := os.Open(root)
			if err != nil {
				log.Fatal(err)
			}
			defer file.Close()

			scanner := bufio.NewScanner(file)
			for scanner.Scan() {
				parts := strings.Split(scanner.Text(), "ko://")
				if len(parts) == 2 {
					image := strings.ReplaceAll(parts[1], "\"", "")
					images = append(images, image)
				}
			}
		}
		return nil
	})

	return images
}
<commit_msg>add variant of install from other places (#131)<commit_after>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package manifest

import (
	"bufio"
	"context"
	"log"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"

	"knative.dev/pkg/injection/clients/dynamicclient"
	"knative.dev/reconciler-test/pkg/environment"
)

func InstallYaml(ctx context.Context, dir string, base map[string]interface{}) (Manifest, error) {
	env := environment.FromContext(ctx)
	cfg := env.TemplateConfig(base)

	yamls, err := ParseTemplates(dir, env.Images(), cfg)
	if err != nil {
		return nil, err
	}

	dynamicClient := dynamicclient.Get(ctx)

	manifest, err := NewYamlManifest(yamls, false, dynamicClient)
	if err != nil {
		return nil, err
	}

	// Apply yaml.
	if err := manifest.ApplyAll(); err != nil {
		return manifest, err
	}

	// Save the refs.
	env.Reference(manifest.References()...)

	// Temp
	refs := manifest.References()
	log.Println("Created:")
	for _, ref := range refs {
		log.Println(ref)
	}
	return manifest, nil

}

func InstallLocalYaml(ctx context.Context, base map[string]interface{}) (Manifest, error) {
	pwd, _ := os.Getwd()
	log.Println("PWD: ", pwd)
	_, filename, _, _ := runtime.Caller(1)
	log.Println("FILENAME: ", filename)

	return InstallYaml(ctx, path.Dir(filename), base)
}

func ImagesLocalYaml() []string {
	pwd, _ := os.Getwd()
	log.Println("PWD: ", pwd)
	_, filename, _, _ := runtime.Caller(1)
	log.Println("FILENAME: ", filename)

	var images []string

	_ = filepath.Walk(path.Dir(filename), func(root string, info os.FileInfo, err error) error {
		if info == nil || info.IsDir() {
			return nil
		}
		if strings.HasSuffix(info.Name(), "yaml") {
			file, err := os.Open(root)
			if err != nil {
				log.Fatal(err)
			}
			defer file.Close()

			scanner := bufio.NewScanner(file)
			for scanner.Scan() {
				parts := strings.Split(scanner.Text(), "ko://")
				if len(parts) == 2 {
					image := strings.ReplaceAll(parts[1], "\"", "")
					images = append(images, image)
				}
			}
		}
		return nil
	})

	return images
}
<|endoftext|>


<commit_before>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"math"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/prometheus/common/log"
	"github.com/prometheus/common/version"
	"github.com/prometheus/statsd_exporter/pkg/mapper"
	"gopkg.in/alecthomas/kingpin.v2"
)

var (
	listenAddress   = kingpin.Flag("web.listen-address", "Address on which to expose metrics.").Default(":9108").String()
	metricsPath     = kingpin.Flag("web.telemetry-path", "Path under which to expose Prometheus metrics.").Default("/metrics").String()
	graphiteAddress = kingpin.Flag("graphite.listen-address", "TCP and UDP address on which to accept samples.").Default(":9109").String()
	mappingConfig   = kingpin.Flag("graphite.mapping-config", "Metric mapping configuration file name.").Default("").String()
	sampleExpiry    = kingpin.Flag("graphite.sample-expiry", "How long a sample is valid for.").Default("5m").Duration()
	strictMatch     = kingpin.Flag("graphite.mapping-strict-match", "Only store metrics that match the mapping configuration.").Bool()
	dumpFSMPath     = kingpin.Flag("debug.dump-fsm", "The path to dump internal FSM generated for glob matching as Dot file.").Default("").String()

	lastProcessed = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Name: "graphite_last_processed_timestamp_seconds",
			Help: "Unix timestamp of the last processed graphite metric.",
		},
	)
	sampleExpiryMetric = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Name: "graphite_sample_expiry_seconds",
			Help: "How long in seconds a metric sample is valid for.",
		},
	)
	invalidMetricChars = regexp.MustCompile("[^a-zA-Z0-9_:]")
)

type graphiteSample struct {
	OriginalName string
	Name         string
	Labels       map[string]string
	Help         string
	Value        float64
	Type         prometheus.ValueType
	Timestamp    time.Time
}

type metricMapper interface {
	GetMapping(string, mapper.MetricType) (*mapper.MetricMapping, prometheus.Labels, bool)
	InitFromFile(string) error
}

type graphiteCollector struct {
	samples     map[string]*graphiteSample
	mu          *sync.Mutex
	mapper      metricMapper
	ch          chan *graphiteSample
	strictMatch bool
}

func newGraphiteCollector() *graphiteCollector {
	c := &graphiteCollector{
		ch:          make(chan *graphiteSample),
		mu:          &sync.Mutex{},
		samples:     map[string]*graphiteSample{},
		strictMatch: *strictMatch,
	}
	go c.processSamples()
	return c
}

func (c *graphiteCollector) processReader(reader io.Reader) {
	lineScanner := bufio.NewScanner(reader)
	for {
		if ok := lineScanner.Scan(); !ok {
			break
		}
		c.processLine(lineScanner.Text())
	}
}

func (c *graphiteCollector) processLine(line string) {
	line = strings.TrimSpace(line)
	log.Debugf("Incoming line : %s", line)
	parts := strings.Split(line, " ")
	if len(parts) != 3 {
		log.Infof("Invalid part count of %d in line: %s", len(parts), line)
		return
	}
	originalName := parts[0]
	var name string
	mapping, labels, present := c.mapper.GetMapping(originalName, mapper.MetricTypeGauge)

	if (present && mapping.Action == mapper.ActionTypeDrop) || (!present && c.strictMatch) {
		return
	}

	if present {
		name = invalidMetricChars.ReplaceAllString(mapping.Name, "_")
	} else {
		name = invalidMetricChars.ReplaceAllString(originalName, "_")
	}

	value, err := strconv.ParseFloat(parts[1], 64)
	if err != nil {
		log.Infof("Invalid value in line: %s", line)
		return
	}
	timestamp, err := strconv.ParseFloat(parts[2], 64)
	if err != nil {
		log.Infof("Invalid timestamp in line: %s", line)
		return
	}
	sample := graphiteSample{
		OriginalName: originalName,
		Name:         name,
		Value:        value,
		Labels:       labels,
		Type:         prometheus.GaugeValue,
		Help:         fmt.Sprintf("Graphite metric %s", originalName),
		Timestamp:    time.Unix(int64(timestamp), int64(math.Mod(timestamp, 1.0)*1e9)),
	}
	log.Debugf("Sample: %+v", sample)
	lastProcessed.Set(float64(time.Now().UnixNano()) / 1e9)
	c.ch <- &sample
}

func (c *graphiteCollector) processSamples() {
	ticker := time.NewTicker(time.Minute).C

	for {
		select {
		case sample, ok := <-c.ch:
			if sample == nil || !ok {
				return
			}
			c.mu.Lock()
			c.samples[sample.OriginalName] = sample
			c.mu.Unlock()
		case <-ticker:
			// Garbage collect expired samples.
			ageLimit := time.Now().Add(-*sampleExpiry)
			c.mu.Lock()
			for k, sample := range c.samples {
				if ageLimit.After(sample.Timestamp) {
					delete(c.samples, k)
				}
			}
			c.mu.Unlock()
		}
	}
}

// Collect implements prometheus.Collector.
func (c graphiteCollector) Collect(ch chan<- prometheus.Metric) {
	ch <- lastProcessed

	c.mu.Lock()
	samples := make([]*graphiteSample, 0, len(c.samples))
	for _, sample := range c.samples {
		samples = append(samples, sample)
	}
	c.mu.Unlock()

	ageLimit := time.Now().Add(-*sampleExpiry)
	for _, sample := range samples {
		if ageLimit.After(sample.Timestamp) {
			continue
		}
		ch <- prometheus.MustNewConstMetric(
			prometheus.NewDesc(sample.Name, sample.Help, []string{}, sample.Labels),
			sample.Type,
			sample.Value,
		)
	}
}

// Describe implements prometheus.Collector.
func (c graphiteCollector) Describe(ch chan<- *prometheus.Desc) {
	ch <- lastProcessed.Desc()
}

func init() {
	prometheus.MustRegister(version.NewCollector("graphite_exporter"))
}

func dumpFSM(mapper *mapper.MetricMapper, dumpFilename string) error {
	f, err := os.Create(dumpFilename)
	if err != nil {
		return err
	}
	log.Infoln("Start dumping FSM to", dumpFilename)
	w := bufio.NewWriter(f)
	mapper.FSM.DumpFSM(w)
	w.Flush()
	f.Close()
	log.Infoln("Finish dumping FSM")
	return nil
}

func main() {
	log.AddFlags(kingpin.CommandLine)
	kingpin.Version(version.Print("graphite_exporter"))
	kingpin.HelpFlag.Short('h')
	kingpin.Parse()

	prometheus.MustRegister(sampleExpiryMetric)
	sampleExpiryMetric.Set(sampleExpiry.Seconds())

	log.Infoln("Starting graphite_exporter", version.Info())
	log.Infoln("Build context", version.BuildContext())

	http.Handle(*metricsPath, promhttp.Handler())
	c := newGraphiteCollector()
	prometheus.MustRegister(c)

	c.mapper = &mapper.MetricMapper{}
	if *mappingConfig != "" {
		err := c.mapper.InitFromFile(*mappingConfig)
		if err != nil {
			log.Fatalf("Error loading metric mapping config: %s", err)
		}
	}

	if *dumpFSMPath != "" {
		err := dumpFSM(c.mapper.(*mapper.MetricMapper), *dumpFSMPath)
		if err != nil {
			log.Fatal("Error dumping FSM:", err)
		}
	}

	tcpSock, err := net.Listen("tcp", *graphiteAddress)
	if err != nil {
		log.Fatalf("Error binding to TCP socket: %s", err)
	}
	go func() {
		for {
			conn, err := tcpSock.Accept()
			if err != nil {
				log.Errorf("Error accepting TCP connection: %s", err)
				continue
			}
			go func() {
				defer conn.Close()
				c.processReader(conn)
			}()
		}
	}()

	udpAddress, err := net.ResolveUDPAddr("udp", *graphiteAddress)
	if err != nil {
		log.Fatalf("Error resolving UDP address: %s", err)
	}
	udpSock, err := net.ListenUDP("udp", udpAddress)
	if err != nil {
		log.Fatalf("Error listening to UDP address: %s", err)
	}
	go func() {
		defer udpSock.Close()
		for {
			buf := make([]byte, 65536)
			chars, srcAddress, err := udpSock.ReadFromUDP(buf)
			if err != nil {
				log.Errorf("Error reading UDP packet from %s: %s", srcAddress, err)
				continue
			}
			go c.processReader(bytes.NewReader(buf[0:chars]))
		}
	}()

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != "/" {
			http.NotFound(w, r)
			return
		}
		w.Write([]byte(`
      <title>Graphite Exporter</title>

Graphite Exporter

Accepting plaintext Graphite samples over TCP and UDP on ` + *graphiteAddress + `

<a href="` + *metricsPath + `">Metrics

      `))
	})

	log.Infoln("Listening on", *listenAddress)
	log.Fatal(http.ListenAndServe(*listenAddress, nil))
}
<commit_msg>fix Prometheus HELP text to be consistent<commit_after>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"math"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/prometheus/common/log"
	"github.com/prometheus/common/version"
	"github.com/prometheus/statsd_exporter/pkg/mapper"
	"gopkg.in/alecthomas/kingpin.v2"
)

var (
	listenAddress   = kingpin.Flag("web.listen-address", "Address on which to expose metrics.").Default(":9108").String()
	metricsPath     = kingpin.Flag("web.telemetry-path", "Path under which to expose Prometheus metrics.").Default("/metrics").String()
	graphiteAddress = kingpin.Flag("graphite.listen-address", "TCP and UDP address on which to accept samples.").Default(":9109").String()
	mappingConfig   = kingpin.Flag("graphite.mapping-config", "Metric mapping configuration file name.").Default("").String()
	sampleExpiry    = kingpin.Flag("graphite.sample-expiry", "How long a sample is valid for.").Default("5m").Duration()
	strictMatch     = kingpin.Flag("graphite.mapping-strict-match", "Only store metrics that match the mapping configuration.").Bool()
	dumpFSMPath     = kingpin.Flag("debug.dump-fsm", "The path to dump internal FSM generated for glob matching as Dot file.").Default("").String()

	lastProcessed = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Name: "graphite_last_processed_timestamp_seconds",
			Help: "Unix timestamp of the last processed graphite metric.",
		},
	)
	sampleExpiryMetric = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Name: "graphite_sample_expiry_seconds",
			Help: "How long in seconds a metric sample is valid for.",
		},
	)
	invalidMetricChars = regexp.MustCompile("[^a-zA-Z0-9_:]")
)

type graphiteSample struct {
	OriginalName string
	Name         string
	Labels       map[string]string
	Help         string
	Value        float64
	Type         prometheus.ValueType
	Timestamp    time.Time
}

type metricMapper interface {
	GetMapping(string, mapper.MetricType) (*mapper.MetricMapping, prometheus.Labels, bool)
	InitFromFile(string) error
}

type graphiteCollector struct {
	samples     map[string]*graphiteSample
	mu          *sync.Mutex
	mapper      metricMapper
	ch          chan *graphiteSample
	strictMatch bool
}

func newGraphiteCollector() *graphiteCollector {
	c := &graphiteCollector{
		ch:          make(chan *graphiteSample),
		mu:          &sync.Mutex{},
		samples:     map[string]*graphiteSample{},
		strictMatch: *strictMatch,
	}
	go c.processSamples()
	return c
}

func (c *graphiteCollector) processReader(reader io.Reader) {
	lineScanner := bufio.NewScanner(reader)
	for {
		if ok := lineScanner.Scan(); !ok {
			break
		}
		c.processLine(lineScanner.Text())
	}
}

func (c *graphiteCollector) processLine(line string) {
	line = strings.TrimSpace(line)
	log.Debugf("Incoming line : %s", line)
	parts := strings.Split(line, " ")
	if len(parts) != 3 {
		log.Infof("Invalid part count of %d in line: %s", len(parts), line)
		return
	}
	originalName := parts[0]
	var name string
	mapping, labels, present := c.mapper.GetMapping(originalName, mapper.MetricTypeGauge)

	if (present && mapping.Action == mapper.ActionTypeDrop) || (!present && c.strictMatch) {
		return
	}

	if present {
		name = invalidMetricChars.ReplaceAllString(mapping.Name, "_")
	} else {
		name = invalidMetricChars.ReplaceAllString(originalName, "_")
	}

	value, err := strconv.ParseFloat(parts[1], 64)
	if err != nil {
		log.Infof("Invalid value in line: %s", line)
		return
	}
	timestamp, err := strconv.ParseFloat(parts[2], 64)
	if err != nil {
		log.Infof("Invalid timestamp in line: %s", line)
		return
	}
	sample := graphiteSample{
		OriginalName: originalName,
		Name:         name,
		Value:        value,
		Labels:       labels,
		Type:         prometheus.GaugeValue,
		Help:         fmt.Sprintf("Graphite metric %s", name),
		Timestamp:    time.Unix(int64(timestamp), int64(math.Mod(timestamp, 1.0)*1e9)),
	}
	log.Debugf("Sample: %+v", sample)
	lastProcessed.Set(float64(time.Now().UnixNano()) / 1e9)
	c.ch <- &sample
}

func (c *graphiteCollector) processSamples() {
	ticker := time.NewTicker(time.Minute).C

	for {
		select {
		case sample, ok := <-c.ch:
			if sample == nil || !ok {
				return
			}
			c.mu.Lock()
			c.samples[sample.OriginalName] = sample
			c.mu.Unlock()
		case <-ticker:
			// Garbage collect expired samples.
			ageLimit := time.Now().Add(-*sampleExpiry)
			c.mu.Lock()
			for k, sample := range c.samples {
				if ageLimit.After(sample.Timestamp) {
					delete(c.samples, k)
				}
			}
			c.mu.Unlock()
		}
	}
}

// Collect implements prometheus.Collector.
func (c graphiteCollector) Collect(ch chan<- prometheus.Metric) {
	ch <- lastProcessed

	c.mu.Lock()
	samples := make([]*graphiteSample, 0, len(c.samples))
	for _, sample := range c.samples {
		samples = append(samples, sample)
	}
	c.mu.Unlock()

	ageLimit := time.Now().Add(-*sampleExpiry)
	for _, sample := range samples {
		if ageLimit.After(sample.Timestamp) {
			continue
		}
		ch <- prometheus.MustNewConstMetric(
			prometheus.NewDesc(sample.Name, sample.Help, []string{}, sample.Labels),
			sample.Type,
			sample.Value,
		)
	}
}

// Describe implements prometheus.Collector.
func (c graphiteCollector) Describe(ch chan<- *prometheus.Desc) {
	ch <- lastProcessed.Desc()
}

func init() {
	prometheus.MustRegister(version.NewCollector("graphite_exporter"))
}

func dumpFSM(mapper *mapper.MetricMapper, dumpFilename string) error {
	f, err := os.Create(dumpFilename)
	if err != nil {
		return err
	}
	log.Infoln("Start dumping FSM to", dumpFilename)
	w := bufio.NewWriter(f)
	mapper.FSM.DumpFSM(w)
	w.Flush()
	f.Close()
	log.Infoln("Finish dumping FSM")
	return nil
}

func main() {
	log.AddFlags(kingpin.CommandLine)
	kingpin.Version(version.Print("graphite_exporter"))
	kingpin.HelpFlag.Short('h')
	kingpin.Parse()

	prometheus.MustRegister(sampleExpiryMetric)
	sampleExpiryMetric.Set(sampleExpiry.Seconds())

	log.Infoln("Starting graphite_exporter", version.Info())
	log.Infoln("Build context", version.BuildContext())

	http.Handle(*metricsPath, promhttp.Handler())
	c := newGraphiteCollector()
	prometheus.MustRegister(c)

	c.mapper = &mapper.MetricMapper{}
	if *mappingConfig != "" {
		err := c.mapper.InitFromFile(*mappingConfig)
		if err != nil {
			log.Fatalf("Error loading metric mapping config: %s", err)
		}
	}

	if *dumpFSMPath != "" {
		err := dumpFSM(c.mapper.(*mapper.MetricMapper), *dumpFSMPath)
		if err != nil {
			log.Fatal("Error dumping FSM:", err)
		}
	}

	tcpSock, err := net.Listen("tcp", *graphiteAddress)
	if err != nil {
		log.Fatalf("Error binding to TCP socket: %s", err)
	}
	go func() {
		for {
			conn, err := tcpSock.Accept()
			if err != nil {
				log.Errorf("Error accepting TCP connection: %s", err)
				continue
			}
			go func() {
				defer conn.Close()
				c.processReader(conn)
			}()
		}
	}()

	udpAddress, err := net.ResolveUDPAddr("udp", *graphiteAddress)
	if err != nil {
		log.Fatalf("Error resolving UDP address: %s", err)
	}
	udpSock, err := net.ListenUDP("udp", udpAddress)
	if err != nil {
		log.Fatalf("Error listening to UDP address: %s", err)
	}
	go func() {
		defer udpSock.Close()
		for {
			buf := make([]byte, 65536)
			chars, srcAddress, err := udpSock.ReadFromUDP(buf)
			if err != nil {
				log.Errorf("Error reading UDP packet from %s: %s", srcAddress, err)
				continue
			}
			go c.processReader(bytes.NewReader(buf[0:chars]))
		}
	}()

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != "/" {
			http.NotFound(w, r)
			return
		}
		w.Write([]byte(`
      <title>Graphite Exporter</title>

Graphite Exporter

Accepting plaintext Graphite samples over TCP and UDP on ` + *graphiteAddress + `

<a href="` + *metricsPath + `">Metrics

      `))
	})

	log.Infoln("Listening on", *listenAddress)
	log.Fatal(http.ListenAndServe(*listenAddress, nil))
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package registry

import (
	"fmt"
	"net"
	"strconv"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/golang/glog"
)

func MakeEndpointController(serviceRegistry ServiceRegistry, podRegistry PodRegistry) *EndpointController {
	return &EndpointController{
		serviceRegistry: serviceRegistry,
		podRegistry:     podRegistry,
	}
}

type EndpointController struct {
	serviceRegistry ServiceRegistry
	podRegistry     PodRegistry
}

func findPort(manifest *api.ContainerManifest, portName util.IntOrString) (int, error) {
	if ((portName.Kind == util.IntstrString && len(portName.StrVal) == 0) ||
		(portName.Kind == util.IntstrInt && portName.IntVal == 0)) &&
		len(manifest.Containers[0].Ports) > 0 {
		return manifest.Containers[0].Ports[0].ContainerPort, nil
	}
	if portName.Kind == util.IntstrInt {
		return portName.IntVal, nil
	}
	name := portName.StrVal
	for _, container := range manifest.Containers {
		for _, port := range container.Ports {
			if port.Name == name {
				return port.ContainerPort, nil
			}
		}
	}
	return -1, fmt.Errorf("no suitable port for manifest: %s", manifest.ID)
}

func (e *EndpointController) SyncServiceEndpoints() error {
	services, err := e.serviceRegistry.ListServices()
	if err != nil {
		return err
	}
	var resultErr error
	for _, service := range services.Items {
		pods, err := e.podRegistry.ListPods(labels.Set(service.Selector).AsSelector())
		if err != nil {
			glog.Errorf("Error syncing service: %#v, skipping.", service)
			resultErr = err
			continue
		}
		endpoints := make([]string, len(pods))
		for ix, pod := range pods {
			port, err := findPort(&pod.DesiredState.Manifest, service.ContainerPort)
			if err != nil {
				glog.Errorf("Failed to find port for service: %v, %v", service, err)
				continue
			}
			endpoints[ix] = net.JoinHostPort(pod.CurrentState.PodIP, strconv.Itoa(port))
		}
		err = e.serviceRegistry.UpdateEndpoints(api.Endpoints{
			Name:      service.ID,
			Endpoints: endpoints,
		})
		if err != nil {
			glog.Errorf("Error updating endpoints: %#v", err)
			continue
		}
	}
	return resultErr
}
<commit_msg>Add some more logging to service updates.<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package registry

import (
	"fmt"
	"net"
	"strconv"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/golang/glog"
)

func MakeEndpointController(serviceRegistry ServiceRegistry, podRegistry PodRegistry) *EndpointController {
	return &EndpointController{
		serviceRegistry: serviceRegistry,
		podRegistry:     podRegistry,
	}
}

type EndpointController struct {
	serviceRegistry ServiceRegistry
	podRegistry     PodRegistry
}

func findPort(manifest *api.ContainerManifest, portName util.IntOrString) (int, error) {
	if ((portName.Kind == util.IntstrString && len(portName.StrVal) == 0) ||
		(portName.Kind == util.IntstrInt && portName.IntVal == 0)) &&
		len(manifest.Containers[0].Ports) > 0 {
		return manifest.Containers[0].Ports[0].ContainerPort, nil
	}
	if portName.Kind == util.IntstrInt {
		return portName.IntVal, nil
	}
	name := portName.StrVal
	for _, container := range manifest.Containers {
		for _, port := range container.Ports {
			if port.Name == name {
				return port.ContainerPort, nil
			}
		}
	}
	return -1, fmt.Errorf("no suitable port for manifest: %s", manifest.ID)
}

func (e *EndpointController) SyncServiceEndpoints() error {
	services, err := e.serviceRegistry.ListServices()
	if err != nil {
		return err
	}
	var resultErr error
	for _, service := range services.Items {
		pods, err := e.podRegistry.ListPods(labels.Set(service.Selector).AsSelector())
		if err != nil {
			glog.Errorf("Error syncing service: %#v, skipping.", service)
			resultErr = err
			continue
		}
		endpoints := make([]string, len(pods))
		for ix, pod := range pods {
			port, err := findPort(&pod.DesiredState.Manifest, service.ContainerPort)
			if err != nil {
				glog.Errorf("Failed to find port for service: %v, %v", service, err)
				continue
			}
			if len(pod.CurrentState.PodIP) == 0 {
				glog.Errorf("Failed to find an IP for pod: %v", pod)
				continue
			}
			endpoints[ix] = net.JoinHostPort(pod.CurrentState.PodIP, strconv.Itoa(port))
		}
		err = e.serviceRegistry.UpdateEndpoints(api.Endpoints{
			Name:      service.ID,
			Endpoints: endpoints,
		})
		if err != nil {
			glog.Errorf("Error updating endpoints: %#v", err)
			continue
		}
	}
	return resultErr
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package sync

import (
	"context"
	"fmt"
	"os/exec"
	"path"
	"path/filepath"
	"strings"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/filemon"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/bmatcuk/doublestar"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"golang.org/x/sync/errgroup"
	v1 "k8s.io/api/core/v1"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

var (
	// WorkingDir is here for testing
	WorkingDir = docker.RetrieveWorkingDir
)

func NewItem(a *latest.Artifact, e filemon.Events, builds []build.Artifact, insecureRegistries map[string]bool, destProvider DestinationProvider) (*Item, error) {
	if !e.HasChanged() || a.Sync == nil {
		return nil, nil
	}

	if len(a.Sync.Manual) > 0 {
		return manualSyncItem(a, e, builds, insecureRegistries)
	}

	if len(a.Sync.Infer) > 0 {
		return inferredSyncItem(a, e, builds, destProvider)
	}

	return nil, nil
}

func manualSyncItem(a *latest.Artifact, e filemon.Events, builds []build.Artifact, insecureRegistries map[string]bool) (*Item, error) {
	tag := latestTag(a.ImageName, builds)
	if tag == "" {
		return nil, fmt.Errorf("could not find latest tag for image %s in builds: %v", a.ImageName, builds)
	}

	containerWd, err := WorkingDir(tag, insecureRegistries)
	if err != nil {
		return nil, errors.Wrapf(err, "retrieving working dir for %s", tag)
	}

	toCopy, err := intersect(a.Workspace, containerWd, a.Sync.Manual, append(e.Added, e.Modified...))
	if err != nil {
		return nil, errors.Wrap(err, "intersecting sync map and added, modified files")
	}

	toDelete, err := intersect(a.Workspace, containerWd, a.Sync.Manual, e.Deleted)
	if err != nil {
		return nil, errors.Wrap(err, "intersecting sync map and deleted files")
	}

	// Something went wrong, don't sync, rebuild.
	if toCopy == nil || toDelete == nil {
		return nil, nil
	}

	return &Item{Image: tag, Copy: toCopy, Delete: toDelete}, nil
}

func inferredSyncItem(a *latest.Artifact, e filemon.Events, builds []build.Artifact, provider DestinationProvider) (*Item, error) {
	// deleted files are no longer contained in the syncMap, so we need to rebuild
	if len(e.Deleted) > 0 {
		return nil, nil
	}

	tag := latestTag(a.ImageName, builds)
	if tag == "" {
		return nil, fmt.Errorf("could not find latest tag for image %s in builds: %v", a.ImageName, builds)
	}

	syncMap, err := provider()
	if err != nil {
		return nil, errors.Wrapf(err, "inferring syncmap for image %s", a.ImageName)
	}

	toCopy := make(map[string][]string)
	for _, f := range append(e.Modified, e.Added...) {
		relPath, err := filepath.Rel(a.Workspace, f)
		if err != nil {
			return nil, errors.Wrapf(err, "finding changed file %s relative to context %s", f, a.Workspace)
		}

		matches := false
		for _, p := range a.Sync.Infer {
			matches, err = doublestar.PathMatch(filepath.FromSlash(p), relPath)
			if err != nil {
				return nil, errors.Wrapf(err, "pattern error for %s", relPath)
			}
			if matches {
				break
			}
		}
		if !matches {
			logrus.Infof("Changed file %s does not match any sync pattern. Skipping sync", relPath)
			return nil, nil
		}

		if dsts, ok := syncMap[relPath]; ok {
			toCopy[f] = dsts
		} else {
			logrus.Infof("Changed file %s is not syncable. Skipping sync", relPath)
			return nil, nil
		}
	}

	return &Item{Image: tag, Copy: toCopy}, nil
}

func latestTag(image string, builds []build.Artifact) string {
	for _, build := range builds {
		if build.ImageName == image {
			return build.Tag
		}
	}
	return ""
}

func intersect(contextWd, containerWd string, syncRules []*latest.SyncRule, files []string) (syncMap, error) {
	ret := make(syncMap)
	for _, f := range files {
		relPath, err := filepath.Rel(contextWd, f)
		if err != nil {
			return nil, errors.Wrapf(err, "changed file %s can't be found relative to context %s", f, contextWd)
		}

		dsts, err := matchSyncRules(syncRules, relPath, containerWd)
		if err != nil {
			return nil, err
		}

		if len(dsts) == 0 {
			logrus.Infof("Changed file %s does not match any sync pattern. Skipping sync", relPath)
			return nil, nil
		}

		ret[f] = dsts
	}
	return ret, nil
}

func matchSyncRules(syncRules []*latest.SyncRule, relPath, containerWd string) ([]string, error) {
	dsts := make([]string, 0, 1)
	for _, r := range syncRules {
		matches, err := doublestar.PathMatch(filepath.FromSlash(r.Src), relPath)
		if err != nil {
			return nil, errors.Wrapf(err, "pattern error for %s", relPath)
		}

		if !matches {
			continue
		}

		wd := ""
		if !path.IsAbs(r.Dest) {
			// Convert relative destinations to absolute via the working dir in the container.
			wd = containerWd
		}

		// Map the paths as a tree from the prefix.
		subPath := strings.TrimPrefix(filepath.ToSlash(relPath), r.Strip)
		dsts = append(dsts, path.Join(wd, r.Dest, subPath))
	}
	return dsts, nil
}

func (k *podSyncer) Sync(ctx context.Context, s *Item) error {
	if len(s.Copy) > 0 {
		logrus.Infoln("Copying files:", s.Copy, "to", s.Image)

		if err := Perform(ctx, s.Image, s.Copy, k.copyFileFn, k.namespaces); err != nil {
			return errors.Wrap(err, "copying files")
		}
	}

	if len(s.Delete) > 0 {
		logrus.Infoln("Deleting files:", s.Delete, "from", s.Image)

		if err := Perform(ctx, s.Image, s.Delete, k.deleteFileFn, k.namespaces); err != nil {
			return errors.Wrap(err, "deleting files")
		}
	}

	return nil
}

func Perform(ctx context.Context, image string, files syncMap, cmdFn func(context.Context, v1.Pod, v1.Container, syncMap) *exec.Cmd, namespaces []string) error {
	if len(files) == 0 {
		return nil
	}

	errs, ctx := errgroup.WithContext(ctx)

	client, err := kubernetes.Client()
	if err != nil {
		return errors.Wrap(err, "getting kubernetes client")
	}

	numSynced := 0
	for _, ns := range namespaces {
		pods, err := client.CoreV1().Pods(ns).List(meta_v1.ListOptions{})
		if err != nil {
			return errors.Wrap(err, "getting pods for namespace "+ns)
		}

		for _, p := range pods.Items {

			if p.Status.Phase != v1.PodRunning {
				logrus.Infof("Skipping sync with pod %s because it's not running", p.Name)
				continue
			}

			for _, c := range p.Spec.Containers {
				if c.Image != image {
					continue
				}

				cmd := cmdFn(ctx, p, c, files)
				errs.Go(func() error {
					_, err := util.RunCmdOut(cmd)
					return err
				})
				numSynced++
			}
		}
	}

	if numSynced == 0 {
		return errors.New("didn't sync any files")
	}

	return errs.Wait()
}
<commit_msg>Remove misleading log (#2802)<commit_after>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package sync

import (
	"context"
	"fmt"
	"os/exec"
	"path"
	"path/filepath"
	"strings"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/filemon"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/bmatcuk/doublestar"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"golang.org/x/sync/errgroup"
	v1 "k8s.io/api/core/v1"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

var (
	// WorkingDir is here for testing
	WorkingDir = docker.RetrieveWorkingDir
)

func NewItem(a *latest.Artifact, e filemon.Events, builds []build.Artifact, insecureRegistries map[string]bool, destProvider DestinationProvider) (*Item, error) {
	if !e.HasChanged() || a.Sync == nil {
		return nil, nil
	}

	if len(a.Sync.Manual) > 0 {
		return manualSyncItem(a, e, builds, insecureRegistries)
	}

	if len(a.Sync.Infer) > 0 {
		return inferredSyncItem(a, e, builds, destProvider)
	}

	return nil, nil
}

func manualSyncItem(a *latest.Artifact, e filemon.Events, builds []build.Artifact, insecureRegistries map[string]bool) (*Item, error) {
	tag := latestTag(a.ImageName, builds)
	if tag == "" {
		return nil, fmt.Errorf("could not find latest tag for image %s in builds: %v", a.ImageName, builds)
	}

	containerWd, err := WorkingDir(tag, insecureRegistries)
	if err != nil {
		return nil, errors.Wrapf(err, "retrieving working dir for %s", tag)
	}

	toCopy, err := intersect(a.Workspace, containerWd, a.Sync.Manual, append(e.Added, e.Modified...))
	if err != nil {
		return nil, errors.Wrap(err, "intersecting sync map and added, modified files")
	}

	toDelete, err := intersect(a.Workspace, containerWd, a.Sync.Manual, e.Deleted)
	if err != nil {
		return nil, errors.Wrap(err, "intersecting sync map and deleted files")
	}

	// Something went wrong, don't sync, rebuild.
	if toCopy == nil || toDelete == nil {
		return nil, nil
	}

	return &Item{Image: tag, Copy: toCopy, Delete: toDelete}, nil
}

func inferredSyncItem(a *latest.Artifact, e filemon.Events, builds []build.Artifact, provider DestinationProvider) (*Item, error) {
	// deleted files are no longer contained in the syncMap, so we need to rebuild
	if len(e.Deleted) > 0 {
		return nil, nil
	}

	tag := latestTag(a.ImageName, builds)
	if tag == "" {
		return nil, fmt.Errorf("could not find latest tag for image %s in builds: %v", a.ImageName, builds)
	}

	syncMap, err := provider()
	if err != nil {
		return nil, errors.Wrapf(err, "inferring syncmap for image %s", a.ImageName)
	}

	toCopy := make(map[string][]string)
	for _, f := range append(e.Modified, e.Added...) {
		relPath, err := filepath.Rel(a.Workspace, f)
		if err != nil {
			return nil, errors.Wrapf(err, "finding changed file %s relative to context %s", f, a.Workspace)
		}

		matches := false
		for _, p := range a.Sync.Infer {
			matches, err = doublestar.PathMatch(filepath.FromSlash(p), relPath)
			if err != nil {
				return nil, errors.Wrapf(err, "pattern error for %s", relPath)
			}
			if matches {
				break
			}
		}
		if !matches {
			logrus.Infof("Changed file %s does not match any sync pattern. Skipping sync", relPath)
			return nil, nil
		}

		if dsts, ok := syncMap[relPath]; ok {
			toCopy[f] = dsts
		} else {
			logrus.Infof("Changed file %s is not syncable. Skipping sync", relPath)
			return nil, nil
		}
	}

	return &Item{Image: tag, Copy: toCopy}, nil
}

func latestTag(image string, builds []build.Artifact) string {
	for _, build := range builds {
		if build.ImageName == image {
			return build.Tag
		}
	}
	return ""
}

func intersect(contextWd, containerWd string, syncRules []*latest.SyncRule, files []string) (syncMap, error) {
	ret := make(syncMap)
	for _, f := range files {
		relPath, err := filepath.Rel(contextWd, f)
		if err != nil {
			return nil, errors.Wrapf(err, "changed file %s can't be found relative to context %s", f, contextWd)
		}

		dsts, err := matchSyncRules(syncRules, relPath, containerWd)
		if err != nil {
			return nil, err
		}

		if len(dsts) == 0 {
			logrus.Infof("Changed file %s does not match any sync pattern. Skipping sync", relPath)
			return nil, nil
		}

		ret[f] = dsts
	}
	return ret, nil
}

func matchSyncRules(syncRules []*latest.SyncRule, relPath, containerWd string) ([]string, error) {
	dsts := make([]string, 0, 1)
	for _, r := range syncRules {
		matches, err := doublestar.PathMatch(filepath.FromSlash(r.Src), relPath)
		if err != nil {
			return nil, errors.Wrapf(err, "pattern error for %s", relPath)
		}

		if !matches {
			continue
		}

		wd := ""
		if !path.IsAbs(r.Dest) {
			// Convert relative destinations to absolute via the working dir in the container.
			wd = containerWd
		}

		// Map the paths as a tree from the prefix.
		subPath := strings.TrimPrefix(filepath.ToSlash(relPath), r.Strip)
		dsts = append(dsts, path.Join(wd, r.Dest, subPath))
	}
	return dsts, nil
}

func (k *podSyncer) Sync(ctx context.Context, s *Item) error {
	if len(s.Copy) > 0 {
		logrus.Infoln("Copying files:", s.Copy, "to", s.Image)

		if err := Perform(ctx, s.Image, s.Copy, k.copyFileFn, k.namespaces); err != nil {
			return errors.Wrap(err, "copying files")
		}
	}

	if len(s.Delete) > 0 {
		logrus.Infoln("Deleting files:", s.Delete, "from", s.Image)

		if err := Perform(ctx, s.Image, s.Delete, k.deleteFileFn, k.namespaces); err != nil {
			return errors.Wrap(err, "deleting files")
		}
	}

	return nil
}

func Perform(ctx context.Context, image string, files syncMap, cmdFn func(context.Context, v1.Pod, v1.Container, syncMap) *exec.Cmd, namespaces []string) error {
	if len(files) == 0 {
		return nil
	}

	errs, ctx := errgroup.WithContext(ctx)

	client, err := kubernetes.Client()
	if err != nil {
		return errors.Wrap(err, "getting kubernetes client")
	}

	numSynced := 0
	for _, ns := range namespaces {
		pods, err := client.CoreV1().Pods(ns).List(meta_v1.ListOptions{})
		if err != nil {
			return errors.Wrap(err, "getting pods for namespace "+ns)
		}

		for _, p := range pods.Items {
			if p.Status.Phase != v1.PodRunning {
				continue
			}

			for _, c := range p.Spec.Containers {
				if c.Image != image {
					continue
				}

				cmd := cmdFn(ctx, p, c, files)
				errs.Go(func() error {
					_, err := util.RunCmdOut(cmd)
					return err
				})
				numSynced++
			}
		}
	}

	if numSynced == 0 {
		return errors.New("didn't sync any files")
	}

	return errs.Wait()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"net/http"

	"github.com/gophergala2016/gophertron/routes"
	"github.com/pkg/profile"
)

var addr = flag.String("http", "localhost:8080", "http service address")
var cpuprof = flag.Bool("cpuprofile", false, "cpu profile")

func main() {
	flag.Parse()
	log.SetFlags(log.Lshortfile)

	if *cpuprof {
		log.Println("Profiling CPU usage")
		defer profile.Start().Stop()
	}

	routes.InitRoutes()
	log.Println("Serving on ", *addr)
	log.Fatal(http.ListenAndServe(*addr, http.DefaultServeMux))
}
<commit_msg>Fix spacing<commit_after>package main

import (
	"flag"
	"log"
	"net/http"

	"github.com/gophergala2016/gophertron/routes"
	"github.com/pkg/profile"
)

var addr = flag.String("http", "localhost:8080", "http service address")
var cpuprof = flag.Bool("cpuprofile", false, "cpu profile")

func main() {
	flag.Parse()
	log.SetFlags(log.Lshortfile)

	if *cpuprof {
		log.Println("Profiling CPU usage")
		defer profile.Start().Stop()
	}

	routes.InitRoutes()
	log.Printf("Serving on %s", *addr)
	log.Fatal(http.ListenAndServe(*addr, http.DefaultServeMux))
}
<|endoftext|>


<commit_before>// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.

package storage

import (
	"runtime"
	"runtime/debug"
	"sync/atomic"
	"time"

	"github.com/cockroachdb/cockroach/pkg/util/syncutil"
	"github.com/cockroachdb/cockroach/pkg/util/timeutil"

	"golang.org/x/net/context"
)

// timedMutex is a mutex which dumps a stack trace via the supplied callback
// whenever a lock is unlocked after having been held for longer than the
// supplied duration, which must be strictly positive.
type timedMutex struct {
	mu       *syncutil.Mutex // intentionally pointer to make zero value unusable
	lockedAt time.Time       // protected by mu
	isLocked int32           // updated atomically

	// Non-mutable fields.
	cb timingFn
}

// timingFn is a callback passed to MakeTimedMutex. It is invoked with the
// measured duration of the critical section of the associated mutex after
// each Unlock operation.
type timingFn func(heldFor time.Duration)

// thresholdLogger returns a timing function which calls 'record' for each
// measured duration and, for measurements exceeding 'warnDuration', invokes
// 'printf' with the passed context and a detailed stack trace.
func thresholdLogger(
	ctx context.Context,
	warnDuration time.Duration,
	printf func(context.Context, string, ...interface{}),
	record timingFn,
) timingFn {
	return func(heldFor time.Duration) {
		record(heldFor)
		if heldFor > warnDuration {
			// NB: this doesn't use `util/caller.Lookup` because that would result
			// in an import cycle.
			pc, _, _, ok := runtime.Caller(2)
			fun := "?"
			if ok {
				if f := runtime.FuncForPC(pc); f != nil {
					fun = f.Name()
				}
			}

			printf(
				ctx, "mutex held by %s for %s (>%s):\n%s",
				fun, heldFor, warnDuration, debug.Stack(),
			)
		}
	}
}

// makeTimedMutex creates a TimedMutex which warns when an Unlock happens more
// than warnDuration after the corresponding lock. It will use the supplied
// context and logging callback for the warning message; a nil logger falls
// back to Fprintf to os.Stderr.
func makeTimedMutex(cb timingFn) timedMutex {
	return timedMutex{
		cb: cb,
		mu: &syncutil.Mutex{},
	}
}

// Lock implements sync.Locker.
func (tm *timedMutex) Lock() {
	tm.mu.Lock()
	atomic.StoreInt32(&tm.isLocked, 1)
	tm.lockedAt = timeutil.Now()
}

// Unlock implements sync.Locker.
func (tm *timedMutex) Unlock() {
	lockedAt := tm.lockedAt
	atomic.StoreInt32(&tm.isLocked, 0)
	tm.mu.Unlock()
	if tm.cb != nil {
		tm.cb(timeutil.Since(lockedAt))
	}
}

// AssertHeld may panic if the mutex is not locked (but it is not
// required to do so). Functions which require that their callers hold
// a particular lock may use this to enforce this requirement more
// directly than relying on the race detector.
//
// Note that we do not require the lock to be held by any particular
// thread, just that some thread holds the lock. This is both more
// efficient and allows for rare cases where a mutex is locked in one
// thread and used in another.
//
// TODO(bdarnell): Add an equivalent method to syncutil.Mutex
// (possibly a no-op depending on a build tag)
func (tm *timedMutex) AssertHeld() {
	isLocked := atomic.LoadInt32(&tm.isLocked)
	if isLocked == 0 {
		panic("mutex is not locked")
	}
}
<commit_msg>storage: add COCKROACH_DISABLE_TIMED_MUTEX env var<commit_after>// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.

package storage

import (
	"runtime"
	"runtime/debug"
	"sync/atomic"
	"time"

	"github.com/cockroachdb/cockroach/pkg/util/envutil"
	"github.com/cockroachdb/cockroach/pkg/util/syncutil"
	"github.com/cockroachdb/cockroach/pkg/util/timeutil"

	"golang.org/x/net/context"
)

var disableTimedMutex = envutil.EnvOrDefaultBool("COCKROACH_DISABLE_TIMED_MUTEX", false)

// timedMutex is a mutex which dumps a stack trace via the supplied callback
// whenever a lock is unlocked after having been held for longer than the
// supplied duration, which must be strictly positive.
type timedMutex struct {
	mu       *syncutil.Mutex // intentionally pointer to make zero value unusable
	lockedAt time.Time       // protected by mu
	isLocked int32           // updated atomically

	// Non-mutable fields.
	cb timingFn
}

// timingFn is a callback passed to MakeTimedMutex. It is invoked with the
// measured duration of the critical section of the associated mutex after
// each Unlock operation.
type timingFn func(heldFor time.Duration)

// thresholdLogger returns a timing function which calls 'record' for each
// measured duration and, for measurements exceeding 'warnDuration', invokes
// 'printf' with the passed context and a detailed stack trace.
func thresholdLogger(
	ctx context.Context,
	warnDuration time.Duration,
	printf func(context.Context, string, ...interface{}),
	record timingFn,
) timingFn {
	return func(heldFor time.Duration) {
		record(heldFor)
		if heldFor > warnDuration {
			// NB: this doesn't use `util/caller.Lookup` because that would result
			// in an import cycle.
			pc, _, _, ok := runtime.Caller(2)
			fun := "?"
			if ok {
				if f := runtime.FuncForPC(pc); f != nil {
					fun = f.Name()
				}
			}

			printf(
				ctx, "mutex held by %s for %s (>%s):\n%s",
				fun, heldFor, warnDuration, debug.Stack(),
			)
		}
	}
}

// makeTimedMutex creates a TimedMutex which warns when an Unlock happens more
// than warnDuration after the corresponding lock. If non-nil, it will invoke
// the supplied timingFn callback after each unlock with the elapsed time that
// the mutex was held for.
func makeTimedMutex(cb timingFn) timedMutex {
	if disableTimedMutex {
		cb = nil
	}
	return timedMutex{
		cb: cb,
		mu: &syncutil.Mutex{},
	}
}

// Lock implements sync.Locker.
func (tm *timedMutex) Lock() {
	tm.mu.Lock()
	atomic.StoreInt32(&tm.isLocked, 1)
	tm.lockedAt = timeutil.Now()
}

// Unlock implements sync.Locker.
func (tm *timedMutex) Unlock() {
	lockedAt := tm.lockedAt
	atomic.StoreInt32(&tm.isLocked, 0)
	tm.mu.Unlock()
	if tm.cb != nil {
		tm.cb(timeutil.Since(lockedAt))
	}
}

// AssertHeld may panic if the mutex is not locked (but it is not
// required to do so). Functions which require that their callers hold
// a particular lock may use this to enforce this requirement more
// directly than relying on the race detector.
//
// Note that we do not require the lock to be held by any particular
// thread, just that some thread holds the lock. This is both more
// efficient and allows for rare cases where a mutex is locked in one
// thread and used in another.
//
// TODO(bdarnell): Add an equivalent method to syncutil.Mutex
// (possibly a no-op depending on a build tag)
func (tm *timedMutex) AssertHeld() {
	isLocked := atomic.LoadInt32(&tm.isLocked)
	if isLocked == 0 {
		panic("mutex is not locked")
	}
}
<|endoftext|>


<commit_before>package main

//go:generate genqrc assets

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"time"

	"gopkg.in/qml.v1"
)

var net string

func readIntFromFile(path string) (int, error) {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		return 0, err
	}
	b = b[:len(b)-1] // delete "\n"
	i, err := strconv.Atoi(string(b))
	if err != nil {
		return 0, err
	}
	return i, nil
}

func getNetIn() int {
	i, err := readIntFromFile("/sys/class/net/" + net + "/statistics/rx_bytes")
	if err != nil {
		log.Fatal(err)
	}
	return i
}

func getNetOut() int {
	i, err := readIntFromFile("/sys/class/net/" + net + "/statistics/tx_bytes")
	if err != nil {
		log.Fatal(err)
	}
	return i
}

func parseSize(src int) string {
	if src == 0 {
		return "0 B/s"
	}
	var unit string

	for i := 0; ; i++ {
		size := 1 << uint(i*10)
		if src < size {
			size = 1 << uint((i-1)*10)
			src = src / size
			switch i - 1 {
			case 0:
				unit = "B"
			case 1:
				unit = "KB"
			case 2:
				unit = "MB"
			case 3:
				unit = "GB"
			}
			break
		}
	}
	return strconv.Itoa(src) + " " + unit + "/s"
}

func main() {
	if len(os.Args) == 1 {
		net = "wlan0"
	} else {
		net = os.Args[1]
	}

	if err := qml.Run(run); err != nil {
		fmt.Fprintf(os.Stderr, "error: %v\n", err)
		os.Exit(1)
	}
}

func run() error {
	engine := qml.NewEngine()
	component, err := engine.LoadFile("qrc:///assets/main.qml")
	if err != nil {
		return err
	}

	window := component.CreateWindow(nil)
	root := window.Root()
	upText := root.ObjectByName("upText")
	downText := root.ObjectByName("downText")
	window.Show()

	go func() {
		var (
			oldNetIn  = getNetIn()
			oldNetOut = getNetOut()
			ticker    = time.NewTicker(time.Second * 1)
		)
		for _ = range ticker.C {
			netIn := getNetIn()
			netOut := getNetOut()
			upText.Set("text", `<font color="green">⇧</font> `+parseSize(netIn-oldNetIn))
			downText.Set("text", `<font color="red">⇩</font> `+parseSize(netOut-oldNetOut))
			oldNetIn = netIn
			oldNetOut = netOut
		}
	}()

	window.Wait()

	return nil
}
<commit_msg>修复顺序错误<commit_after>package main

//go:generate genqrc assets

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"time"

	"gopkg.in/qml.v1"
)

var net string

func readIntFromFile(path string) (int, error) {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		return 0, err
	}
	b = b[:len(b)-1] // delete "\n"
	i, err := strconv.Atoi(string(b))
	if err != nil {
		return 0, err
	}
	return i, nil
}

func getNetIn() int {
	i, err := readIntFromFile("/sys/class/net/" + net + "/statistics/rx_bytes")
	if err != nil {
		log.Fatal(err)
	}
	return i
}

func getNetOut() int {
	i, err := readIntFromFile("/sys/class/net/" + net + "/statistics/tx_bytes")
	if err != nil {
		log.Fatal(err)
	}
	return i
}

func parseSize(src int) string {
	if src == 0 {
		return "0 B/s"
	}
	var unit string

	for i := 0; ; i++ {
		size := 1 << uint(i*10)
		if src < size {
			size = 1 << uint((i-1)*10)
			src = src / size
			switch i - 1 {
			case 0:
				unit = "B"
			case 1:
				unit = "KB"
			case 2:
				unit = "MB"
			case 3:
				unit = "GB"
			}
			break
		}
	}
	return strconv.Itoa(src) + " " + unit + "/s"
}

func main() {
	if len(os.Args) == 1 {
		net = "wlan0"
	} else {
		net = os.Args[1]
	}

	if err := qml.Run(run); err != nil {
		fmt.Fprintf(os.Stderr, "error: %v\n", err)
		os.Exit(1)
	}
}

func run() error {
	engine := qml.NewEngine()
	component, err := engine.LoadFile("qrc:///assets/main.qml")
	if err != nil {
		return err
	}

	window := component.CreateWindow(nil)
	root := window.Root()
	upText := root.ObjectByName("upText")
	downText := root.ObjectByName("downText")
	window.Show()

	go func() {
		var (
			oldNetIn  = getNetIn()
			oldNetOut = getNetOut()
			ticker    = time.NewTicker(time.Second * 1)
		)
		for _ = range ticker.C {
			netIn := getNetIn()
			netOut := getNetOut()
			upText.Set("text", `<font color="green">⇧</font> `+parseSize(netOut-oldNetOut))
			downText.Set("text", `<font color="red">⇩</font> `+parseSize(netIn-oldNetIn))
			oldNetIn = netIn
			oldNetOut = netOut
		}
	}()

	window.Wait()

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package transformers

import (
	"fmt"
	"regexp"
	"strings"

	"sigs.k8s.io/kustomize/pkg/image"
	"sigs.k8s.io/kustomize/pkg/resmap"
	"sigs.k8s.io/kustomize/pkg/transformers/config"
)

// imageTransformer replace image names and tags
type imageTransformer struct {
	images     []image.Image
	fieldSpecs []config.FieldSpec
}

var _ Transformer = &imageTransformer{}

// NewImageTransformer constructs an imageTransformer.
func NewImageTransformer(slice []image.Image, fs []config.FieldSpec) (Transformer, error) {
	return &imageTransformer{slice, fs}, nil
}

// Transform finds the matching images and replaces name, tag and/or digest
func (pt *imageTransformer) Transform(m resmap.ResMap) error {
	if len(pt.images) == 0 {
		return nil
	}
	for id := range m {
		objMap := m[id].Map()
		for _, path := range pt.fieldSpecs {
			if !id.Gvk().IsSelected(&path.Gvk) {
				continue
			}
			err := mutateField(objMap, path.PathSlice(), false, pt.updateContainers)
			if err != nil {
				return err
			}
		}
		// Keep for backward compatibility
		err := pt.findAndReplaceImage(objMap)
		if err != nil {
			return err
		}
	}
	return nil
}

/*
 findAndReplaceImage replaces the image name and tags inside one object
 It searches the object for container session
 then loops though all images inside containers session,
 finds matched ones and update the image name and tag name
*/
func (pt *imageTransformer) findAndReplaceImage(obj map[string]interface{}) error {
	paths := []string{"containers", "initContainers"}
	found := false
	for _, path := range paths {
		containers, found := obj[path]
		if found {
			_, err := pt.updateContainers(containers)
			if err != nil {
				return err
			}
		}
	}
	if !found {
		return pt.findContainers(obj)
	}
	return nil
}

func (pt *imageTransformer) updateContainers(in interface{}) (interface{}, error) {
	containers, ok := in.([]interface{})
	if !ok {
		return nil, fmt.Errorf("containers path is not of type []interface{} but %T", in)
	}
	for i := range containers {
		container := containers[i].(map[string]interface{})
		containerImage, found := container["image"]
		if !found {
			continue
		}

		imageName := containerImage.(string)
		for _, img := range pt.images {
			if !isImageMatched(imageName, img.Name) {
				continue
			}
			name, tag := split(imageName)
			if img.NewName != "" {
				name = img.NewName
			}
			if img.NewTag != "" {
				tag = ":" + img.NewTag
			}
			if img.Digest != "" {
				tag = "@" + img.Digest
			}
			container["image"] = name + tag
			break
		}
	}
	return containers, nil
}

func (pt *imageTransformer) findContainers(obj map[string]interface{}) error {
	for key := range obj {
		switch typedV := obj[key].(type) {
		case map[string]interface{}:
			err := pt.findAndReplaceImage(typedV)
			if err != nil {
				return err
			}
		case []interface{}:
			for i := range typedV {
				item := typedV[i]
				typedItem, ok := item.(map[string]interface{})
				if ok {
					err := pt.findAndReplaceImage(typedItem)
					if err != nil {
						return err
					}
				}
			}
		}
	}
	return nil
}

func isImageMatched(s, t string) bool {
	// Tag values are limited to [a-zA-Z0-9_.-].
	pattern, _ := regexp.Compile("^" + t + "(:[a-zA-Z0-9_.-]*)?$")
	return pattern.MatchString(s)
}

// split separates and returns the name and tag parts
// from the image string using either colon `:` or at `@` separators.
// Note that the returned tag keeps its separator.
func split(imageName string) (name string, tag string) {
	// check if image name contains a domain
	// if domain is present, ignore domain and check for `:`
	ic := -1
	if slashIndex := strings.Index(imageName, "/"); slashIndex < 0 {
		ic = strings.LastIndex(imageName, ":")
	} else {
		lastIc := strings.LastIndex(imageName[slashIndex:], ":")
		// set ic only if `:` is present
		if lastIc > 0 {
			ic = slashIndex + lastIc
		}
	}
	ia := strings.LastIndex(imageName, "@")
	if ic < 0 && ia < 0 {
		return imageName, ""
	}

	i := ic
	if ic < 0 {
		i = ia
	}

	name = imageName[:i]
	tag = imageName[i:]
	return
}
<commit_msg>Fix 918<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package transformers

import (
	"fmt"
	"regexp"
	"strings"

	"sigs.k8s.io/kustomize/pkg/image"
	"sigs.k8s.io/kustomize/pkg/resmap"
	"sigs.k8s.io/kustomize/pkg/transformers/config"
)

// imageTransformer replace image names and tags
type imageTransformer struct {
	images     []image.Image
	fieldSpecs []config.FieldSpec
}

var _ Transformer = &imageTransformer{}

// NewImageTransformer constructs an imageTransformer.
func NewImageTransformer(slice []image.Image, fs []config.FieldSpec) (Transformer, error) {
	return &imageTransformer{slice, fs}, nil
}

// Transform finds the matching images and replaces name, tag and/or digest
func (pt *imageTransformer) Transform(m resmap.ResMap) error {
	if len(pt.images) == 0 {
		return nil
	}
	for id := range m {
		objMap := m[id].Map()
		for _, path := range pt.fieldSpecs {
			if !id.Gvk().IsSelected(&path.Gvk) {
				continue
			}
			err := mutateField(objMap, path.PathSlice(), false, pt.updateContainers)
			if err != nil {
				return err
			}
		}
		// Keep for backward compatibility
		err := pt.findAndReplaceImage(objMap)
		if err != nil {
			return err
		}
	}
	return nil
}

/*
 findAndReplaceImage replaces the image name and tags inside one object
 It searches the object for container session
 then loops though all images inside containers session,
 finds matched ones and update the image name and tag name
*/
func (pt *imageTransformer) findAndReplaceImage(obj map[string]interface{}) error {
	paths := []string{"containers", "initContainers"}
	updated := false
	for _, path := range paths {
		containers, found := obj[path]
		if found {
			_, err := pt.updateContainers(containers)
			if err != nil {
				return err
			}
			updated = true
		}
	}
	if !updated {
		return pt.findContainers(obj)
	}
	return nil
}

func (pt *imageTransformer) updateContainers(in interface{}) (interface{}, error) {
	containers, ok := in.([]interface{})
	if !ok {
		return nil, fmt.Errorf("containers path is not of type []interface{} but %T", in)
	}
	for i := range containers {
		container := containers[i].(map[string]interface{})
		containerImage, found := container["image"]
		if !found {
			continue
		}

		imageName := containerImage.(string)
		for _, img := range pt.images {
			if !isImageMatched(imageName, img.Name) {
				continue
			}
			name, tag := split(imageName)
			if img.NewName != "" {
				name = img.NewName
			}
			if img.NewTag != "" {
				tag = ":" + img.NewTag
			}
			if img.Digest != "" {
				tag = "@" + img.Digest
			}
			container["image"] = name + tag
			break
		}
	}
	return containers, nil
}

func (pt *imageTransformer) findContainers(obj map[string]interface{}) error {
	for key := range obj {
		switch typedV := obj[key].(type) {
		case map[string]interface{}:
			err := pt.findAndReplaceImage(typedV)
			if err != nil {
				return err
			}
		case []interface{}:
			for i := range typedV {
				item := typedV[i]
				typedItem, ok := item.(map[string]interface{})
				if ok {
					err := pt.findAndReplaceImage(typedItem)
					if err != nil {
						return err
					}
				}
			}
		}
	}
	return nil
}

func isImageMatched(s, t string) bool {
	// Tag values are limited to [a-zA-Z0-9_.-].
	pattern, _ := regexp.Compile("^" + t + "(:[a-zA-Z0-9_.-]*)?$")
	return pattern.MatchString(s)
}

// split separates and returns the name and tag parts
// from the image string using either colon `:` or at `@` separators.
// Note that the returned tag keeps its separator.
func split(imageName string) (name string, tag string) {
	// check if image name contains a domain
	// if domain is present, ignore domain and check for `:`
	ic := -1
	if slashIndex := strings.Index(imageName, "/"); slashIndex < 0 {
		ic = strings.LastIndex(imageName, ":")
	} else {
		lastIc := strings.LastIndex(imageName[slashIndex:], ":")
		// set ic only if `:` is present
		if lastIc > 0 {
			ic = slashIndex + lastIc
		}
	}
	ia := strings.LastIndex(imageName, "@")
	if ic < 0 && ia < 0 {
		return imageName, ""
	}

	i := ic
	if ic < 0 {
		i = ia
	}

	name = imageName[:i]
	tag = imageName[i:]
	return
}
<|endoftext|>