<commit_before>/*

	Autoreader is a simple program, designed to be run from go:generate, that
	helps generate the annoying boilerplate to implement
	boardgame.PropertyReader and boardgame.PropertyReadSetter.

*/
package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"github.com/MarcGrol/golangAnnotations/parser"
	"io"
	"log"
	"os"
	"strings"
	"text/template"
)

var headerTemplate *template.Template
var readerTemplate *template.Template
var readSetterTemplate *template.Template

const magicDocLinePrefix = "+autoreader"

type appOptions struct {
	OutputFile       string
	PackageDirectory string
	flagSet          *flag.FlagSet
}

type templateConfig struct {
	FirstLetter string
	StructName  string
}

func init() {
	headerTemplate = template.Must(template.New("header").Parse(headerTemplateText))
	readerTemplate = template.Must(template.New("reader").Parse(readerTemplateText))
	readSetterTemplate = template.Must(template.New("readsetter").Parse(readSetterTemplateText))
}

func defineFlags(options *appOptions) {
	options.flagSet.StringVar(&options.OutputFile, "out", "auto_reader.go", "Defines which file to render output to. WARNING: it will be overwritten!")
	options.flagSet.StringVar(&options.PackageDirectory, "pkg", "examplepkg/", "Which package to process")
}

func getOptions(flagSet *flag.FlagSet, flagArguments []string) *appOptions {
	options := &appOptions{flagSet: flagSet}
	defineFlags(options)
	flagSet.Parse(flagArguments)
	return options
}

func main() {
	flagSet := flag.CommandLine
	process(getOptions(flagSet, os.Args[1:]), os.Stderr)
}

func process(options *appOptions, errOut io.ReadWriter) {
	output, err := processPackage(options.PackageDirectory)

	if err != nil {
		fmt.Fprintln(errOut, "ERROR", err)
		return
	}

	fmt.Fprintln(errOut, output)
}

func processPackage(location string) (output string, err error) {
	sources, err := parser.ParseSourceDir(location, ".*")

	if err != nil {
		return "", errors.New("Couldn't parse sources: " + err.Error())
	}

	haveOutputHeader := false

	for _, theStruct := range sources.Structs {

		if !haveOutputHeader {
			output += headerForPackage(theStruct.PackageName)
			haveOutputHeader = true
		}

		enableAutoReader := false

		for _, docLine := range theStruct.DocLines {
			docLine = strings.TrimPrefix(docLine, "//")
			docLine = strings.TrimSpace(docLine)
			if strings.HasPrefix(docLine, magicDocLinePrefix) {
				enableAutoReader = true
				break
			}
		}

		if !enableAutoReader {
			continue
		}

		output += readerForStruct(theStruct.Name)
		output += readSetterForStruct(theStruct.Name)
	}

	return output, nil
}

func templateOutput(template *template.Template, values interface{}) string {
	buf := new(bytes.Buffer)

	err := template.Execute(buf, values)

	if err != nil {
		log.Println(err)
	}

	return buf.String()
}

func headerForPackage(packageName string) string {
	return templateOutput(headerTemplate, map[string]string{
		"packageName": packageName,
	})
}

func readerForStruct(structName string) string {

	return templateOutput(readerTemplate, templateConfig{
		FirstLetter: structName[:1],
		StructName:  structName,
	})

}

func readSetterForStruct(structName string) string {
	return templateOutput(readSetterTemplate, templateConfig{
		FirstLetter: structName[:1],
		StructName:  structName,
	})
}

const headerTemplateText = `/************************************
 *
 * This file was auto-generated by autoreader. DO NOT EDIT.
 *
 ************************************/
package {{.packageName}}

import (
	"github.com/jkomoros/boardgame"
)

`

const readerTemplateText = `func ({{.FirstLetter}} *{{.StructName}}) Reader() boardgame.PropertyReader {
	return boardgame.DefaultReader({{.FirstLetter}})
}

`

const readSetterTemplateText = `func ({{.FirstLetter}} *{{.StructName}}) ReadSetter() boardgame.PropertyReadSetter {
	return boardgame.DefaultReadSetter({{.FirstLetter}})
}

`
<commit_msg>Print help and quit if "-h" provided. part of #301.<commit_after>/*

	Autoreader is a simple program, designed to be run from go:generate, that
	helps generate the annoying boilerplate to implement
	boardgame.PropertyReader and boardgame.PropertyReadSetter.

*/
package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"github.com/MarcGrol/golangAnnotations/parser"
	"io"
	"log"
	"os"
	"strings"
	"text/template"
)

var headerTemplate *template.Template
var readerTemplate *template.Template
var readSetterTemplate *template.Template

const magicDocLinePrefix = "+autoreader"

type appOptions struct {
	OutputFile       string
	PackageDirectory string
	Help             bool
	flagSet          *flag.FlagSet
}

type templateConfig struct {
	FirstLetter string
	StructName  string
}

func init() {
	headerTemplate = template.Must(template.New("header").Parse(headerTemplateText))
	readerTemplate = template.Must(template.New("reader").Parse(readerTemplateText))
	readSetterTemplate = template.Must(template.New("readsetter").Parse(readSetterTemplateText))
}

func defineFlags(options *appOptions) {
	options.flagSet.StringVar(&options.OutputFile, "out", "auto_reader.go", "Defines which file to render output to. WARNING: it will be overwritten!")
	options.flagSet.StringVar(&options.PackageDirectory, "pkg", "examplepkg/", "Which package to process")
	options.flagSet.BoolVar(&options.Help, "h", false, "If set, print help message and quit.")
}

func getOptions(flagSet *flag.FlagSet, flagArguments []string) *appOptions {
	options := &appOptions{flagSet: flagSet}
	defineFlags(options)
	flagSet.Parse(flagArguments)
	return options
}

func main() {
	flagSet := flag.CommandLine
	process(getOptions(flagSet, os.Args[1:]), os.Stderr)
}

func process(options *appOptions, errOut io.ReadWriter) {

	if options.Help {
		options.flagSet.SetOutput(errOut)
		options.flagSet.PrintDefaults()
		return
	}

	output, err := processPackage(options.PackageDirectory)

	if err != nil {
		fmt.Fprintln(errOut, "ERROR", err)
		return
	}

	fmt.Fprintln(errOut, output)
}

func processPackage(location string) (output string, err error) {
	sources, err := parser.ParseSourceDir(location, ".*")

	if err != nil {
		return "", errors.New("Couldn't parse sources: " + err.Error())
	}

	haveOutputHeader := false

	for _, theStruct := range sources.Structs {

		if !haveOutputHeader {
			output += headerForPackage(theStruct.PackageName)
			haveOutputHeader = true
		}

		enableAutoReader := false

		for _, docLine := range theStruct.DocLines {
			docLine = strings.TrimPrefix(docLine, "//")
			docLine = strings.TrimSpace(docLine)
			if strings.HasPrefix(docLine, magicDocLinePrefix) {
				enableAutoReader = true
				break
			}
		}

		if !enableAutoReader {
			continue
		}

		output += readerForStruct(theStruct.Name)
		output += readSetterForStruct(theStruct.Name)
	}

	return output, nil
}

func templateOutput(template *template.Template, values interface{}) string {
	buf := new(bytes.Buffer)

	err := template.Execute(buf, values)

	if err != nil {
		log.Println(err)
	}

	return buf.String()
}

func headerForPackage(packageName string) string {
	return templateOutput(headerTemplate, map[string]string{
		"packageName": packageName,
	})
}

func readerForStruct(structName string) string {

	return templateOutput(readerTemplate, templateConfig{
		FirstLetter: structName[:1],
		StructName:  structName,
	})

}

func readSetterForStruct(structName string) string {
	return templateOutput(readSetterTemplate, templateConfig{
		FirstLetter: structName[:1],
		StructName:  structName,
	})
}

const headerTemplateText = `/************************************
 *
 * This file was auto-generated by autoreader. DO NOT EDIT.
 *
 ************************************/
package {{.packageName}}

import (
	"github.com/jkomoros/boardgame"
)

`

const readerTemplateText = `func ({{.FirstLetter}} *{{.StructName}}) Reader() boardgame.PropertyReader {
	return boardgame.DefaultReader({{.FirstLetter}})
}

`

const readSetterTemplateText = `func ({{.FirstLetter}} *{{.StructName}}) ReadSetter() boardgame.PropertyReadSetter {
	return boardgame.DefaultReadSetter({{.FirstLetter}})
}

`
<|endoftext|>


<commit_before>/*

	Autoreader is a simple program, designed to be run from go:generate, that
	helps generate the annoying boilerplate to implement
	boardgame.PropertyReader and boardgame.PropertyReadSetter.

*/
package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"github.com/MarcGrol/golangAnnotations/parser"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"text/template"
)

var headerTemplate *template.Template
var readerTemplate *template.Template
var readSetterTemplate *template.Template

const magicDocLinePrefix = "+autoreader"

type appOptions struct {
	OutputFile       string
	PackageDirectory string
	PrintToConsole   bool
	Help             bool
	flagSet          *flag.FlagSet
}

type templateConfig struct {
	FirstLetter string
	StructName  string
}

func init() {
	headerTemplate = template.Must(template.New("header").Parse(headerTemplateText))
	readerTemplate = template.Must(template.New("reader").Parse(readerTemplateText))
	readSetterTemplate = template.Must(template.New("readsetter").Parse(readSetterTemplateText))
}

func defineFlags(options *appOptions) {
	options.flagSet.StringVar(&options.OutputFile, "out", "auto_reader.go", "Defines which file to render output to. WARNING: it will be overwritten!")
	options.flagSet.StringVar(&options.PackageDirectory, "pkg", "examplepkg/", "Which package to process")
	options.flagSet.BoolVar(&options.Help, "h", false, "If set, print help message and quit.")
	options.flagSet.BoolVar(&options.PrintToConsole, "print", false, "If true, will print result to console instead of writing to out.")
}

func getOptions(flagSet *flag.FlagSet, flagArguments []string) *appOptions {
	options := &appOptions{flagSet: flagSet}
	defineFlags(options)
	flagSet.Parse(flagArguments)
	return options
}

func main() {
	flagSet := flag.CommandLine
	process(getOptions(flagSet, os.Args[1:]), os.Stdout, os.Stderr)
}

func process(options *appOptions, out io.ReadWriter, errOut io.ReadWriter) {

	if options.Help {
		options.flagSet.SetOutput(out)
		options.flagSet.PrintDefaults()
		return
	}

	output, err := processPackage(options.PackageDirectory)

	if err != nil {
		fmt.Fprintln(errOut, "ERROR", err)
		return
	}

	if options.PrintToConsole {
		fmt.Fprintln(out, output)
	} else {
		ioutil.WriteFile(options.OutputFile, []byte(output), 0644)
	}

}

func processPackage(location string) (output string, err error) {
	sources, err := parser.ParseSourceDir(location, ".*")

	if err != nil {
		return "", errors.New("Couldn't parse sources: " + err.Error())
	}

	haveOutputHeader := false

	for _, theStruct := range sources.Structs {

		if !haveOutputHeader {
			output += headerForPackage(theStruct.PackageName)
			haveOutputHeader = true
		}

		enableAutoReader := false

		for _, docLine := range theStruct.DocLines {
			docLine = strings.TrimPrefix(docLine, "//")
			docLine = strings.TrimSpace(docLine)
			if strings.HasPrefix(docLine, magicDocLinePrefix) {
				enableAutoReader = true
				break
			}
		}

		if !enableAutoReader {
			continue
		}

		output += readerForStruct(theStruct.Name)
		output += readSetterForStruct(theStruct.Name)
	}

	return output, nil
}

func templateOutput(template *template.Template, values interface{}) string {
	buf := new(bytes.Buffer)

	err := template.Execute(buf, values)

	if err != nil {
		log.Println(err)
	}

	return buf.String()
}

func headerForPackage(packageName string) string {
	return templateOutput(headerTemplate, map[string]string{
		"packageName": packageName,
	})
}

func readerForStruct(structName string) string {

	return templateOutput(readerTemplate, templateConfig{
		FirstLetter: structName[:1],
		StructName:  structName,
	})

}

func readSetterForStruct(structName string) string {
	return templateOutput(readSetterTemplate, templateConfig{
		FirstLetter: structName[:1],
		StructName:  structName,
	})
}

const headerTemplateText = `/************************************
 *
 * This file was auto-generated by autoreader. DO NOT EDIT.
 *
 ************************************/
package {{.packageName}}

import (
	"github.com/jkomoros/boardgame"
)

`

const readerTemplateText = `func ({{.FirstLetter}} *{{.StructName}}) Reader() boardgame.PropertyReader {
	return boardgame.DefaultReader({{.FirstLetter}})
}

`

const readSetterTemplateText = `func ({{.FirstLetter}} *{{.StructName}}) ReadSetter() boardgame.PropertyReadSetter {
	return boardgame.DefaultReadSetter({{.FirstLetter}})
}

`
<commit_msg>Set the default for package directory to "." now that we test the normal flow. Part of #301.<commit_after>/*

	Autoreader is a simple program, designed to be run from go:generate, that
	helps generate the annoying boilerplate to implement
	boardgame.PropertyReader and boardgame.PropertyReadSetter.

*/
package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"github.com/MarcGrol/golangAnnotations/parser"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"text/template"
)

var headerTemplate *template.Template
var readerTemplate *template.Template
var readSetterTemplate *template.Template

const magicDocLinePrefix = "+autoreader"

type appOptions struct {
	OutputFile       string
	PackageDirectory string
	PrintToConsole   bool
	Help             bool
	flagSet          *flag.FlagSet
}

type templateConfig struct {
	FirstLetter string
	StructName  string
}

func init() {
	headerTemplate = template.Must(template.New("header").Parse(headerTemplateText))
	readerTemplate = template.Must(template.New("reader").Parse(readerTemplateText))
	readSetterTemplate = template.Must(template.New("readsetter").Parse(readSetterTemplateText))
}

func defineFlags(options *appOptions) {
	options.flagSet.StringVar(&options.OutputFile, "out", "auto_reader.go", "Defines which file to render output to. WARNING: it will be overwritten!")
	options.flagSet.StringVar(&options.PackageDirectory, "pkg", ".", "Which package to process")
	options.flagSet.BoolVar(&options.Help, "h", false, "If set, print help message and quit.")
	options.flagSet.BoolVar(&options.PrintToConsole, "print", false, "If true, will print result to console instead of writing to out.")
}

func getOptions(flagSet *flag.FlagSet, flagArguments []string) *appOptions {
	options := &appOptions{flagSet: flagSet}
	defineFlags(options)
	flagSet.Parse(flagArguments)
	return options
}

func main() {
	flagSet := flag.CommandLine
	process(getOptions(flagSet, os.Args[1:]), os.Stdout, os.Stderr)
}

func process(options *appOptions, out io.ReadWriter, errOut io.ReadWriter) {

	if options.Help {
		options.flagSet.SetOutput(out)
		options.flagSet.PrintDefaults()
		return
	}

	output, err := processPackage(options.PackageDirectory)

	if err != nil {
		fmt.Fprintln(errOut, "ERROR", err)
		return
	}

	if options.PrintToConsole {
		fmt.Fprintln(out, output)
	} else {
		ioutil.WriteFile(options.OutputFile, []byte(output), 0644)
	}

}

func processPackage(location string) (output string, err error) {
	sources, err := parser.ParseSourceDir(location, ".*")

	if err != nil {
		return "", errors.New("Couldn't parse sources: " + err.Error())
	}

	haveOutputHeader := false

	for _, theStruct := range sources.Structs {

		if !haveOutputHeader {
			output += headerForPackage(theStruct.PackageName)
			haveOutputHeader = true
		}

		enableAutoReader := false

		for _, docLine := range theStruct.DocLines {
			docLine = strings.TrimPrefix(docLine, "//")
			docLine = strings.TrimSpace(docLine)
			if strings.HasPrefix(docLine, magicDocLinePrefix) {
				enableAutoReader = true
				break
			}
		}

		if !enableAutoReader {
			continue
		}

		output += readerForStruct(theStruct.Name)
		output += readSetterForStruct(theStruct.Name)
	}

	return output, nil
}

func templateOutput(template *template.Template, values interface{}) string {
	buf := new(bytes.Buffer)

	err := template.Execute(buf, values)

	if err != nil {
		log.Println(err)
	}

	return buf.String()
}

func headerForPackage(packageName string) string {
	return templateOutput(headerTemplate, map[string]string{
		"packageName": packageName,
	})
}

func readerForStruct(structName string) string {

	return templateOutput(readerTemplate, templateConfig{
		FirstLetter: structName[:1],
		StructName:  structName,
	})

}

func readSetterForStruct(structName string) string {
	return templateOutput(readSetterTemplate, templateConfig{
		FirstLetter: structName[:1],
		StructName:  structName,
	})
}

const headerTemplateText = `/************************************
 *
 * This file was auto-generated by autoreader. DO NOT EDIT.
 *
 ************************************/
package {{.packageName}}

import (
	"github.com/jkomoros/boardgame"
)

`

const readerTemplateText = `func ({{.FirstLetter}} *{{.StructName}}) Reader() boardgame.PropertyReader {
	return boardgame.DefaultReader({{.FirstLetter}})
}

`

const readSetterTemplateText = `func ({{.FirstLetter}} *{{.StructName}}) ReadSetter() boardgame.PropertyReadSetter {
	return boardgame.DefaultReadSetter({{.FirstLetter}})
}

`
<|endoftext|>


<commit_before>/*
Copyright 2018 Google Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"context"
	"encoding/gob"
	"flag"
	"net/http"
	"time"

	"go.opencensus.io/exporter/prometheus"
	"go.opencensus.io/stats/view"
	"go.uber.org/zap"

	"github.com/knative/serving/cmd/util"
	"github.com/knative/serving/pkg/apis/serving/v1alpha1"
	ela_autoscaler "github.com/knative/serving/pkg/autoscaler"
	clientset "github.com/knative/serving/pkg/client/clientset/versioned"
	"github.com/knative/serving/pkg/logging"
	"github.com/knative/serving/pkg/logging/logkey"
	"github.com/josephburnett/k8sflag/pkg/k8sflag"

	"github.com/gorilla/websocket"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
)

const (
	// A big enough buffer to handle 1000 pods sending stats every 1
	// second while we do the autoscaling computation (a few hundred
	// milliseconds).
	statBufferSize = 1000

	// Enough buffer to store scale requests generated every 2
	// seconds while an http request is taking the full timeout of 5
	// second.
	scaleBufferSize = 10
)

var (
	upgrader          = websocket.Upgrader{}
	elaClient         clientset.Interface
	kubeClient        *kubernetes.Clientset
	statChan          = make(chan ela_autoscaler.Stat, statBufferSize)
	scaleChan         = make(chan int32, scaleBufferSize)
	statsReporter     ela_autoscaler.StatsReporter
	elaNamespace      string
	elaDeployment     string
	elaConfig         string
	elaRevision       string
	elaAutoscalerPort string
	logger            *zap.SugaredLogger

	// Revision-level configuration
	concurrencyModel = flag.String("concurrencyModel", string(v1alpha1.RevisionRequestConcurrencyModelMulti), "")

	// Cluster-level configuration
	autoscaleFlagSet        = k8sflag.NewFlagSet("/etc/config-autoscaler")
	enableScaleToZero       = autoscaleFlagSet.Bool("enable-scale-to-zero", false)
	multiConcurrencyTarget  = autoscaleFlagSet.Float64("multi-concurrency-target", 0.0, k8sflag.Required)
	singleConcurrencyTarget = autoscaleFlagSet.Float64("single-concurrency-target", 0.0, k8sflag.Required)
)

func initEnv() {
	elaNamespace = util.GetRequiredEnvOrFatal("ELA_NAMESPACE", logger)
	elaDeployment = util.GetRequiredEnvOrFatal("ELA_DEPLOYMENT", logger)
	elaConfig = util.GetRequiredEnvOrFatal("ELA_CONFIGURATION", logger)
	elaRevision = util.GetRequiredEnvOrFatal("ELA_REVISION", logger)
	elaAutoscalerPort = util.GetRequiredEnvOrFatal("ELA_AUTOSCALER_PORT", logger)
}

func autoscaler() {
	var targetConcurrency *k8sflag.Float64Flag
	switch *concurrencyModel {
	case string(v1alpha1.RevisionRequestConcurrencyModelSingle):
		targetConcurrency = singleConcurrencyTarget
	case string(v1alpha1.RevisionRequestConcurrencyModelMulti):
		targetConcurrency = multiConcurrencyTarget
	default:
		logger.Fatalf("Unrecognized concurrency model: " + *concurrencyModel)
	}
	config := ela_autoscaler.Config{
		TargetConcurrency:    targetConcurrency,
		MaxScaleUpRate:       autoscaleFlagSet.Float64("max-scale-up-rate", 0.0, k8sflag.Required),
		StableWindow:         autoscaleFlagSet.Duration("stable-window", nil, k8sflag.Required),
		PanicWindow:          autoscaleFlagSet.Duration("panic-window", nil, k8sflag.Required),
		ScaleToZeroThreshold: autoscaleFlagSet.Duration("scale-to-zero-threshold", nil, k8sflag.Required, k8sflag.Dynamic),
	}
	a := ela_autoscaler.NewAutoscaler(config, statsReporter)
	ticker := time.NewTicker(2 * time.Second)
	ctx := logging.WithLogger(context.TODO(), logger)

	for {
		select {
		case <-ticker.C:
			scale, ok := a.Scale(ctx, time.Now())
			if ok {
				// Flag guard scale to zero.
				if !enableScaleToZero.Get() && scale == 0 {
					continue
				}

				scaleChan <- scale

				// Stop the autoscaler from doing any more work.
				if scale == 0 {
					return
				}
			}
		case s := <-statChan:
			a.Record(ctx, s)
		}
	}
}

func scaleSerializer() {
	for {
		select {
		case desiredPodCount := <-scaleChan:
		FastForward:
			// Fast forward to the most recent desired pod
			// count since the http timeout (5 sec) is more
			// than the autoscaling rate (2 sec) and there
			// could be multiple pending scale requests.
			for {
				select {
				case p := <-scaleChan:
					logger.Info("Scaling is not keeping up with autoscaling requests.")
					desiredPodCount = p
				default:
					break FastForward
				}
			}
			scaleTo(desiredPodCount)
		}
	}
}

func scaleTo(podCount int32) {
	dc := kubeClient.ExtensionsV1beta1().Deployments(elaNamespace)
	deployment, err := dc.Get(elaDeployment, metav1.GetOptions{})
	if err != nil {
		logger.Error("Error getting Deployment %q: %s", elaDeployment, zap.Error(err))
		return
	}
	logger.Debugf("===SCALE=== %v %v %v %v %v",
		time.Now().Unix(),
		podCount,
		deployment.Status.Replicas,
		deployment.Status.AvailableReplicas,
		deployment.Status.ReadyReplicas)
	statsReporter.Report(ela_autoscaler.DesiredPodCountM, (int64)(podCount))
	statsReporter.Report(ela_autoscaler.RequestedPodCountM, (int64)(deployment.Status.Replicas))
	statsReporter.Report(ela_autoscaler.ActualPodCountM, (int64)(deployment.Status.ReadyReplicas))

	if *deployment.Spec.Replicas == podCount {
		return
	}

	logger.Infof("Scaling to %v", podCount)
	if podCount == 0 {
		revisionClient := elaClient.ServingV1alpha1().Revisions(elaNamespace)
		revision, err := revisionClient.Get(elaRevision, metav1.GetOptions{})

		if err != nil {
			logger.Errorf("Error getting Revision %q: %s", elaRevision, zap.Error(err))
		}
		revision.Spec.ServingState = v1alpha1.RevisionServingStateReserve
		revision, err = revisionClient.Update(revision)
		if err != nil {
			logger.Errorf("Error updating Revision %q: %s", elaRevision, zap.Error(err))
		}

	}
	deployment.Spec.Replicas = &podCount
	_, err = dc.Update(deployment)
	if err != nil {
		logger.Errorf("Error updating Deployment %q: %s", elaDeployment, err)
	}
	logger.Info("Successfully scaled.")
}

func handler(w http.ResponseWriter, r *http.Request) {
	conn, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		logger.Error("Failed to upgrade http connection to websocket", zap.Error(err))
		return
	}
	for {
		messageType, msg, err := conn.ReadMessage()
		if err != nil {
			return
		}
		if messageType != websocket.BinaryMessage {
			logger.Error("Dropping non-binary message.")
			continue
		}
		dec := gob.NewDecoder(bytes.NewBuffer(msg))
		var stat ela_autoscaler.Stat
		err = dec.Decode(&stat)
		if err != nil {
			logger.Error("Failed to decode stats", zap.Error(err))
			continue
		}
		statChan <- stat
	}
}

func main() {
	flag.Parse()
	logger = logging.NewLoggerFromDefaultConfigMap("loglevel.autoscaler").Named("ela-autoscaler")
	defer logger.Sync()

	initEnv()
	logger = logger.With(
		zap.String(logkey.Namespace, elaNamespace),
		zap.String(logkey.Configuration, elaConfig),
		zap.String(logkey.Revision, elaRevision))

	logger.Info("Starting autoscaler")
	config, err := rest.InClusterConfig()
	if err != nil {
		logger.Fatal("Failed to get in cluster configuration", zap.Error(err))
	}
	config.Timeout = time.Duration(5 * time.Second)
	kc, err := kubernetes.NewForConfig(config)
	if err != nil {
		logger.Fatal("Failed to create a new clientset", zap.Error(err))
	}
	kubeClient = kc
	ec, err := clientset.NewForConfig(config)
	if err != nil {
		logger.Fatal("Failed to create a new clientset", zap.Error(err))
	}
	elaClient = ec

	exporter, err := prometheus.NewExporter(prometheus.Options{Namespace: "autoscaler"})
	if err != nil {
		logger.Fatal("Failed to create prometheus exporter", zap.Error(err))
	}
	view.RegisterExporter(exporter)
	view.SetReportingPeriod(1 * time.Second)

	reporter, err := ela_autoscaler.NewStatsReporter(elaNamespace, elaConfig, elaRevision)
	if err != nil {
		logger.Fatal("Failed to create stats reporter", zap.Error(err))
	}
	statsReporter = reporter

	go autoscaler()
	go scaleSerializer()

	mux := http.NewServeMux()
	mux.HandleFunc("/", handler)
	mux.Handle("/metrics", exporter)
	http.ListenAndServe(":"+elaAutoscalerPort, mux)
}
<commit_msg>Autoscaler no stop. (#1015)<commit_after>/*
Copyright 2018 Google Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"context"
	"encoding/gob"
	"flag"
	"net/http"
	"time"

	"go.opencensus.io/exporter/prometheus"
	"go.opencensus.io/stats/view"
	"go.uber.org/zap"

	"github.com/knative/serving/cmd/util"
	"github.com/knative/serving/pkg/apis/serving/v1alpha1"
	ela_autoscaler "github.com/knative/serving/pkg/autoscaler"
	clientset "github.com/knative/serving/pkg/client/clientset/versioned"
	"github.com/knative/serving/pkg/logging"
	"github.com/knative/serving/pkg/logging/logkey"
	"github.com/josephburnett/k8sflag/pkg/k8sflag"

	"github.com/gorilla/websocket"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
)

const (
	// A big enough buffer to handle 1000 pods sending stats every 1
	// second while we do the autoscaling computation (a few hundred
	// milliseconds).
	statBufferSize = 1000

	// Enough buffer to store scale requests generated every 2
	// seconds while an http request is taking the full timeout of 5
	// second.
	scaleBufferSize = 10
)

var (
	upgrader          = websocket.Upgrader{}
	elaClient         clientset.Interface
	kubeClient        *kubernetes.Clientset
	statChan          = make(chan ela_autoscaler.Stat, statBufferSize)
	scaleChan         = make(chan int32, scaleBufferSize)
	statsReporter     ela_autoscaler.StatsReporter
	elaNamespace      string
	elaDeployment     string
	elaConfig         string
	elaRevision       string
	elaAutoscalerPort string
	currentScale      int32
	logger            *zap.SugaredLogger

	// Revision-level configuration
	concurrencyModel = flag.String("concurrencyModel", string(v1alpha1.RevisionRequestConcurrencyModelMulti), "")

	// Cluster-level configuration
	autoscaleFlagSet        = k8sflag.NewFlagSet("/etc/config-autoscaler")
	enableScaleToZero       = autoscaleFlagSet.Bool("enable-scale-to-zero", false)
	multiConcurrencyTarget  = autoscaleFlagSet.Float64("multi-concurrency-target", 0.0, k8sflag.Required)
	singleConcurrencyTarget = autoscaleFlagSet.Float64("single-concurrency-target", 0.0, k8sflag.Required)
)

func initEnv() {
	elaNamespace = util.GetRequiredEnvOrFatal("ELA_NAMESPACE", logger)
	elaDeployment = util.GetRequiredEnvOrFatal("ELA_DEPLOYMENT", logger)
	elaConfig = util.GetRequiredEnvOrFatal("ELA_CONFIGURATION", logger)
	elaRevision = util.GetRequiredEnvOrFatal("ELA_REVISION", logger)
	elaAutoscalerPort = util.GetRequiredEnvOrFatal("ELA_AUTOSCALER_PORT", logger)
}

func autoscaler() {
	var targetConcurrency *k8sflag.Float64Flag
	switch *concurrencyModel {
	case string(v1alpha1.RevisionRequestConcurrencyModelSingle):
		targetConcurrency = singleConcurrencyTarget
	case string(v1alpha1.RevisionRequestConcurrencyModelMulti):
		targetConcurrency = multiConcurrencyTarget
	default:
		logger.Fatalf("Unrecognized concurrency model: " + *concurrencyModel)
	}
	config := ela_autoscaler.Config{
		TargetConcurrency:    targetConcurrency,
		MaxScaleUpRate:       autoscaleFlagSet.Float64("max-scale-up-rate", 0.0, k8sflag.Required),
		StableWindow:         autoscaleFlagSet.Duration("stable-window", nil, k8sflag.Required),
		PanicWindow:          autoscaleFlagSet.Duration("panic-window", nil, k8sflag.Required),
		ScaleToZeroThreshold: autoscaleFlagSet.Duration("scale-to-zero-threshold", nil, k8sflag.Required, k8sflag.Dynamic),
	}
	a := ela_autoscaler.NewAutoscaler(config, statsReporter)
	ticker := time.NewTicker(2 * time.Second)
	ctx := logging.WithLogger(context.TODO(), logger)

	for {
		select {
		case <-ticker.C:
			scale, ok := a.Scale(ctx, time.Now())
			if ok {
				// Flag guard scale to zero.
				if !enableScaleToZero.Get() && scale == 0 {
					continue
				}

				scaleChan <- scale
			}
		case s := <-statChan:
			a.Record(ctx, s)
		}
	}
}

func scaleSerializer() {
	for {
		select {
		case desiredPodCount := <-scaleChan:
		FastForward:
			// Fast forward to the most recent desired pod
			// count since the http timeout (5 sec) is more
			// than the autoscaling rate (2 sec) and there
			// could be multiple pending scale requests.
			for {
				select {
				case p := <-scaleChan:
					logger.Info("Scaling is not keeping up with autoscaling requests.")
					desiredPodCount = p
				default:
					break FastForward
				}
			}
			scaleTo(desiredPodCount)
		}
	}
}

func scaleTo(podCount int32) {
	statsReporter.Report(ela_autoscaler.DesiredPodCountM, (int64)(podCount))
	if currentScale == podCount {
		return
	}
	dc := kubeClient.ExtensionsV1beta1().Deployments(elaNamespace)
	deployment, err := dc.Get(elaDeployment, metav1.GetOptions{})
	if err != nil {
		logger.Error("Error getting Deployment %q: %s", elaDeployment, zap.Error(err))
		return
	}
	statsReporter.Report(ela_autoscaler.DesiredPodCountM, (int64)(podCount))
	statsReporter.Report(ela_autoscaler.RequestedPodCountM, (int64)(deployment.Status.Replicas))
	statsReporter.Report(ela_autoscaler.ActualPodCountM, (int64)(deployment.Status.ReadyReplicas))

	if *deployment.Spec.Replicas == podCount {
		currentScale = podCount
		return
	}

	logger.Infof("Scaling from %v to %v", currentScale, podCount)
	if podCount == 0 {
		revisionClient := elaClient.ServingV1alpha1().Revisions(elaNamespace)
		revision, err := revisionClient.Get(elaRevision, metav1.GetOptions{})

		if err != nil {
			logger.Errorf("Error getting Revision %q: %s", elaRevision, zap.Error(err))
		}
		revision.Spec.ServingState = v1alpha1.RevisionServingStateReserve
		revision, err = revisionClient.Update(revision)
		if err != nil {
			logger.Errorf("Error updating Revision %q: %s", elaRevision, zap.Error(err))
		}
		currentScale = 0
		return
	}
	deployment.Spec.Replicas = &podCount
	_, err = dc.Update(deployment)
	if err != nil {
		logger.Errorf("Error updating Deployment %q: %s", elaDeployment, err)
	}
	logger.Info("Successfully scaled.")
	currentScale = podCount
}

func handler(w http.ResponseWriter, r *http.Request) {
	conn, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		logger.Error("Failed to upgrade http connection to websocket", zap.Error(err))
		return
	}
	for {
		messageType, msg, err := conn.ReadMessage()
		if err != nil {
			return
		}
		if messageType != websocket.BinaryMessage {
			logger.Error("Dropping non-binary message.")
			continue
		}
		dec := gob.NewDecoder(bytes.NewBuffer(msg))
		var stat ela_autoscaler.Stat
		err = dec.Decode(&stat)
		if err != nil {
			logger.Error("Failed to decode stats", zap.Error(err))
			continue
		}
		statChan <- stat
	}
}

func main() {
	flag.Parse()
	logger = logging.NewLoggerFromDefaultConfigMap("loglevel.autoscaler").Named("ela-autoscaler")
	defer logger.Sync()

	initEnv()
	logger = logger.With(
		zap.String(logkey.Namespace, elaNamespace),
		zap.String(logkey.Configuration, elaConfig),
		zap.String(logkey.Revision, elaRevision))

	logger.Info("Starting autoscaler")
	config, err := rest.InClusterConfig()
	if err != nil {
		logger.Fatal("Failed to get in cluster configuration", zap.Error(err))
	}
	config.Timeout = time.Duration(5 * time.Second)
	kc, err := kubernetes.NewForConfig(config)
	if err != nil {
		logger.Fatal("Failed to create a new clientset", zap.Error(err))
	}
	kubeClient = kc
	ec, err := clientset.NewForConfig(config)
	if err != nil {
		logger.Fatal("Failed to create a new clientset", zap.Error(err))
	}
	elaClient = ec

	exporter, err := prometheus.NewExporter(prometheus.Options{Namespace: "autoscaler"})
	if err != nil {
		logger.Fatal("Failed to create prometheus exporter", zap.Error(err))
	}
	view.RegisterExporter(exporter)
	view.SetReportingPeriod(1 * time.Second)

	reporter, err := ela_autoscaler.NewStatsReporter(elaNamespace, elaConfig, elaRevision)
	if err != nil {
		logger.Fatal("Failed to create stats reporter", zap.Error(err))
	}
	statsReporter = reporter

	go autoscaler()
	go scaleSerializer()

	mux := http.NewServeMux()
	mux.HandleFunc("/", handler)
	mux.Handle("/metrics", exporter)
	http.ListenAndServe(":"+elaAutoscalerPort, mux)
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"net/url"
	"strconv"
	"strings"

	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/app"
	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/app/with"
	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/util"
	"github.com/BytemarkHosting/bytemark-client/lib"
	"github.com/BytemarkHosting/bytemark-client/util/log"
	"github.com/urfave/cli"
)

func validateEndpointForConfig(endpoint string) error {
	url, err := url.Parse(endpoint)
	if err != nil {
		return err
	}
	if url.Scheme != "http" && url.Scheme != "https" {
		return errors.New("The endpoint URL should start with http:// or https://")
	}
	if url.Host == "" {
		return errors.New("The endpoint URL should have a hostname")
	}
	return nil
}

func validateAccountForConfig(c *app.Context, name string) (err error) {
	_, err = c.Client().GetAccount(name)
	if err != nil {
		if _, ok := err.(lib.NotFoundError); ok {
			return fmt.Errorf("No such account %s - check your typing and specify --yubikey if necessary", name)
		}
		return err
	}
	return
}

func validateGroupForConfig(c *app.Context, name string) (err error) {
	// we can't just use with.Group because it expects NextArg() to be the account name - there's no way to pass one in.
	groupName := lib.ParseGroupName(name, c.Config().GetGroup())
	_, err = c.Client().GetGroup(groupName)
	if err != nil {
		if _, ok := err.(lib.NotFoundError); ok {
			return fmt.Errorf("No such group %v - check your typing and specify --yubikey if necessary", groupName)
		}
		return err
	}
	return
}

func validateConfigValue(c *app.Context, varname string, value string) error {
	if c.Bool("force") {
		return nil
	}
	switch varname {
	case "endpoint", "api-endpoint", "billing-endpoint", "spp-endpoint", "auth-endpoint":
		return validateEndpointForConfig(value)
	case "account":
		return validateAccountForConfig(c, value)
	case "group":
		return validateGroupForConfig(c, value)
	case "debug-level":
		_, err := strconv.ParseUint(value, 10, 32)
		if err != nil {
			return errors.New("debug-level must be an integer")
		}
	}
	return nil
}

func init() {
	commands = append(commands, cli.Command{
		Name:      "config",
		Usage:     "manage the bytemark client's configuration",
		UsageText: "config [ set | unset ]",
		Description: `view and manipulate the bytemark-client configuration

When invoked with no subcommand, outputs the current values of all variables and what source they were derived from.
The set and unset subcommands can be used to set and unset such variables.

    Available variables:
        account - the default account, used when you do not explicitly state an account - defaults to the same as your user name
        token - the token used for authentication
        user - the user that you log in as by default
        group - the default group, used when you do not explicitly state a group (defaults to 'default')

        debug-level - the default debug level. Set to 0 unless you like lots of output.
	api-endpoint - the endpoint for domains (among other things?)
        auth-endpoint - the endpoint to authenticate to. https://auth.bytemark.co.uk is the default.
        endpoint - the brain endpoint to connect to. https://uk0.bigv.io is the default.
        billing-endpoint - the billing API endpoint to connect to. https://bmbilling.bytemark.co.uk is the default.
        spp-endpoint - the SPP endpoint to use. https://spp-submissions.bytemark.co.uk is the default.`,
		Subcommands: []cli.Command{
			{
				Name:        "set",
				UsageText:   "bytemark config set <variable> <value>",
				Usage:       "sets a bytemark client configuration request",
				Description: "Sets the named variable to the given value. See `bytemark help config` for which variables are available",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:  "force",
						Usage: "Don't run any validation checks against the value",
					},
				},
				Action: app.Action(func(ctx *app.Context) error {
					varname, err := ctx.NextArg()
					if err != nil {
						return err
					}
					varname = strings.ToLower(varname)

					if !util.IsConfigVar(varname) {
						return ctx.Help(fmt.Sprintf("%s is not a valid variable name", varname))
					}

					oldVar, err := ctx.Config().GetV(varname)
					if err != nil {
						return err
					}

					value, err := ctx.NextArg()
					if err != nil {
						return err
					}

					if varname == "account" || varname == "group" {
						err = with.Auth(ctx)
						if err != nil {
							return err
						}
					}

					err = validateConfigValue(ctx, varname, value)
					if err != nil {
						return err
					}

					err = ctx.Config().SetPersistent(varname, value, "CMD set")
					if err != nil {
						return err
					}

					if oldVar.Source == "config" {
						log.Logf("%s has been changed.\r\nOld value: %s\r\nNew value: %s\r\n", varname, oldVar.Value, ctx.Config().GetIgnoreErr(varname))
					} else {
						log.Logf("%s has been set. \r\nNew value: %s\r\n", varname, ctx.Config().GetIgnoreErr(varname))
					}
					return nil
				}),
			}, {
				Name:        "unset",
				UsageText:   "bytemark config unset <variable>",
				Usage:       "unsets a bytemark client configuration option",
				Description: "Unsets the named variable.",
				Action: app.Action(func(ctx *app.Context) error {
					varname, err := ctx.NextArg()
					if err != nil {
						return err
					}
					varname = strings.ToLower(varname)
					return ctx.Config().Unset(varname)
				}),
			},
		},
		Action: app.Action(func(ctx *app.Context) (err error) {
			if ctx.Bool("help") {
				if ccw, ok := ctx.Context.(cliContextWrapper); ok {
					err = cli.ShowSubcommandHelp(ccw.Context)
					return
				}
			}
			vars, err := ctx.Config().GetAll()
			if err != nil {
				return
			}
			for _, v := range vars {
				log.Logf("%s\t: '%s' (%s)\r\n", v.Name, v.Value, v.Source)
			}
			return
		}),
	})
}
<commit_msg>Fix config<commit_after>package main

import (
	"errors"
	"fmt"
	"net/url"
	"strconv"
	"strings"

	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/app"
	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/app/with"
	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/util"
	"github.com/BytemarkHosting/bytemark-client/lib"
	"github.com/BytemarkHosting/bytemark-client/util/log"
	"github.com/urfave/cli"
)

func validateEndpointForConfig(endpoint string) error {
	url, err := url.Parse(endpoint)
	if err != nil {
		return err
	}
	if url.Scheme != "http" && url.Scheme != "https" {
		return errors.New("The endpoint URL should start with http:// or https://")
	}
	if url.Host == "" {
		return errors.New("The endpoint URL should have a hostname")
	}
	return nil
}

func validateAccountForConfig(c *app.Context, name string) (err error) {
	_, err = c.Client().GetAccount(name)
	if err != nil {
		if _, ok := err.(lib.NotFoundError); ok {
			return fmt.Errorf("No such account %s - check your typing and specify --yubikey if necessary", name)
		}
		return err
	}
	return
}

func validateGroupForConfig(c *app.Context, name string) (err error) {
	// we can't just use with.Group because it expects NextArg() to be the account name - there's no way to pass one in.
	groupName := lib.ParseGroupName(name, c.Config().GetGroup())
	_, err = c.Client().GetGroup(groupName)
	if err != nil {
		if _, ok := err.(lib.NotFoundError); ok {
			return fmt.Errorf("No such group %v - check your typing and specify --yubikey if necessary", groupName)
		}
		return err
	}
	return
}

func validateConfigValue(c *app.Context, varname string, value string) error {
	if c.Bool("force") {
		return nil
	}
	switch varname {
	case "endpoint", "api-endpoint", "billing-endpoint", "spp-endpoint", "auth-endpoint":
		return validateEndpointForConfig(value)
	case "account":
		return validateAccountForConfig(c, value)
	case "group":
		return validateGroupForConfig(c, value)
	case "debug-level":
		_, err := strconv.ParseUint(value, 10, 32)
		if err != nil {
			return errors.New("debug-level must be an integer")
		}
	}
	return nil
}

func init() {
	commands = append(commands, cli.Command{
		Name:      "config",
		Usage:     "manage the bytemark client's configuration",
		UsageText: "config [ set | unset ]",
		Description: `view and manipulate the bytemark-client configuration

When invoked with no subcommand, outputs the current values of all variables and what source they were derived from.
The set and unset subcommands can be used to set and unset such variables.

    Available variables:
        account - the default account, used when you do not explicitly state an account - defaults to the same as your user name
        token - the token used for authentication
        user - the user that you log in as by default
        group - the default group, used when you do not explicitly state a group (defaults to 'default')

        debug-level - the default debug level. Set to 0 unless you like lots of output.
	api-endpoint - the endpoint for domains (among other things?)
        auth-endpoint - the endpoint to authenticate to. https://auth.bytemark.co.uk is the default.
        endpoint - the brain endpoint to connect to. https://uk0.bigv.io is the default.
        billing-endpoint - the billing API endpoint to connect to. https://bmbilling.bytemark.co.uk is the default.
        spp-endpoint - the SPP endpoint to use. https://spp-submissions.bytemark.co.uk is the default.`,
		Subcommands: []cli.Command{
			{
				Name:        "set",
				UsageText:   "bytemark config set <variable> <value>",
				Usage:       "sets a bytemark client configuration request",
				Description: "Sets the named variable to the given value. See `bytemark help config` for which variables are available",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:  "force",
						Usage: "Don't run any validation checks against the value",
					},
				},
				Action: app.Action(func(ctx *app.Context) error {
					varname, err := ctx.NextArg()
					if err != nil {
						return err
					}
					varname = strings.ToLower(varname)

					if !util.IsConfigVar(varname) {
						return ctx.Help(fmt.Sprintf("%s is not a valid variable name", varname))
					}

					oldVar, err := ctx.Config().GetV(varname)
					if err != nil {
						return err
					}

					value, err := ctx.NextArg()
					if err != nil {
						return err
					}

					if varname == "account" || varname == "group" {
						err = with.Auth(ctx)
						if err != nil {
							return err
						}
					}

					err = validateConfigValue(ctx, varname, value)
					if err != nil {
						return err
					}

					err = ctx.Config().SetPersistent(varname, value, "CMD set")
					if err != nil {
						return err
					}

					if oldVar.Source == "config" {
						log.Logf("%s has been changed.\r\nOld value: %s\r\nNew value: %s\r\n", varname, oldVar.Value, ctx.Config().GetIgnoreErr(varname))
					} else {
						log.Logf("%s has been set. \r\nNew value: %s\r\n", varname, ctx.Config().GetIgnoreErr(varname))
					}
					return nil
				}),
			}, {
				Name:        "unset",
				UsageText:   "bytemark config unset <variable>",
				Usage:       "unsets a bytemark client configuration option",
				Description: "Unsets the named variable.",
				Action: app.Action(func(ctx *app.Context) error {
					varname, err := ctx.NextArg()
					if err != nil {
						return err
					}
					varname = strings.ToLower(varname)
					return ctx.Config().Unset(varname)
				}),
			},
		},
		Action: app.Action(func(ctx *app.Context) (err error) {
			if ctx.Bool("help") {
				if ccw, ok := ctx.Context.(app.CliContextWrapper); ok {
					err = cli.ShowSubcommandHelp(ccw.Context)
					return
				}
			}
			vars, err := ctx.Config().GetAll()
			if err != nil {
				return
			}
			for _, v := range vars {
				log.Logf("%s\t: '%s' (%s)\r\n", v.Name, v.Value, v.Source)
			}
			return
		}),
	})
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"log"
	"time"

	kubeinformers "k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/clientcmd"

	"github.com/tektoncd/pipeline/pkg/logging"

	sharedclientset "github.com/knative/pkg/client/clientset/versioned"
	"github.com/knative/pkg/controller"
	"github.com/tektoncd/pipeline/pkg/reconciler"
	"github.com/tektoncd/pipeline/pkg/reconciler/v1alpha1/pipelinerun"
	"github.com/tektoncd/pipeline/pkg/reconciler/v1alpha1/taskrun"
	"github.com/tektoncd/pipeline/pkg/system"

	"github.com/knative/pkg/configmap"
	"github.com/knative/pkg/signals"
	clientset "github.com/tektoncd/pipeline/pkg/client/clientset/versioned"
	pipelineinformers "github.com/tektoncd/pipeline/pkg/client/informers/externalversions"
)

const (
	threadsPerController = 2
	resyncPeriod         = 10 * time.Hour
)

var (
	masterURL  string
	kubeconfig string
)

func main() {
	flag.Parse()
	loggingConfigMap, err := configmap.Load("/etc/config-logging")
	if err != nil {
		log.Fatalf("Error loading logging configuration: %v", err)
	}
	loggingConfig, err := logging.NewConfigFromMap(loggingConfigMap)
	if err != nil {
		log.Fatalf("Error parsing logging configuration: %v", err)
	}
	logger, atomicLevel := logging.NewLoggerFromConfig(loggingConfig, logging.ControllerLogKey)
	defer logger.Sync()

	logger.Info("Starting the Pipeline Controller")

	// set up signals so we handle the first shutdown signal gracefully
	stopCh := signals.SetupSignalHandler()

	cfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfig)
	if err != nil {
		logger.Fatalf("Error building kubeconfig: %v", err)
	}

	kubeClient, err := kubernetes.NewForConfig(cfg)
	if err != nil {
		logger.Fatalf("Error building kubernetes clientset: %v", err)
	}

	sharedClient, err := sharedclientset.NewForConfig(cfg)
	if err != nil {
		logger.Fatalf("Error building shared clientset: %v", err)
	}

	pipelineClient, err := clientset.NewForConfig(cfg)
	if err != nil {
		logger.Fatalf("Error building pipeline clientset: %v", err)
	}

	configMapWatcher := configmap.NewInformedWatcher(kubeClient, system.GetNamespace())

	opt := reconciler.Options{
		KubeClientSet:     kubeClient,
		SharedClientSet:   sharedClient,
		PipelineClientSet: pipelineClient,
		ConfigMapWatcher:  configMapWatcher,
		ResyncPeriod:      resyncPeriod,
		Logger:            logger,
	}

	kubeInformerFactory := kubeinformers.NewSharedInformerFactory(kubeClient, opt.ResyncPeriod)
	pipelineInformerFactory := pipelineinformers.NewSharedInformerFactory(pipelineClient, opt.ResyncPeriod)

	taskInformer := pipelineInformerFactory.Tekton().V1alpha1().Tasks()
	clusterTaskInformer := pipelineInformerFactory.Tekton().V1alpha1().ClusterTasks()
	taskRunInformer := pipelineInformerFactory.Tekton().V1alpha1().TaskRuns()
	resourceInformer := pipelineInformerFactory.Tekton().V1alpha1().PipelineResources()
	podInformer := kubeInformerFactory.Core().V1().Pods()

	pipelineInformer := pipelineInformerFactory.Tekton().V1alpha1().Pipelines()
	pipelineRunInformer := pipelineInformerFactory.Tekton().V1alpha1().PipelineRuns()
	timeoutHandler := reconciler.NewTimeoutHandler(kubeClient, pipelineClient, stopCh, logger)

	trc := taskrun.NewController(opt,
		taskRunInformer,
		taskInformer,
		clusterTaskInformer,
		resourceInformer,
		podInformer,
		nil, //entrypoint cache will be initialized by controller if not provided
		timeoutHandler,
	)
	prc := pipelinerun.NewController(opt,
		pipelineRunInformer,
		pipelineInformer,
		taskInformer,
		clusterTaskInformer,
		taskRunInformer,
		resourceInformer,
		timeoutHandler,
	)
	// Build all of our controllers, with the clients constructed above.
	controllers := []*controller.Impl{
		// Pipeline Controllers
		trc,
		prc,
	}
	timeoutHandler.SetTaskRunCallbackFunc(trc.Enqueue)
	timeoutHandler.SetPipelineRunCallbackFunc(prc.Enqueue)
	timeoutHandler.CheckTimeouts()

	// Watch the logging config map and dynamically update logging levels.
	configMapWatcher.Watch(logging.ConfigName, logging.UpdateLevelFromConfigMap(logger, atomicLevel, logging.ControllerLogKey))

	kubeInformerFactory.Start(stopCh)
	pipelineInformerFactory.Start(stopCh)
	if err := configMapWatcher.Start(stopCh); err != nil {
		logger.Fatalf("failed to start configuration manager: %v", err)
	}

	// Wait for the caches to be synced before starting controllers.
	logger.Info("Waiting for informer caches to sync")
	for i, synced := range []cache.InformerSynced{
		taskInformer.Informer().HasSynced,
		clusterTaskInformer.Informer().HasSynced,
		taskRunInformer.Informer().HasSynced,
		resourceInformer.Informer().HasSynced,
		podInformer.Informer().HasSynced,
	} {
		if ok := cache.WaitForCacheSync(stopCh, synced); !ok {
			logger.Fatalf("failed to wait for cache at index %v to sync", i)
		}
	}

	logger.Info("Starting controllers")
	// Start all of the controllers.
	for _, ctrlr := range controllers {
		go func(ctrlr *controller.Impl) {
			// We don't expect this to return until stop is called,
			// but if it does, propagate it back.
			if runErr := ctrlr.Run(threadsPerController, stopCh); runErr != nil {
				logger.Fatalf("Error running controller: %v", runErr)
			}
		}(ctrlr)
	}

	<-stopCh
}

func init() {
	flag.StringVar(&kubeconfig, "kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.")
	flag.StringVar(&masterURL, "master", "", "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.")
}
<commit_msg>Allow controller to watch specified namespace only<commit_after>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"log"
	"time"

	corev1 "k8s.io/api/core/v1"
	kubeinformers "k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/clientcmd"

	"github.com/tektoncd/pipeline/pkg/logging"

	sharedclientset "github.com/knative/pkg/client/clientset/versioned"
	"github.com/knative/pkg/controller"
	"github.com/tektoncd/pipeline/pkg/reconciler"
	"github.com/tektoncd/pipeline/pkg/reconciler/v1alpha1/pipelinerun"
	"github.com/tektoncd/pipeline/pkg/reconciler/v1alpha1/taskrun"
	"github.com/tektoncd/pipeline/pkg/system"

	"github.com/knative/pkg/configmap"
	"github.com/knative/pkg/signals"
	clientset "github.com/tektoncd/pipeline/pkg/client/clientset/versioned"
	pipelineinformers "github.com/tektoncd/pipeline/pkg/client/informers/externalversions"
)

const (
	threadsPerController = 2
	resyncPeriod         = 10 * time.Hour
)

var (
	masterURL  string
	kubeconfig string
	namespace  string
)

func main() {
	flag.Parse()
	loggingConfigMap, err := configmap.Load("/etc/config-logging")
	if err != nil {
		log.Fatalf("Error loading logging configuration: %v", err)
	}
	loggingConfig, err := logging.NewConfigFromMap(loggingConfigMap)
	if err != nil {
		log.Fatalf("Error parsing logging configuration: %v", err)
	}
	logger, atomicLevel := logging.NewLoggerFromConfig(loggingConfig, logging.ControllerLogKey)
	defer logger.Sync()

	logger.Info("Starting the Pipeline Controller")

	// set up signals so we handle the first shutdown signal gracefully
	stopCh := signals.SetupSignalHandler()

	cfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfig)
	if err != nil {
		logger.Fatalf("Error building kubeconfig: %v", err)
	}

	kubeClient, err := kubernetes.NewForConfig(cfg)
	if err != nil {
		logger.Fatalf("Error building kubernetes clientset: %v", err)
	}

	sharedClient, err := sharedclientset.NewForConfig(cfg)
	if err != nil {
		logger.Fatalf("Error building shared clientset: %v", err)
	}

	pipelineClient, err := clientset.NewForConfig(cfg)
	if err != nil {
		logger.Fatalf("Error building pipeline clientset: %v", err)
	}

	configMapWatcher := configmap.NewInformedWatcher(kubeClient, system.GetNamespace())

	opt := reconciler.Options{
		KubeClientSet:     kubeClient,
		SharedClientSet:   sharedClient,
		PipelineClientSet: pipelineClient,
		ConfigMapWatcher:  configMapWatcher,
		ResyncPeriod:      resyncPeriod,
		Logger:            logger,
	}

	kubeInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(kubeClient, opt.ResyncPeriod, kubeinformers.WithNamespace(namespace))
	pipelineInformerFactory := pipelineinformers.NewSharedInformerFactoryWithOptions(pipelineClient, opt.ResyncPeriod, pipelineinformers.WithNamespace(namespace))

	taskInformer := pipelineInformerFactory.Tekton().V1alpha1().Tasks()
	clusterTaskInformer := pipelineInformerFactory.Tekton().V1alpha1().ClusterTasks()
	taskRunInformer := pipelineInformerFactory.Tekton().V1alpha1().TaskRuns()
	resourceInformer := pipelineInformerFactory.Tekton().V1alpha1().PipelineResources()
	podInformer := kubeInformerFactory.Core().V1().Pods()

	pipelineInformer := pipelineInformerFactory.Tekton().V1alpha1().Pipelines()
	pipelineRunInformer := pipelineInformerFactory.Tekton().V1alpha1().PipelineRuns()
	timeoutHandler := reconciler.NewTimeoutHandler(kubeClient, pipelineClient, stopCh, logger)

	trc := taskrun.NewController(opt,
		taskRunInformer,
		taskInformer,
		clusterTaskInformer,
		resourceInformer,
		podInformer,
		nil, //entrypoint cache will be initialized by controller if not provided
		timeoutHandler,
	)
	prc := pipelinerun.NewController(opt,
		pipelineRunInformer,
		pipelineInformer,
		taskInformer,
		clusterTaskInformer,
		taskRunInformer,
		resourceInformer,
		timeoutHandler,
	)
	// Build all of our controllers, with the clients constructed above.
	controllers := []*controller.Impl{
		// Pipeline Controllers
		trc,
		prc,
	}
	timeoutHandler.SetTaskRunCallbackFunc(trc.Enqueue)
	timeoutHandler.SetPipelineRunCallbackFunc(prc.Enqueue)
	timeoutHandler.CheckTimeouts()

	// Watch the logging config map and dynamically update logging levels.
	configMapWatcher.Watch(logging.ConfigName, logging.UpdateLevelFromConfigMap(logger, atomicLevel, logging.ControllerLogKey))

	kubeInformerFactory.Start(stopCh)
	pipelineInformerFactory.Start(stopCh)
	if err := configMapWatcher.Start(stopCh); err != nil {
		logger.Fatalf("failed to start configuration manager: %v", err)
	}

	// Wait for the caches to be synced before starting controllers.
	logger.Info("Waiting for informer caches to sync")
	for i, synced := range []cache.InformerSynced{
		taskInformer.Informer().HasSynced,
		clusterTaskInformer.Informer().HasSynced,
		taskRunInformer.Informer().HasSynced,
		resourceInformer.Informer().HasSynced,
		podInformer.Informer().HasSynced,
	} {
		if ok := cache.WaitForCacheSync(stopCh, synced); !ok {
			logger.Fatalf("failed to wait for cache at index %v to sync", i)
		}
	}

	logger.Info("Starting controllers")
	// Start all of the controllers.
	for _, ctrlr := range controllers {
		go func(ctrlr *controller.Impl) {
			// We don't expect this to return until stop is called,
			// but if it does, propagate it back.
			if runErr := ctrlr.Run(threadsPerController, stopCh); runErr != nil {
				logger.Fatalf("Error running controller: %v", runErr)
			}
		}(ctrlr)
	}

	<-stopCh
}

func init() {
	flag.StringVar(&kubeconfig, "kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.")
	flag.StringVar(&masterURL, "master", "", "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.")
	flag.StringVar(&namespace, "namespace", corev1.NamespaceAll, "Namespace to restrict informer to. Optional, defaults to all namespaces.")
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/codeclimate/test-reporter/formatters"
	"github.com/codeclimate/test-reporter/formatters/clover"
	"github.com/codeclimate/test-reporter/formatters/coveragepy"
	"github.com/codeclimate/test-reporter/formatters/gocov"
	"github.com/codeclimate/test-reporter/formatters/lcov"
	"github.com/codeclimate/test-reporter/formatters/simplecov"
	"github.com/gobuffalo/envy"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

type CoverageFormatter struct {
	CoveragePath string
	In           formatters.Formatter
	InputType    string
	Output       string
	Prefix       string
	writer       io.Writer
}

var formatOptions = CoverageFormatter{}

// a prioritized list of the formatters to use
var formatterList = []string{"simplecov", "lcov", "coverage.py", "clover", "gocov"}

// a map of the formatters to use
var formatterMap = map[string]formatters.Formatter{
	"simplecov":   &simplecov.Formatter{},
	"lcov":        &lcov.Formatter{},
	"coverage.py": &coveragepy.Formatter{},
	"gocov":       &gocov.Formatter{},
	"clover":      &clover.Formatter{},
}

// formatCoverageCmd represents the format command
var formatCoverageCmd = &cobra.Command{
	Use:   "format-coverage",
	Short: "Locate, parse, and re-format supported coverage sources.",
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) != 0 {
			logrus.Debugf("coverage path %s", args[0])
			formatOptions.CoveragePath = args[0]
		}
		return runFormatter(formatOptions)
	},
}

func runFormatter(formatOptions CoverageFormatter) error {
	envy.Set("PREFIX", formatOptions.Prefix)

	// if a type is specified use that
	if formatOptions.InputType != "" {
		if f, ok := formatterMap[formatOptions.InputType]; ok {
			logrus.Debugf("using formatter %s", formatOptions.InputType)
			if formatOptions.CoveragePath != "" {
				_, err := f.Search(formatOptions.CoveragePath)
				if err != nil {
					logrus.Errorf("could not find coverage file %s\n%s", formatOptions.CoveragePath, err)
					return errors.WithStack(err)
				}
			}
			formatOptions.In = f
		} else {
			return errors.WithStack(errors.Errorf("could not find a formatter of type %s", formatOptions.InputType))
		}
	} else {
		logrus.Debug("searching for a formatter to use")
		// else start searching for files:
		for n, f := range formatterMap {
			logrus.Debugf("checking %s formatter", n)
			if p, err := f.Search(); err == nil {
				logrus.Debugf("found file %s for %s formatter", p, n)
				formatOptions.In = f
				break
			}
		}
	}

	if formatOptions.In == nil {
		return errors.WithStack(errors.Errorf("could not find any viable formatter. available formatters: %s", strings.Join(formatterList, ", ")))
	}

	return formatOptions.Save()
}

func (f CoverageFormatter) Save() error {
	err := f.In.Parse()
	if err != nil {
		return errors.WithStack(err)
	}

	if f.writer == nil {
		if formatOptions.Output == "-" {
			f.writer = os.Stdout
		} else {
			err = os.MkdirAll(filepath.Dir(formatOptions.Output), 0755)
			if err != nil {
				return errors.WithStack(err)
			}
			f.writer, err = os.Create(formatOptions.Output)
			if err != nil {
				return errors.WithStack(err)
			}
		}
	}

	rep, err := f.In.Format()
	if err != nil {
		return errors.WithStack(err)
	}

	err = rep.Save(f.writer)
	if err != nil {
		return errors.WithStack(err)
	}
	return nil
}

func init() {
	pwd, _ := os.Getwd()
	formatCoverageCmd.Flags().StringVarP(&formatOptions.Prefix, "prefix", "p", pwd, "the root directory where the coverage analysis was performed")
	formatCoverageCmd.Flags().StringVarP(&formatOptions.Output, "output", "o", ccDefaultCoveragePath, "output path")
	formatCoverageCmd.Flags().StringVarP(&formatOptions.InputType, "input-type", "t", "", fmt.Sprintf("type of input source to use [%s]", strings.Join(formatterList, ", ")))
	RootCmd.AddCommand(formatCoverageCmd)
}
<commit_msg>Raise an error when invalid format-coverage path usage (#143)<commit_after>package cmd

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/codeclimate/test-reporter/formatters"
	"github.com/codeclimate/test-reporter/formatters/clover"
	"github.com/codeclimate/test-reporter/formatters/coveragepy"
	"github.com/codeclimate/test-reporter/formatters/gocov"
	"github.com/codeclimate/test-reporter/formatters/lcov"
	"github.com/codeclimate/test-reporter/formatters/simplecov"
	"github.com/gobuffalo/envy"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

type CoverageFormatter struct {
	CoveragePath string
	In           formatters.Formatter
	InputType    string
	Output       string
	Prefix       string
	writer       io.Writer
}

var formatOptions = CoverageFormatter{}

// a prioritized list of the formatters to use
var formatterList = []string{"simplecov", "lcov", "coverage.py", "clover", "gocov"}

// a map of the formatters to use
var formatterMap = map[string]formatters.Formatter{
	"simplecov":   &simplecov.Formatter{},
	"lcov":        &lcov.Formatter{},
	"coverage.py": &coveragepy.Formatter{},
	"gocov":       &gocov.Formatter{},
	"clover":      &clover.Formatter{},
}

// formatCoverageCmd represents the format command
var formatCoverageCmd = &cobra.Command{
	Use:   "format-coverage",
	Short: "Locate, parse, and re-format supported coverage sources.",
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) != 0 {
			logrus.Debugf("coverage path %s", args[0])
			formatOptions.CoveragePath = args[0]
			if formatOptions.InputType == "" {
				return errors.WithStack(errors.Errorf("please specify the format of the coverage file \"%s\" using the --input-type flag", formatOptions.CoveragePath))
			}
		}
		return runFormatter(formatOptions)
	},
}

func runFormatter(formatOptions CoverageFormatter) error {
	envy.Set("PREFIX", formatOptions.Prefix)

	// if a type is specified use that
	if formatOptions.InputType != "" {
		if f, ok := formatterMap[formatOptions.InputType]; ok {
			logrus.Debugf("using formatter %s", formatOptions.InputType)
			if formatOptions.CoveragePath != "" {
				_, err := f.Search(formatOptions.CoveragePath)
				if err != nil {
					logrus.Errorf("could not find coverage file %s\n%s", formatOptions.CoveragePath, err)
					return errors.WithStack(err)
				}
			}
			formatOptions.In = f
		} else {
			return errors.WithStack(errors.Errorf("could not find a formatter of type %s", formatOptions.InputType))
		}
	} else {
		logrus.Debug("searching for a formatter to use")
		// else start searching for files:
		for n, f := range formatterMap {
			logrus.Debugf("checking %s formatter", n)
			if p, err := f.Search(); err == nil {
				logrus.Debugf("found file %s for %s formatter", p, n)
				formatOptions.In = f
				break
			}
		}
	}

	if formatOptions.In == nil {
		return errors.WithStack(errors.Errorf("could not find any viable formatter. available formatters: %s", strings.Join(formatterList, ", ")))
	}

	return formatOptions.Save()
}

func (f CoverageFormatter) Save() error {
	err := f.In.Parse()
	if err != nil {
		return errors.WithStack(err)
	}

	if f.writer == nil {
		if formatOptions.Output == "-" {
			f.writer = os.Stdout
		} else {
			err = os.MkdirAll(filepath.Dir(formatOptions.Output), 0755)
			if err != nil {
				return errors.WithStack(err)
			}
			f.writer, err = os.Create(formatOptions.Output)
			if err != nil {
				return errors.WithStack(err)
			}
		}
	}

	rep, err := f.In.Format()
	if err != nil {
		return errors.WithStack(err)
	}

	err = rep.Save(f.writer)
	if err != nil {
		return errors.WithStack(err)
	}
	return nil
}

func init() {
	pwd, _ := os.Getwd()
	formatCoverageCmd.Flags().StringVarP(&formatOptions.Prefix, "prefix", "p", pwd, "the root directory where the coverage analysis was performed")
	formatCoverageCmd.Flags().StringVarP(&formatOptions.Output, "output", "o", ccDefaultCoveragePath, "output path")
	formatCoverageCmd.Flags().StringVarP(&formatOptions.InputType, "input-type", "t", "", fmt.Sprintf("type of input source to use [%s]", strings.Join(formatterList, ", ")))
	RootCmd.AddCommand(formatCoverageCmd)
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Frank Braun <frank@cryptogroup.net>
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

// mdserve serves Markdown files as HTML on localhost.
package main

import (
	"fmt"
	"io/ioutil"
	"os"

	"github.com/frankbraun/kitchensink/markup"
	"github.com/russross/blackfriday"
)

type markdownRenderer struct{}

func (r markdownRenderer) Render(filename string) ([]byte, error) {
	fmt.Println("rendering", filename)
	md, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	return blackfriday.MarkdownCommon(md), nil
}

func fatal(err error) {
	fmt.Fprintf(os.Stderr, "%s: error: %s\n", os.Args[0], err)
	os.Exit(1)
}

func usage() {
	fmt.Fprintf(os.Stderr, "usage: %s markdown_file\n", os.Args[0])
	os.Exit(1)
}

func main() {
	if len(os.Args) != 2 {
		usage()
	}
	err := markup.Serve(new(markdownRenderer), os.Args[1])
	if err != nil {
		fatal(err)
	}
}
<commit_msg>mdserve: use generic renderer<commit_after>// Copyright (c) 2017 Frank Braun <frank@cryptogroup.net>
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

// mdserve serves Markdown files as HTML on localhost.
package main

import (
	"fmt"
	"io/ioutil"
	"os"

	"github.com/frankbraun/kitchensink/markup"
	"github.com/russross/blackfriday"
)

const (
	commonHtmlFlags = 0 |
		blackfriday.HTML_USE_XHTML |
		blackfriday.HTML_USE_SMARTYPANTS |
		blackfriday.HTML_SMARTYPANTS_FRACTIONS |
		blackfriday.HTML_SMARTYPANTS_DASHES |
		blackfriday.HTML_SMARTYPANTS_LATEX_DASHES

	commonExtensions = 0 |
		blackfriday.EXTENSION_NO_INTRA_EMPHASIS |
		blackfriday.EXTENSION_TABLES |
		blackfriday.EXTENSION_FENCED_CODE |
		blackfriday.EXTENSION_AUTOLINK |
		blackfriday.EXTENSION_STRIKETHROUGH |
		blackfriday.EXTENSION_SPACE_HEADERS |
		blackfriday.EXTENSION_HEADER_IDS |
		blackfriday.EXTENSION_BACKSLASH_LINE_BREAK |
		blackfriday.EXTENSION_DEFINITION_LISTS
)

type markdownRenderer struct{}

func (r markdownRenderer) Render(filename string) ([]byte, error) {
	fmt.Println("rendering", filename)
	md, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	renderer := blackfriday.HtmlRenderer(commonHtmlFlags, "", "")
	options := blackfriday.Options{
		Extensions: commonExtensions,
	}
	return blackfriday.MarkdownOptions(md, renderer, options), nil
}

func fatal(err error) {
	fmt.Fprintf(os.Stderr, "%s: error: %s\n", os.Args[0], err)
	os.Exit(1)
}

func usage() {
	fmt.Fprintf(os.Stderr, "usage: %s markdown_file\n", os.Args[0])
	os.Exit(1)
}

func main() {
	if len(os.Args) != 2 {
		usage()
	}
	err := markup.Serve(new(markdownRenderer), os.Args[1])
	if err != nil {
		fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"regexp"

	"github.com/koron/gomigemo/migemo"
)

func dictdir() string {
	d := os.Getenv("GMIGEMO_DICTDIR")
	if d != "" {
		return d
	}
	d = os.Getenv("GOPATH")
	if d == "" {
		return "./_dict"
	}
	return filepath.Join(d, "src", "github.com", "koron", "gomigemo", "_dict")
}

var dictPath = flag.String("d", dictdir(),  "Location to dictionary")

func grep(r io.Reader, re *regexp.Regexp) error {
	buf := bufio.NewReader(r)
	for {
		b, _, err := buf.ReadLine()
		if err != nil {
			if err == io.EOF {
				return nil
			}
			return err
		}
		line := string(b)
		if re.MatchString(line) {
			fmt.Println(line)
		}
	}
	return nil
}

func main() {
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage of main: [pattern] [files...]")
	}

	flag.Parse()

	if flag.NArg() != 1 && flag.NArg() != 2 {
		flag.Usage()
		flag.PrintDefaults()
		os.Exit(1)
	}

	dict, err := migemo.Load(*dictPath)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	re, err := migemo.Compile(dict, flag.Arg(0))
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	if flag.NArg() == 1 {
		if err = grep(os.Stdin, re); err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
	} else {
		for _, arg := range flag.Args()[1:] {
			f, err := os.Open(arg)
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				os.Exit(1)
			}
			defer f.Close()
			if err = grep(f, re); err != nil {
				fmt.Fprintln(os.Stderr, err)
				os.Exit(1)
			}
		}
	}
}<commit_msg>Properly handle GOPATH containing multiple paths<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"regexp"

	"github.com/koron/gomigemo/migemo"
)

func dictdir() string {
	d := os.Getenv("GMIGEMO_DICTDIR")
	if d != "" {
		return d
	}
	d = os.Getenv("GOPATH")
	if d == "" {
		d = "."
	}
	for _, p := range filepath.SplitList(d) {
		candidate := filepath.Join(p, "src", "github.com", "koron", "gomigemo", "_dict")
		if _, err := os.Stat(candidate); err != nil {
			continue
		}
		return candidate
	}

	// fallback to current directory
	return d
}

var dictPath = flag.String("d", dictdir(), "Location to dictionary")

func grep(r io.Reader, re *regexp.Regexp) error {
	buf := bufio.NewReader(r)
	for {
		b, _, err := buf.ReadLine()
		if err != nil {
			if err == io.EOF {
				return nil
			}
			return err
		}
		line := string(b)
		if re.MatchString(line) {
			fmt.Println(line)
		}
	}
	return nil
}

func main() {
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage of main: [pattern] [files...]")
	}

	flag.Parse()

	if flag.NArg() != 1 && flag.NArg() != 2 {
		flag.Usage()
		flag.PrintDefaults()
		os.Exit(1)
	}

	dict, err := migemo.Load(*dictPath)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	re, err := migemo.Compile(dict, flag.Arg(0))
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	if flag.NArg() == 1 {
		if err = grep(os.Stdin, re); err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
	} else {
		for _, arg := range flag.Args()[1:] {
			f, err := os.Open(arg)
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				os.Exit(1)
			}
			defer f.Close()
			if err = grep(f, re); err != nil {
				fmt.Fprintln(os.Stderr, err)
				os.Exit(1)
			}
		}
	}
}<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"flag"
	"net"
	"os"
	"strconv"
	"time"

	"github.com/coredns/coredns/coremain"
	_ "github.com/coredns/coredns/plugin/bind"
	_ "github.com/coredns/coredns/plugin/cache"
	_ "github.com/coredns/coredns/plugin/errors"
	_ "github.com/coredns/coredns/plugin/forward"
	_ "github.com/coredns/coredns/plugin/health"
	_ "github.com/coredns/coredns/plugin/log"
	_ "github.com/coredns/coredns/plugin/loop"
	_ "github.com/coredns/coredns/plugin/metrics"
	clog "github.com/coredns/coredns/plugin/pkg/log"
	_ "github.com/coredns/coredns/plugin/reload"
	"github.com/mholt/caddy"
	"k8s.io/dns/pkg/netif"
	"k8s.io/kubernetes/pkg/util/dbus"
	utilexec "k8s.io/kubernetes/pkg/util/exec"
	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
)

// configParams lists the configuration options that can be provided to dns-cache
type configParams struct {
	localIP   string        // ip address for the local cache agent to listen for dns requests
	localPort string        // port to listen for dns requests
	intfName  string        // Name of the interface to be created
	interval  time.Duration // specifies how often to run iptables rules check
	exitChan  chan bool     // Channel to terminate background goroutines
}

type iptablesRule struct {
	table utiliptables.Table
	chain utiliptables.Chain
	args  []string
}

type cacheApp struct {
	iptables      utiliptables.Interface
	iptablesRules []iptablesRule
	params        configParams
	netifHandle   *netif.NetifManager
}

var cache = cacheApp{params: configParams{localPort: "53"}}

func (c *cacheApp) Init() {
	err := c.parseAndValidateFlags()
	if err != nil {
		clog.Fatalf("Error parsing flags - %s, Exiting", err)
	}
	c.netifHandle = netif.NewNetifManager(net.ParseIP(c.params.localIP))
	c.initIptables()
	err = c.teardownNetworking()
	if err != nil {
		// It is likely to hit errors here if previous shutdown cleaned up all iptables rules and interface.
		// Logging error at info level
		clog.Infof("Hit error during teardown - %s", err)
	}
	err = c.setupNetworking()
	if err != nil {
		cache.teardownNetworking()
		clog.Fatalf("Failed to setup - %s, Exiting", err)
	}
}

func init() {
	cache.Init()
	caddy.OnProcessExit = append(caddy.OnProcessExit, func() { cache.teardownNetworking() })
}

func (c *cacheApp) initIptables() {

	c.iptablesRules = []iptablesRule{
		// Match traffic destined for localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking
		{utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "tcp", "-d", c.params.localIP,
			"--dport", c.params.localPort, "-j", "NOTRACK"}},
		{utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "udp", "-d", c.params.localIP,
			"--dport", c.params.localPort, "-j", "NOTRACK"}},
		// There are rules in filter table to allow tracked connections to be accepted. Since we skipped connection tracking,
		// need these additional filter table rules.
		{utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "tcp", "-d", c.params.localIP,
			"--dport", c.params.localPort, "-j", "ACCEPT"}},
		{utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "udp", "-d", c.params.localIP,
			"--dport", c.params.localPort, "-j", "ACCEPT"}},
		// Match traffic from localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking
		{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", c.params.localIP,
			"--sport", c.params.localPort, "-j", "NOTRACK"}},
		{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-s", c.params.localIP,
			"--sport", c.params.localPort, "-j", "NOTRACK"}},
		// Additional filter table rules for traffic frpm localIp:localPort
		{utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "tcp", "-s", c.params.localIP,
			"--sport", c.params.localPort, "-j", "ACCEPT"}},
		{utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "udp", "-s", c.params.localIP,
			"--sport", c.params.localPort, "-j", "ACCEPT"}},
	}
	c.iptables = newIPTables()
}

func newIPTables() utiliptables.Interface {
	execer := utilexec.New()
	dbus := dbus.New()
	return utiliptables.New(execer, dbus, utiliptables.ProtocolIpv4)
}

func (c *cacheApp) setupNetworking() error {
	var err error
	clog.Infof("Setting up networking for node cache")
	err = c.netifHandle.AddDummyDevice(c.params.intfName)
	if err != nil {
		return err
	}
	for _, rule := range c.iptablesRules {
		_, err = c.iptables.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...)
		if err != nil {
			return err
		}
	}
	return err
}

func (c *cacheApp) teardownNetworking() error {
	clog.Infof("Tearing down")
	if c.params.exitChan != nil {
		// Stop the goroutine that periodically checks for iptables rules/dummy interface
		// exitChan is a buffered channel of size 1, so this will not block
		c.params.exitChan <- true
	}
	err := c.netifHandle.RemoveDummyDevice(c.params.intfName)
	for _, rule := range c.iptablesRules {
		exists := true
		for exists == true {
			c.iptables.DeleteRule(rule.table, rule.chain, rule.args...)
			exists, _ = c.iptables.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...)
		}
		// Delete the rule one last time since EnsureRule creates the rule if it doesn't exist
		c.iptables.DeleteRule(rule.table, rule.chain, rule.args...)
	}
	return err
}

func (c *cacheApp) parseAndValidateFlags() error {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "Runs coreDNS v1.2.5 as a nodelocal cache listening on the specified ip:port")
		flag.PrintDefaults()
	}

	flag.StringVar(&c.params.localIP, "localip", "", "ip address to bind dnscache to")
	flag.StringVar(&c.params.intfName, "intfname", "nodelocaldns", "name of the interface to be created")
	flag.DurationVar(&c.params.interval, "syncinterval", 60, "interval(in seconds) to check for iptables rules")
	flag.Parse()

	if net.ParseIP(c.params.localIP) == nil {
		return fmt.Errorf("Invalid localip specified - %q", c.params.localIP)
	}
	// lookup specified dns port
	if f := flag.Lookup("dns.port"); f == nil {
		return fmt.Errorf("Failed to lookup \"dns.port\" parameter")
	} else {
		c.params.localPort = f.Value.String()
	}
	if _, err := strconv.Atoi(c.params.localPort); err != nil {
		return fmt.Errorf("Invalid port specified - %q", c.params.localPort)
	}
	return nil
}

func (c *cacheApp) run() {
	c.params.exitChan = make(chan bool, 1)
	tick := time.NewTicker(c.params.interval * time.Second)
	for {
		select {
		case <-tick.C:
			for _, rule := range c.iptablesRules {
				exists, err := c.iptables.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...)
				if !exists {
					if err != nil {
						cache.teardownNetworking()
						clog.Fatalf("Failed to add back non-existent rule %v", rule)
					}
					clog.Infof("Added back nonexistent rule - %v", rule)
				}
				if err != nil {
					clog.Errorf("Failed to check rule %v - %s", rule, err)
				}
			}

			exists, err := c.netifHandle.EnsureDummyDevice(c.params.intfName)
			if !exists {
				if err != nil {
					cache.teardownNetworking()
					clog.Fatalf("Failed to add back non-existent interface %s", c.params.intfName)
				}
				clog.Infof("Added back nonexistent interface - %s", c.params.intfName)
			}
			if err != nil {
				clog.Errorf("Failed to check dummy device %s - %s", c.params.intfName, err)
			}
		case <-c.params.exitChan:
			clog.Warningf("Exiting iptables check goroutine")
			return
		}
	}
}

func main() {
	go cache.run()
	coremain.Run()
	// Unlikely to reach here, if we did it is because coremain exited and the signal was not trapped.
	clog.Errorf("Untrapped signal, tearing down")
	cache.teardownNetworking()
}
<commit_msg>Run checks before starting CoreDNS<commit_after>package main

import (
	"fmt"

	"flag"
	"net"
	"os"
	"strconv"
	"time"

	"github.com/coredns/coredns/coremain"
	_ "github.com/coredns/coredns/plugin/bind"
	_ "github.com/coredns/coredns/plugin/cache"
	_ "github.com/coredns/coredns/plugin/errors"
	_ "github.com/coredns/coredns/plugin/forward"
	_ "github.com/coredns/coredns/plugin/health"
	_ "github.com/coredns/coredns/plugin/log"
	_ "github.com/coredns/coredns/plugin/loop"
	_ "github.com/coredns/coredns/plugin/metrics"
	clog "github.com/coredns/coredns/plugin/pkg/log"
	_ "github.com/coredns/coredns/plugin/reload"
	"github.com/mholt/caddy"
	"k8s.io/dns/pkg/netif"
	"k8s.io/kubernetes/pkg/util/dbus"
	utilexec "k8s.io/kubernetes/pkg/util/exec"
	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
)

// configParams lists the configuration options that can be provided to dns-cache
type configParams struct {
	localIP   string        // ip address for the local cache agent to listen for dns requests
	localPort string        // port to listen for dns requests
	intfName  string        // Name of the interface to be created
	interval  time.Duration // specifies how often to run iptables rules check
	exitChan  chan bool     // Channel to terminate background goroutines
}

type iptablesRule struct {
	table utiliptables.Table
	chain utiliptables.Chain
	args  []string
}

type cacheApp struct {
	iptables      utiliptables.Interface
	iptablesRules []iptablesRule
	params        configParams
	netifHandle   *netif.NetifManager
}

var cache = cacheApp{params: configParams{localPort: "53"}}

func (c *cacheApp) Init() {
	err := c.parseAndValidateFlags()
	if err != nil {
		clog.Fatalf("Error parsing flags - %s, Exiting", err)
	}
	c.netifHandle = netif.NewNetifManager(net.ParseIP(c.params.localIP))
	c.initIptables()
	err = c.teardownNetworking()
	if err != nil {
		// It is likely to hit errors here if previous shutdown cleaned up all iptables rules and interface.
		// Logging error at info level
		clog.Infof("Hit error during teardown - %s", err)
	}
	err = c.setupNetworking()
	if err != nil {
		cache.teardownNetworking()
		clog.Fatalf("Failed to setup - %s, Exiting", err)
	}
}

func init() {
	cache.Init()
	caddy.OnProcessExit = append(caddy.OnProcessExit, func() { cache.teardownNetworking() })
}

func (c *cacheApp) initIptables() {

	c.iptablesRules = []iptablesRule{
		// Match traffic destined for localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking
		{utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "tcp", "-d", c.params.localIP,
			"--dport", c.params.localPort, "-j", "NOTRACK"}},
		{utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "udp", "-d", c.params.localIP,
			"--dport", c.params.localPort, "-j", "NOTRACK"}},
		// There are rules in filter table to allow tracked connections to be accepted. Since we skipped connection tracking,
		// need these additional filter table rules.
		{utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "tcp", "-d", c.params.localIP,
			"--dport", c.params.localPort, "-j", "ACCEPT"}},
		{utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "udp", "-d", c.params.localIP,
			"--dport", c.params.localPort, "-j", "ACCEPT"}},
		// Match traffic from localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking
		{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", c.params.localIP,
			"--sport", c.params.localPort, "-j", "NOTRACK"}},
		{utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-s", c.params.localIP,
			"--sport", c.params.localPort, "-j", "NOTRACK"}},
		// Additional filter table rules for traffic frpm localIp:localPort
		{utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "tcp", "-s", c.params.localIP,
			"--sport", c.params.localPort, "-j", "ACCEPT"}},
		{utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "udp", "-s", c.params.localIP,
			"--sport", c.params.localPort, "-j", "ACCEPT"}},
	}
	c.iptables = newIPTables()
}

func newIPTables() utiliptables.Interface {
	execer := utilexec.New()
	dbus := dbus.New()
	return utiliptables.New(execer, dbus, utiliptables.ProtocolIpv4)
}

func (c *cacheApp) setupNetworking() error {
	var err error
	clog.Infof("Setting up networking for node cache")
	err = c.netifHandle.AddDummyDevice(c.params.intfName)
	if err != nil {
		return err
	}
	for _, rule := range c.iptablesRules {
		_, err = c.iptables.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...)
		if err != nil {
			return err
		}
	}
	return err
}

func (c *cacheApp) teardownNetworking() error {
	clog.Infof("Tearing down")
	if c.params.exitChan != nil {
		// Stop the goroutine that periodically checks for iptables rules/dummy interface
		// exitChan is a buffered channel of size 1, so this will not block
		c.params.exitChan <- true
	}
	err := c.netifHandle.RemoveDummyDevice(c.params.intfName)
	for _, rule := range c.iptablesRules {
		exists := true
		for exists == true {
			c.iptables.DeleteRule(rule.table, rule.chain, rule.args...)
			exists, _ = c.iptables.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...)
		}
		// Delete the rule one last time since EnsureRule creates the rule if it doesn't exist
		c.iptables.DeleteRule(rule.table, rule.chain, rule.args...)
	}
	return err
}

func (c *cacheApp) parseAndValidateFlags() error {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "Runs coreDNS v1.2.5 as a nodelocal cache listening on the specified ip:port")
		flag.PrintDefaults()
	}

	flag.StringVar(&c.params.localIP, "localip", "", "ip address to bind dnscache to")
	flag.StringVar(&c.params.intfName, "intfname", "nodelocaldns", "name of the interface to be created")
	flag.DurationVar(&c.params.interval, "syncinterval", 60, "interval(in seconds) to check for iptables rules")
	flag.Parse()

	if net.ParseIP(c.params.localIP) == nil {
		return fmt.Errorf("Invalid localip specified - %q", c.params.localIP)
	}
	// lookup specified dns port
	if f := flag.Lookup("dns.port"); f == nil {
		return fmt.Errorf("Failed to lookup \"dns.port\" parameter")
	} else {
		c.params.localPort = f.Value.String()
	}
	if _, err := strconv.Atoi(c.params.localPort); err != nil {
		return fmt.Errorf("Invalid port specified - %q", c.params.localPort)
	}
	return nil
}

func (c *cacheApp) runChecks() {
	for _, rule := range c.iptablesRules {
		exists, err := c.iptables.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...)
		if !exists {
			if err != nil {
				cache.teardownNetworking()
				clog.Fatalf("Failed to add back non-existent rule %v", rule)
			}
			clog.Infof("Added back nonexistent rule - %v", rule)
		}
		if err != nil {
			clog.Errorf("Failed to check rule %v - %s", rule, err)
		}
	}

	exists, err := c.netifHandle.EnsureDummyDevice(c.params.intfName)
	if !exists {
		if err != nil {
			cache.teardownNetworking()
			clog.Fatalf("Failed to add back non-existent interface %s", c.params.intfName)
		}
		clog.Infof("Added back nonexistent interface - %s", c.params.intfName)
	}
	if err != nil {
		clog.Errorf("Failed to check dummy device %s - %s", c.params.intfName, err)
	}
}

func (c *cacheApp) run() {
	c.params.exitChan = make(chan bool, 1)
	tick := time.NewTicker(c.params.interval * time.Second)
	for {
		select {
		case <-tick.C:
			c.runChecks()
		case <-c.params.exitChan:
			clog.Warningf("Exiting iptables check goroutine")
			return
		}
	}
}

func main() {
	// Ensure that the required setup is ready
	// https://github.com/kubernetes/dns/issues/282 sometimes the interface gets the ip and then loses it, if added too soon.
	cache.runChecks()
	go cache.run()
	coremain.Run()
	// Unlikely to reach here, if we did it is because coremain exited and the signal was not trapped.
	clog.Errorf("Untrapped signal, tearing down")
	cache.teardownNetworking()
}
<|endoftext|>