<commit_before>// Package golog implements logging functions that log errors to stderr and
// debug messages to stdout. Trace logging is also supported.
// Trace logs go to stdout as well, but they are only written if the program
// is run with environment variable "TRACE=true".
// A stack dump will be printed after the message if "PRINT_STACK=true".
package golog

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"

	"github.com/getlantern/errors"
	"github.com/getlantern/ops"
	"github.com/oxtoacart/bpool"
)

const (
	// ERROR is an error Severity
	ERROR = 500

	// FATAL is an error Severity
	FATAL = 600
)

type outputFn func(prefix string, skipFrames int, printStack bool, severity string, arg interface{}, values map[string]interface{})

// Output is a log output that can optionally support structured logging
type Output interface {
	// Write debug messages
	Debug(prefix string, skipFrames int, printStack bool, severity string, arg interface{}, values map[string]interface{})

	// Write error messages
	Error(prefix string, skipFrames int, printStack bool, severity string, arg interface{}, values map[string]interface{})
}

var (
	output         Output
	outputMx       sync.RWMutex
	prepender      atomic.Value
	reporters      []ErrorReporter
	reportersMutex sync.RWMutex

	bufferPool = bpool.NewBufferPool(200)

	onFatal atomic.Value
)

// Severity is a level of error (higher values are more severe)
type Severity int

func (s Severity) String() string {
	switch s {
	case ERROR:
		return "ERROR"
	case FATAL:
		return "FATAL"
	default:
		return "UNKNOWN"
	}
}

func init() {
	DefaultOnFatal()
	ResetOutputs()
	ResetPrepender()
}

// SetPrepender sets a function to write something, e.g., the timestamp, before
// each line of the log.
func SetPrepender(p func(io.Writer)) {
	prepender.Store(p)
}

func ResetPrepender() {
	SetPrepender(func(io.Writer) {})
}

func GetPrepender() func(io.Writer) {
	return prepender.Load().(func(io.Writer))
}

// SetOutputs sets the outputs for error and debug logs to use the given Outputs.
// Returns a function that resets outputs to their original values prior to calling SetOutputs.
func SetOutputs(errorOut io.Writer, debugOut io.Writer) (reset func()) {
	return SetOutput(TextOutput(errorOut, debugOut))
}

// SetOutput sets the Output to use for errors and debug messages
func SetOutput(out Output) (reset func()) {
	outputMx.Lock()
	defer outputMx.Unlock()
	oldOut := output
	output = out
	return func() {
		outputMx.Lock()
		defer outputMx.Unlock()
		output = oldOut
	}
}

// Deprecated: instead of calling ResetOutputs, use the reset function returned by SetOutputs.
func ResetOutputs() {
	SetOutputs(os.Stderr, os.Stdout)
}

func getErrorOut() outputFn {
	outputMx.RLock()
	defer outputMx.RUnlock()
	return output.Error
}

func getDebugOut() outputFn {
	outputMx.RLock()
	defer outputMx.RUnlock()
	return output.Debug
}

// RegisterReporter registers the given ErrorReporter. All logged Errors are
// sent to this reporter.
func RegisterReporter(reporter ErrorReporter) {
	reportersMutex.Lock()
	reporters = append(reporters, reporter)
	reportersMutex.Unlock()
}

// OnFatal configures golog to call the given function on any FATAL error. By
// default, golog calls os.Exit(1) on any FATAL error.
func OnFatal(fn func(err error)) {
	onFatal.Store(fn)
}

// DefaultOnFatal enables the default behavior for OnFatal
func DefaultOnFatal() {
	onFatal.Store(func(err error) {
		os.Exit(1)
	})
}

// MultiLine is an interface for arguments that support multi-line output.
type MultiLine interface {
	// MultiLinePrinter returns a function that can be used to print the
	// multi-line output. The returned function writes one line to the buffer and
	// returns true if there are more lines to write. This function does not need
	// to take care of trailing carriage returns, golog handles that
	// automatically.
	MultiLinePrinter() func(buf *bytes.Buffer) bool
}

// ErrorReporter is a function to which the logger will report errors.
// It the given error and corresponding message along with associated ops
// context. This should return quickly as it executes on the critical code
// path. The recommended approach is to buffer as much as possible and discard
// new reports if the buffer becomes saturated.
type ErrorReporter func(err error, severity Severity, ctx map[string]interface{})

type Logger interface {
	// Debug logs to stdout
	Debug(arg interface{})
	// Debugf logs to stdout
	Debugf(message string, args ...interface{})

	// Error logs to stderr
	Error(arg interface{}) error
	// Errorf logs to stderr. It returns the first argument that's an error, or
	// a new error built using fmt.Errorf if none of the arguments are errors.
	Errorf(message string, args ...interface{}) error

	// Fatal logs to stderr and then exits with status 1
	Fatal(arg interface{})
	// Fatalf logs to stderr and then exits with status 1
	Fatalf(message string, args ...interface{})

	// Trace logs to stderr only if TRACE=true
	Trace(arg interface{})
	// Tracef logs to stderr only if TRACE=true
	Tracef(message string, args ...interface{})

	// TraceOut provides access to an io.Writer to which trace information can
	// be streamed. If running with environment variable "TRACE=true", TraceOut
	// will point to os.Stderr, otherwise it will point to a ioutil.Discared.
	// Each line of trace information will be prefixed with this Logger's
	// prefix.
	TraceOut() io.Writer

	// IsTraceEnabled() indicates whether or not tracing is enabled for this
	// logger.
	IsTraceEnabled() bool

	// AsStdLogger returns an standard logger
	AsStdLogger() *log.Logger
}

func LoggerFor(prefix string) Logger {
	l := &logger{
		prefix: prefix + ": ",
	}

	trace := os.Getenv("TRACE")
	l.traceOn, _ = strconv.ParseBool(trace)
	if !l.traceOn {
		prefixes := strings.Split(trace, ",")
		for _, p := range prefixes {
			if prefix == strings.Trim(p, " ") {
				l.traceOn = true
				break
			}
		}
	}
	if l.traceOn {
		l.traceOut = l.newTraceWriter()
	} else {
		l.traceOut = ioutil.Discard
	}

	printStack := os.Getenv("PRINT_STACK")
	l.printStack, _ = strconv.ParseBool(printStack)

	return l
}

type logger struct {
	prefix     string
	traceOn    bool
	traceOut   io.Writer
	printStack bool
	outs       atomic.Value
}

func (l *logger) print(write outputFn, skipFrames int, severity string, arg interface{}) {
	write(l.prefix, skipFrames+2, l.printStack, severity, arg, ops.AsMap(arg, false))
}

func (l *logger) printf(write outputFn, skipFrames int, severity string, message string, args ...interface{}) {
	l.print(write, skipFrames+1, severity, fmt.Sprintf(message, args...))
}

func (l *logger) Debug(arg interface{}) {
	l.print(getDebugOut(), 4, "DEBUG", arg)
}

func (l *logger) Debugf(message string, args ...interface{}) {
	l.printf(getDebugOut(), 4, "DEBUG", message, args...)
}

func (l *logger) Error(arg interface{}) error {
	return l.errorSkipFrames(arg, 1, ERROR)
}

func (l *logger) Errorf(message string, args ...interface{}) error {
	return l.errorSkipFrames(errors.NewOffset(1, message, args...), 1, ERROR)
}

func (l *logger) Fatal(arg interface{}) {
	fatal(l.errorSkipFrames(arg, 1, FATAL))
}

func (l *logger) Fatalf(message string, args ...interface{}) {
	fatal(l.errorSkipFrames(errors.NewOffset(1, message, args...), 1, FATAL))
}

func fatal(err error) {
	fn := onFatal.Load().(func(err error))
	fn(err)
}

func (l *logger) errorSkipFrames(arg interface{}, skipFrames int, severity Severity) error {
	var err error
	switch e := arg.(type) {
	case error:
		err = e
	default:
		err = fmt.Errorf("%v", e)
	}
	l.print(getErrorOut(), skipFrames+4, severity.String(), err)
	return report(err, severity)
}

func (l *logger) Trace(arg interface{}) {
	if l.traceOn {
		l.print(getDebugOut(), 4, "TRACE", arg)
	}
}

func (l *logger) Tracef(message string, args ...interface{}) {
	if l.traceOn {
		l.printf(getDebugOut(), 4, "TRACE", message, args...)
	}
}

func (l *logger) TraceOut() io.Writer {
	return l.traceOut
}

func (l *logger) IsTraceEnabled() bool {
	return l.traceOn
}

func (l *logger) newTraceWriter() io.Writer {
	pr, pw := io.Pipe()
	br := bufio.NewReader(pr)

	if !l.traceOn {
		return pw
	}
	go func() {
		defer func() {
			if err := pr.Close(); err != nil {
				errorOnLogging(err)
			}
		}()
		defer func() {
			if err := pw.Close(); err != nil {
				errorOnLogging(err)
			}
		}()

		for {
			line, err := br.ReadString('\n')
			if err == nil {
				// Log the line (minus the trailing newline)
				l.print(getDebugOut(), 6, "TRACE", line[:len(line)-1])
			} else {
				l.printf(getDebugOut(), 6, "TRACE", "TraceWriter closed due to unexpected error: %v", err)
				return
			}
		}
	}()

	return pw
}

type errorWriter struct {
	l *logger
}

// Write implements method of io.Writer, due to different call depth,
// it will not log correct file and line prefix
func (w *errorWriter) Write(p []byte) (n int, err error) {
	s := string(p)
	if s[len(s)-1] == '\n' {
		s = s[:len(s)-1]
	}
	w.l.print(getErrorOut(), 6, "ERROR", s)
	return len(p), nil
}

func (l *logger) AsStdLogger() *log.Logger {
	return log.New(&errorWriter{l}, "", 0)
}

func errorOnLogging(err error) {
	_, _ = fmt.Fprintf(os.Stderr, "Unable to log: %v\n", err)
}

func report(err error, severity Severity) error {
	var reportersCopy []ErrorReporter
	reportersMutex.RLock()
	if len(reporters) > 0 {
		reportersCopy = make([]ErrorReporter, len(reporters))
		copy(reportersCopy, reporters)
	}
	reportersMutex.RUnlock()

	if len(reportersCopy) > 0 {
		ctx := ops.AsMap(err, true)
		ctx["severity"] = severity.String()
		for _, reporter := range reportersCopy {
			// We include globals when reporting
			reporter(err, severity, ctx)
		}
	}
	return err
}

func writeStack(w io.Writer, pcs []uintptr) error {
	for _, pc := range pcs {
		funcForPc := runtime.FuncForPC(pc)
		if funcForPc == nil {
			break
		}
		name := funcForPc.Name()
		if strings.HasPrefix(name, "runtime.") {
			break
		}
		file, line := funcForPc.FileLine(pc)
		_, err := fmt.Fprintf(w, "\t%s\t%s: %d\n", name, file, line)
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>added PRINT_JSON env variable<commit_after>// Package golog implements logging functions that log errors to stderr and
// debug messages to stdout. Trace logging is also supported.
// Trace logs go to stdout as well, but they are only written if the program
// is run with environment variable "TRACE=true".
// A stack dump will be printed after the message if "PRINT_STACK=true".
package golog

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"

	"github.com/getlantern/errors"
	"github.com/getlantern/ops"
	"github.com/oxtoacart/bpool"
)

const (
	// ERROR is an error Severity
	ERROR = 500

	// FATAL is an error Severity
	FATAL = 600
)

type outputFn func(prefix string, skipFrames int, printStack bool, severity string, arg interface{}, values map[string]interface{})

// Output is a log output that can optionally support structured logging
type Output interface {
	// Write debug messages
	Debug(prefix string, skipFrames int, printStack bool, severity string, arg interface{}, values map[string]interface{})

	// Write error messages
	Error(prefix string, skipFrames int, printStack bool, severity string, arg interface{}, values map[string]interface{})
}

var (
	output         Output
	outputMx       sync.RWMutex
	prepender      atomic.Value
	reporters      []ErrorReporter
	reportersMutex sync.RWMutex

	bufferPool = bpool.NewBufferPool(200)

	onFatal atomic.Value
)

// Severity is a level of error (higher values are more severe)
type Severity int

func (s Severity) String() string {
	switch s {
	case ERROR:
		return "ERROR"
	case FATAL:
		return "FATAL"
	default:
		return "UNKNOWN"
	}
}

func init() {
	DefaultOnFatal()
	ResetOutputs()
	ResetPrepender()
}

// SetPrepender sets a function to write something, e.g., the timestamp, before
// each line of the log.
func SetPrepender(p func(io.Writer)) {
	prepender.Store(p)
}

func ResetPrepender() {
	SetPrepender(func(io.Writer) {})
}

func GetPrepender() func(io.Writer) {
	return prepender.Load().(func(io.Writer))
}

// SetOutputs sets the outputs for error and debug logs to use the given Outputs.
// Returns a function that resets outputs to their original values prior to calling SetOutputs.
// If env variable PRINT_JSON is set, use JSON output instead of plain text
func SetOutputs(errorOut io.Writer, debugOut io.Writer) (reset func()) {
	if printJson, _ := strconv.ParseBool(os.Getenv("PRINT_JSON")); printJson {
		return SetOutput(JsonOutput(errorOut, debugOut))
	}

	return SetOutput(TextOutput(errorOut, debugOut))
}

// SetOutput sets the Output to use for errors and debug messages
func SetOutput(out Output) (reset func()) {
	outputMx.Lock()
	defer outputMx.Unlock()
	oldOut := output
	output = out
	return func() {
		outputMx.Lock()
		defer outputMx.Unlock()
		output = oldOut
	}
}

// Deprecated: instead of calling ResetOutputs, use the reset function returned by SetOutputs.
func ResetOutputs() {
	SetOutputs(os.Stderr, os.Stdout)
}

func getErrorOut() outputFn {
	outputMx.RLock()
	defer outputMx.RUnlock()
	return output.Error
}

func getDebugOut() outputFn {
	outputMx.RLock()
	defer outputMx.RUnlock()
	return output.Debug
}

// RegisterReporter registers the given ErrorReporter. All logged Errors are
// sent to this reporter.
func RegisterReporter(reporter ErrorReporter) {
	reportersMutex.Lock()
	reporters = append(reporters, reporter)
	reportersMutex.Unlock()
}

// OnFatal configures golog to call the given function on any FATAL error. By
// default, golog calls os.Exit(1) on any FATAL error.
func OnFatal(fn func(err error)) {
	onFatal.Store(fn)
}

// DefaultOnFatal enables the default behavior for OnFatal
func DefaultOnFatal() {
	onFatal.Store(func(err error) {
		os.Exit(1)
	})
}

// MultiLine is an interface for arguments that support multi-line output.
type MultiLine interface {
	// MultiLinePrinter returns a function that can be used to print the
	// multi-line output. The returned function writes one line to the buffer and
	// returns true if there are more lines to write. This function does not need
	// to take care of trailing carriage returns, golog handles that
	// automatically.
	MultiLinePrinter() func(buf *bytes.Buffer) bool
}

// ErrorReporter is a function to which the logger will report errors.
// It the given error and corresponding message along with associated ops
// context. This should return quickly as it executes on the critical code
// path. The recommended approach is to buffer as much as possible and discard
// new reports if the buffer becomes saturated.
type ErrorReporter func(err error, severity Severity, ctx map[string]interface{})

type Logger interface {
	// Debug logs to stdout
	Debug(arg interface{})
	// Debugf logs to stdout
	Debugf(message string, args ...interface{})

	// Error logs to stderr
	Error(arg interface{}) error
	// Errorf logs to stderr. It returns the first argument that's an error, or
	// a new error built using fmt.Errorf if none of the arguments are errors.
	Errorf(message string, args ...interface{}) error

	// Fatal logs to stderr and then exits with status 1
	Fatal(arg interface{})
	// Fatalf logs to stderr and then exits with status 1
	Fatalf(message string, args ...interface{})

	// Trace logs to stderr only if TRACE=true
	Trace(arg interface{})
	// Tracef logs to stderr only if TRACE=true
	Tracef(message string, args ...interface{})

	// TraceOut provides access to an io.Writer to which trace information can
	// be streamed. If running with environment variable "TRACE=true", TraceOut
	// will point to os.Stderr, otherwise it will point to a ioutil.Discared.
	// Each line of trace information will be prefixed with this Logger's
	// prefix.
	TraceOut() io.Writer

	// IsTraceEnabled() indicates whether or not tracing is enabled for this
	// logger.
	IsTraceEnabled() bool

	// AsStdLogger returns an standard logger
	AsStdLogger() *log.Logger
}

func LoggerFor(prefix string) Logger {
	l := &logger{
		prefix: prefix + ": ",
	}

	trace := os.Getenv("TRACE")
	l.traceOn, _ = strconv.ParseBool(trace)
	if !l.traceOn {
		prefixes := strings.Split(trace, ",")
		for _, p := range prefixes {
			if prefix == strings.Trim(p, " ") {
				l.traceOn = true
				break
			}
		}
	}
	if l.traceOn {
		l.traceOut = l.newTraceWriter()
	} else {
		l.traceOut = ioutil.Discard
	}

	printStack := os.Getenv("PRINT_STACK")
	l.printStack, _ = strconv.ParseBool(printStack)

	return l
}

type logger struct {
	prefix     string
	traceOn    bool
	traceOut   io.Writer
	printStack bool
	outs       atomic.Value
}

func (l *logger) print(write outputFn, skipFrames int, severity string, arg interface{}) {
	write(l.prefix, skipFrames+2, l.printStack, severity, arg, ops.AsMap(arg, false))
}

func (l *logger) printf(write outputFn, skipFrames int, severity string, message string, args ...interface{}) {
	l.print(write, skipFrames+1, severity, fmt.Sprintf(message, args...))
}

func (l *logger) Debug(arg interface{}) {
	l.print(getDebugOut(), 4, "DEBUG", arg)
}

func (l *logger) Debugf(message string, args ...interface{}) {
	l.printf(getDebugOut(), 4, "DEBUG", message, args...)
}

func (l *logger) Error(arg interface{}) error {
	return l.errorSkipFrames(arg, 1, ERROR)
}

func (l *logger) Errorf(message string, args ...interface{}) error {
	return l.errorSkipFrames(errors.NewOffset(1, message, args...), 1, ERROR)
}

func (l *logger) Fatal(arg interface{}) {
	fatal(l.errorSkipFrames(arg, 1, FATAL))
}

func (l *logger) Fatalf(message string, args ...interface{}) {
	fatal(l.errorSkipFrames(errors.NewOffset(1, message, args...), 1, FATAL))
}

func fatal(err error) {
	fn := onFatal.Load().(func(err error))
	fn(err)
}

func (l *logger) errorSkipFrames(arg interface{}, skipFrames int, severity Severity) error {
	var err error
	switch e := arg.(type) {
	case error:
		err = e
	default:
		err = fmt.Errorf("%v", e)
	}
	l.print(getErrorOut(), skipFrames+4, severity.String(), err)
	return report(err, severity)
}

func (l *logger) Trace(arg interface{}) {
	if l.traceOn {
		l.print(getDebugOut(), 4, "TRACE", arg)
	}
}

func (l *logger) Tracef(message string, args ...interface{}) {
	if l.traceOn {
		l.printf(getDebugOut(), 4, "TRACE", message, args...)
	}
}

func (l *logger) TraceOut() io.Writer {
	return l.traceOut
}

func (l *logger) IsTraceEnabled() bool {
	return l.traceOn
}

func (l *logger) newTraceWriter() io.Writer {
	pr, pw := io.Pipe()
	br := bufio.NewReader(pr)

	if !l.traceOn {
		return pw
	}
	go func() {
		defer func() {
			if err := pr.Close(); err != nil {
				errorOnLogging(err)
			}
		}()
		defer func() {
			if err := pw.Close(); err != nil {
				errorOnLogging(err)
			}
		}()

		for {
			line, err := br.ReadString('\n')
			if err == nil {
				// Log the line (minus the trailing newline)
				l.print(getDebugOut(), 6, "TRACE", line[:len(line)-1])
			} else {
				l.printf(getDebugOut(), 6, "TRACE", "TraceWriter closed due to unexpected error: %v", err)
				return
			}
		}
	}()

	return pw
}

type errorWriter struct {
	l *logger
}

// Write implements method of io.Writer, due to different call depth,
// it will not log correct file and line prefix
func (w *errorWriter) Write(p []byte) (n int, err error) {
	s := string(p)
	if s[len(s)-1] == '\n' {
		s = s[:len(s)-1]
	}
	w.l.print(getErrorOut(), 6, "ERROR", s)
	return len(p), nil
}

func (l *logger) AsStdLogger() *log.Logger {
	return log.New(&errorWriter{l}, "", 0)
}

func errorOnLogging(err error) {
	_, _ = fmt.Fprintf(os.Stderr, "Unable to log: %v\n", err)
}

func report(err error, severity Severity) error {
	var reportersCopy []ErrorReporter
	reportersMutex.RLock()
	if len(reporters) > 0 {
		reportersCopy = make([]ErrorReporter, len(reporters))
		copy(reportersCopy, reporters)
	}
	reportersMutex.RUnlock()

	if len(reportersCopy) > 0 {
		ctx := ops.AsMap(err, true)
		ctx["severity"] = severity.String()
		for _, reporter := range reportersCopy {
			// We include globals when reporting
			reporter(err, severity, ctx)
		}
	}
	return err
}

func writeStack(w io.Writer, pcs []uintptr) error {
	for _, pc := range pcs {
		funcForPc := runtime.FuncForPC(pc)
		if funcForPc == nil {
			break
		}
		name := funcForPc.Name()
		if strings.HasPrefix(name, "runtime.") {
			break
		}
		file, line := funcForPc.FileLine(pc)
		_, err := fmt.Fprintf(w, "\t%s\t%s: %d\n", name, file, line)
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/thoj/go-ircevent"
	"io"
	"io/ioutil"
	"strings"
)

type IRCConfig struct {
	Nick     string
	Username string
	Server   string
	Channels []string
}

func addCallbacks(c *irc.Connection) {
	// fmt.Println(c)
}

func getConfig() (config IRCConfig, err error) {
	data, err := ioutil.ReadFile("goyal-config.json")
	if err != nil {
		return IRCConfig{}, fmt.Errorf("Could not read config file.\n")
	}

	decoder := json.NewDecoder(strings.NewReader(string(data)))
	if err := decoder.Decode(&config); err == io.EOF {
		// Return config ...
	} else if err != nil {
		return IRCConfig{}, fmt.Errorf("Could not read config file.\n")
	}

	return config, nil
}

func main() {
	config, err := getConfig()
	if err == nil {
		connection := irc.IRC(config.Nick, config.Username)
		connection.UseTLS = true

		addCallbacks(connection)

		//Connect to the server
		err := connection.Connect(config.Server)
		if err == nil {

			// Join the channels
			for _, channel := range config.Channels {
				fmt.Printf("Joining %s\n", channel)
				connection.Join(channel)
			}

		} else {
			fmt.Println(err)
		}

	} else {
		fmt.Println(err)
	}

}
<commit_msg>Add handlers for the most common types of messages.<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/thoj/go-ircevent"
	"io"
	"io/ioutil"
	"os"
	"strings"
	"time"
)

const (
	TIME_FORMAT   = "Jan 2 2006 15:04:05"
)

type IRCConfig struct {
	Nick        string
	Username    string
	Server      string
	Channels    []string
	LogFileDir  string
	LogFiles    map[string]*os.File
}

func main() {
	config, err := getConfig()
	if err != nil {
		fmt.Println(err)
		return
	}

	connection := irc.IRC(config.Nick, config.Username)
	connection.UseTLS = true

	config = setupLogFiles(config)
	defer closeLogFiles(config)
	addCallbacks(connection, config)

	//Connect to the server
	err = connection.Connect(config.Server)
	if err != nil {
		fmt.Println("Failed to connect.")
		return
	}

	// Connect and wait
	connection.Loop()
}

func addCallbacks(connection *irc.Connection, config IRCConfig) {

	// Join the channels
	connection.AddCallback("001", func(e *irc.Event) {
		for _, channel := range config.Channels {
			logMessage(config.LogFiles[channel], "Joining %s", channel)
			connection.Join(channel)
		}
	})

	connection.AddCallback("JOIN", func(e *irc.Event) {
		channel := e.Message()
		nick := e.Nick
		var message string
		if nick == config.Nick {
			message = fmt.Sprintf("Hello, I am %s - yet another logbot written in Go.", nick)
		} else {
			message = fmt.Sprintf("Hello %s, Welcome to %s!", nick, channel)
		}
		connection.Privmsg(channel, message)
		logMessage(config.LogFiles[channel], "%s entered %s", nick, channel)
	})

	connection.AddCallback("PRIVMSG", func(e *irc.Event) {
		channel := e.Arguments[0]
		nick := e.Nick
		switch channel {
		case config.Nick:
			connection.Privmsg(nick, "Sorry, I don't accept direct messages!")
		default:
			logMessage(config.LogFiles[channel], "%s: %s", nick, e.Message())
		}
	})

	connection.AddCallback("CTCP_ACTION", func(e *irc.Event) {
		channel := e.Arguments[0]
		nick := e.Nick
		logMessage(config.LogFiles[channel], "***%s %s", nick, e.Message())
	})

	connection.AddCallback("PART", func(e *irc.Event) {
		channel := e.Arguments[0]
		nick := e.Nick
		logMessage(config.LogFiles[channel], "%s left %s", nick, channel)
	})

	connection.AddCallback("QUIT", func(e *irc.Event) {
		channel := e.Arguments[0]
		nick := e.Nick
		logMessage(config.LogFiles[channel], "%s closed IRC", nick)
	})

}

func getConfig() (config IRCConfig, err error) {
	data, err := ioutil.ReadFile("goyal-config.json")
	if err != nil {
		return IRCConfig{}, fmt.Errorf("Could not read config file.\n")
	}

	decoder := json.NewDecoder(strings.NewReader(string(data)))
	if err := decoder.Decode(&config); err == io.EOF {
		// Return config ...
	} else if err != nil {
		return IRCConfig{}, fmt.Errorf("Could not read config file.\n")
	}

	return config, nil
}

func setupLogFiles(config IRCConfig) (IRCConfig) {
	config.LogFiles = make(map[string]*os.File)
	for _, channel := range config.Channels {
		/// FIXME: fix path manipulation
		logFileName := config.LogFileDir + "/" + channel
		logFile, err := os.OpenFile(logFileName, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
		if err != nil {
			fmt.Println(err)
			fmt.Println("Log file opening failed.")
		}
		config.LogFiles[channel] = logFile
	}
	return config
}

func closeLogFiles(config IRCConfig) {
	for _, logFile := range config.LogFiles {
		logFile.Close()
	}
}

func logMessage(logFile *os.File, format string, args ...interface{}) {
	if logFile == nil {
		panic(fmt.Sprintf("Writing is failing %+v\n", err))
	}
	now := time.Now().UTC().Format(TIME_FORMAT)
	message := fmt.Sprintf(fmt.Sprintf("<%s> %s\n", now, format), args...)
	_, err := logFile.WriteString(message)
	if err != nil {
		panic(fmt.Sprintf("Writing is failing %+v\n", err))
	}
}
<|endoftext|>


<commit_before>package goka

import (
	"errors"
	"fmt"
	"strings"
)

var (
	tableSuffix = "-v1"
	loopSuffix  = "-loop"
)

// SetTableSuffix changes `tableSuffix` which is a suffix for table topic.
// Use it to modify table's suffix to otherwise in case you cannot use the default suffix.
func SetTableSuffix(suffix string) {
	tableSuffix = suffix
}

// SetLoopSuffix changes `loopSuffix` which is a suffix for loop topic of group.
// Use it to modify loop topic's suffix to otherwise in case you cannot use the default suffix.
func SetLoopSuffix(suffix string) {
	loopSuffix = suffix
}

// Stream is the name of an event stream topic in Kafka, ie, a topic with
// cleanup.policy=delete
type Stream string

// Streams is a slice of Stream names.
type Streams []Stream

// Table is the name of a table topic in Kafka, ie, a topic with
// cleanup.policy=compact
type Table string

// Group is the name of a consumer group in Kafka and represents a processor
// group in Goka. A processor group may have a group table and a group loopback
// stream. By default, the group table is named <group>-table and the loopback
// stream <group>-loop.
type Group string

// GroupGraph is the specification of a processor group. It contains all input,
// output, and any other topic from which and into which the processor group
// may consume or produce events. Each of these links to Kafka is called Edge.
type GroupGraph struct {
	// the group marks multiple processor instances to be long together
	group string

	// the edges define the group graph
	inputTables   []Edge
	crossTables   []Edge
	inputStreams  []Edge
	outputStreams []Edge
	loopStream    []Edge
	groupTable    []Edge
	visitors      []Edge

	// those fields cache the info from above edges or are used to avoid naming/codec collisions
	codecs    map[string]Codec
	callbacks map[string]ProcessCallback

	outputStreamTopics map[Stream]struct{}

	joinCheck map[string]bool
}

// Group returns the group name.
func (gg *GroupGraph) Group() Group {
	return Group(gg.group)
}

// InputStreams returns all input stream edges of the group.
func (gg *GroupGraph) InputStreams() Edges {
	return gg.inputStreams
}

// JointTables retuns all joint table edges of the group.
func (gg *GroupGraph) JointTables() Edges {
	return gg.inputTables
}

// LookupTables retuns all lookup table edges  of the group.
func (gg *GroupGraph) LookupTables() Edges {
	return gg.crossTables
}

// LoopStream returns the loopback edge of the group.
func (gg *GroupGraph) LoopStream() Edge {
	// only 1 loop stream is valid
	if len(gg.loopStream) > 0 {
		return gg.loopStream[0]
	}
	return nil
}

// GroupTable returns the group table edge of the group.
func (gg *GroupGraph) GroupTable() Edge {
	// only 1 group table is valid
	if len(gg.groupTable) > 0 {
		return gg.groupTable[0]
	}
	return nil
}

// OutputStreams returns the output stream edges of the group.
func (gg *GroupGraph) OutputStreams() Edges {
	return gg.outputStreams
}

// AllEdges returns a list of all edges for the group graph.
// This allows to modify a graph by cloning it's edges into a new one.
//
//  var existing Graph
//  edges := existiting.AllEdges()
//  // modify edges as required
//  // recreate the modifiedg raph
//  newGraph := DefineGroup(existing.Groug(), edges...)
func (gg *GroupGraph) AllEdges() Edges {
	return chainEdges(
		gg.inputTables,
		gg.crossTables,
		gg.inputStreams,
		gg.outputStreams,
		gg.loopStream,
		gg.groupTable,
		gg.visitors)
}

// returns whether the passed topic is a valid group output topic
func (gg *GroupGraph) isOutputTopic(topic Stream) bool {
	_, ok := gg.outputStreamTopics[topic]
	return ok
}

// inputs returns all input topics (tables and streams)
func (gg *GroupGraph) inputs() Edges {
	return chainEdges(gg.inputStreams, gg.inputTables, gg.crossTables)
}

// copartitioned returns all copartitioned topics (joint tables and input streams)
func (gg *GroupGraph) copartitioned() Edges {
	return chainEdges(gg.inputStreams, gg.inputTables)
}

func (gg *GroupGraph) codec(topic string) Codec {
	return gg.codecs[topic]
}

func (gg *GroupGraph) callback(topic string) ProcessCallback {
	return gg.callbacks[topic]
}

func (gg *GroupGraph) joint(topic string) bool {
	return gg.joinCheck[topic]
}

// DefineGroup creates a group graph with a given group name and a list of
// edges.
func DefineGroup(group Group, edges ...Edge) *GroupGraph {
	gg := GroupGraph{group: string(group),
		codecs:             make(map[string]Codec),
		callbacks:          make(map[string]ProcessCallback),
		joinCheck:          make(map[string]bool),
		outputStreamTopics: make(map[Stream]struct{}),
	}

	for _, e := range edges {
		switch e := e.(type) {
		case inputStreams:
			for _, input := range e {
				gg.validateInputTopic(input.Topic())
				inputStr := input.(*inputStream)
				gg.codecs[input.Topic()] = input.Codec()
				gg.callbacks[input.Topic()] = inputStr.cb
				gg.inputStreams = append(gg.inputStreams, inputStr)
			}
		case *inputStream:
			gg.validateInputTopic(e.Topic())
			gg.codecs[e.Topic()] = e.Codec()
			gg.callbacks[e.Topic()] = e.cb
			gg.inputStreams = append(gg.inputStreams, e)
		case *loopStream:
			e.setGroup(group)
			gg.codecs[e.Topic()] = e.Codec()
			gg.callbacks[e.Topic()] = e.cb
			gg.loopStream = append(gg.loopStream, e)
		case *outputStream:
			gg.codecs[e.Topic()] = e.Codec()
			gg.outputStreams = append(gg.outputStreams, e)
			gg.outputStreamTopics[Stream(e.Topic())] = struct{}{}
		case *inputTable:
			gg.codecs[e.Topic()] = e.Codec()
			gg.inputTables = append(gg.inputTables, e)
			gg.joinCheck[e.Topic()] = true
		case *crossTable:
			gg.codecs[e.Topic()] = e.Codec()
			gg.crossTables = append(gg.crossTables, e)
		case *groupTable:
			e.setGroup(group)
			gg.codecs[e.Topic()] = e.Codec()
			gg.groupTable = append(gg.groupTable, e)
		case *visitor:
			gg.visitors = append(gg.visitors, e)
		}
	}

	return &gg
}

func (gg *GroupGraph) validateInputTopic(topic string) {
	if topic == "" {
		panic("Input topic cannot be empty. This will not work.")
	}

	if _, exists := gg.callbacks[topic]; exists {
		panic(fmt.Errorf("Callback for topic %s already exists. It is illegal to consume a topic twice", topic))
	}
}

// Validate validates the group graph and returns an error if invalid.
// Main validation checks are:
// - at most one loopback stream edge is allowed
// - at most one group table edge is allowed
// - at least one input stream is required
// - table and loopback topics cannot be used in any other edge.
func (gg *GroupGraph) Validate() error {
	if len(gg.loopStream) > 1 {
		return errors.New("more than one loop stream in group graph")
	}
	if len(gg.groupTable) > 1 {
		return errors.New("more than one group table in group graph")
	}
	if len(gg.inputStreams) == 0 {
		return errors.New("no input stream in group graph")
	}
	for _, t := range chainEdges(gg.outputStreams, gg.inputStreams, gg.inputTables, gg.crossTables) {
		if t.Topic() == loopName(gg.Group()) {
			return errors.New("should not directly use loop stream")
		}
		if t.Topic() == tableName(gg.Group()) {
			return errors.New("should not directly use group table")
		}
	}
	if len(gg.visitors) > 0 && len(gg.groupTable) == 0 {
		return fmt.Errorf("visitors cannot be used in a stateless processor")
	}
	return nil
}

// Edge represents a topic in Kafka and the corresponding codec to encode and
// decode the messages of that topic.
type Edge interface {
	String() string
	Topic() string
	Codec() Codec
}

// Edges is a slice of edge objects.
type Edges []Edge

// chainEdges chains edges together to avoid error-prone
// append(edges, moreEdges...) constructs in the graph
func chainEdges(edgeList ...Edges) Edges {
	var sum int
	for _, edges := range edgeList {
		sum += len(edges)
	}
	chained := make(Edges, 0, sum)

	for _, edges := range edgeList {
		chained = append(chained, edges...)
	}
	return chained
}

// Topics returns the names of the topics of the edges.
func (e Edges) Topics() []string {
	var t []string
	for _, i := range e {
		t = append(t, i.Topic())
	}
	return t
}

type topicDef struct {
	name  string
	codec Codec
}

func (t *topicDef) Topic() string {
	return t.name
}

func (t *topicDef) String() string {
	return fmt.Sprintf("%s/%T", t.name, t.codec)
}

func (t *topicDef) Codec() Codec {
	return t.codec
}

type inputStream struct {
	*topicDef
	cb ProcessCallback
}

// Input represents an edge of an input stream topic. The edge
// specifies the topic name, its codec and the ProcessorCallback used to
// process it. The topic has to be copartitioned with any other input stream of
// the group and with the group table.
// The group starts reading the topic from the newest offset.
func Input(topic Stream, c Codec, cb ProcessCallback) Edge {
	return &inputStream{&topicDef{string(topic), c}, cb}
}

type inputStreams Edges

func (is inputStreams) String() string {
	if is == nil {
		return "empty input streams"
	}

	return fmt.Sprintf("input streams: %s/%T", is.Topic(), is.Codec())
}

func (is inputStreams) Topic() string {
	if is == nil {
		return ""
	}
	var topics []string

	for _, stream := range is {
		topics = append(topics, stream.Topic())
	}
	return strings.Join(topics, ",")
}

func (is inputStreams) Codec() Codec {
	if is == nil {
		return nil
	}
	return is[0].Codec()
}

// Inputs creates edges of multiple input streams sharing the same
// codec and callback.
func Inputs(topics Streams, c Codec, cb ProcessCallback) Edge {
	if len(topics) == 0 {
		return nil
	}
	var edges Edges
	for _, topic := range topics {
		edges = append(edges, Input(topic, c, cb))
	}
	return inputStreams(edges)
}

type visitor struct {
	name string
	cb   ProcessCallback
}

func (m *visitor) Topic() string {
	return m.name
}
func (m *visitor) Codec() Codec {
	return nil
}
func (m *visitor) String() string {
	return fmt.Sprintf("visitor %s", m.name)
}

// Visitor adds a visitor edge to the processor. This allows to iterate over the whole processor state
// while running. Note that this can block rebalance or processor shutdown.
// EXPERIMENTAL! This feature is not fully tested and might trigger unknown bugs. Be careful!
func Visitor(name string, cb ProcessCallback) Edge {
	return &visitor{
		name: name,
		cb:   cb,
	}
}

type loopStream inputStream

// Loop represents the edge of the loopback topic of the group. The edge
// specifies the codec of the messages in the topic and ProcesCallback to
// process the messages of the topic. Context.Loopback() is used to write
// messages into this topic from any callback of the group.
func Loop(c Codec, cb ProcessCallback) Edge {
	return &loopStream{&topicDef{codec: c}, cb}
}

func (s *loopStream) setGroup(group Group) {
	s.topicDef.name = loopName(group)
}

type inputTable struct {
	*topicDef
}

// Join represents an edge of a copartitioned, log-compacted table topic. The
// edge specifies the topic name and the codec of the messages of the topic.
// The group starts reading the topic from the oldest offset.
// The processing of input streams is blocked until all partitions of the table
// are recovered.
func Join(topic Table, c Codec) Edge {
	return &inputTable{&topicDef{string(topic), c}}
}

type crossTable struct {
	*topicDef
}

// Lookup represents an edge of a non-copartitioned, log-compacted table
// topic. The edge specifies the topic name and the codec of the messages of
// the topic.  The group starts reading the topic from the oldest offset.
// The processing of input streams is blocked until the table is fully
// recovered.
func Lookup(topic Table, c Codec) Edge {
	return &crossTable{&topicDef{string(topic), c}}
}

type groupTable struct {
	*topicDef
}

// Persist represents the edge of the group table, which is log-compacted and
// copartitioned with the input streams.
// Without Persist, calls to ctx.Value or ctx.SetValue in the consume callback will
// fail and lead to shutdown of the processor.
//
// This edge specifies the codec of the
// messages in the topic, ie, the codec of the values of the table.
// The processing of input streams is blocked until all partitions of the group
// table are recovered.
//
// The topic name is derived from the group name by appending "-table".
func Persist(c Codec) Edge {
	return &groupTable{&topicDef{codec: c}}
}

func (t *groupTable) setGroup(group Group) {
	t.topicDef.name = string(GroupTable(group))
}

type outputStream struct {
	*topicDef
}

// Output represents an edge of an output stream topic. The edge
// specifies the topic name and the codec of the messages of the topic.
// Context.Emit() only emits messages into Output edges defined in the group
// graph.
// The topic does not have to be copartitioned with the input streams.
func Output(topic Stream, c Codec) Edge {
	return &outputStream{&topicDef{string(topic), c}}
}

// GroupTable returns the name of the group table of group.
func GroupTable(group Group) Table {
	return Table(tableName(group))
}

func tableName(group Group) string {
	return string(group) + tableSuffix
}

// loopName returns the name of the loop topic of group.
func loopName(group Group) string {
	return string(group) + loopSuffix
}

// StringsToStreams is a simple cast/conversion functions that allows to pass a slice
// of strings as a slice of Stream (Streams)
// Avoids the boilerplate loop over the string array that would be necessary otherwise.
func StringsToStreams(strings ...string) Streams {
	streams := make(Streams, 0, len(strings))

	for _, str := range strings {
		streams = append(streams, Stream(str))
	}
	return streams
}
<commit_msg>Revert "Test"<commit_after>package goka

import (
	"errors"
	"fmt"
	"strings"
)

var (
	tableSuffix = "-table"
	loopSuffix  = "-loop"
)

// SetTableSuffix changes `tableSuffix` which is a suffix for table topic.
// Use it to modify table's suffix to otherwise in case you cannot use the default suffix.
func SetTableSuffix(suffix string) {
	tableSuffix = suffix
}

// SetLoopSuffix changes `loopSuffix` which is a suffix for loop topic of group.
// Use it to modify loop topic's suffix to otherwise in case you cannot use the default suffix.
func SetLoopSuffix(suffix string) {
	loopSuffix = suffix
}

// Stream is the name of an event stream topic in Kafka, ie, a topic with
// cleanup.policy=delete
type Stream string

// Streams is a slice of Stream names.
type Streams []Stream

// Table is the name of a table topic in Kafka, ie, a topic with
// cleanup.policy=compact
type Table string

// Group is the name of a consumer group in Kafka and represents a processor
// group in Goka. A processor group may have a group table and a group loopback
// stream. By default, the group table is named <group>-table and the loopback
// stream <group>-loop.
type Group string

// GroupGraph is the specification of a processor group. It contains all input,
// output, and any other topic from which and into which the processor group
// may consume or produce events. Each of these links to Kafka is called Edge.
type GroupGraph struct {
	// the group marks multiple processor instances to be long together
	group string

	// the edges define the group graph
	inputTables   []Edge
	crossTables   []Edge
	inputStreams  []Edge
	outputStreams []Edge
	loopStream    []Edge
	groupTable    []Edge
	visitors      []Edge

	// those fields cache the info from above edges or are used to avoid naming/codec collisions
	codecs    map[string]Codec
	callbacks map[string]ProcessCallback

	outputStreamTopics map[Stream]struct{}

	joinCheck map[string]bool
}

// Group returns the group name.
func (gg *GroupGraph) Group() Group {
	return Group(gg.group)
}

// InputStreams returns all input stream edges of the group.
func (gg *GroupGraph) InputStreams() Edges {
	return gg.inputStreams
}

// JointTables retuns all joint table edges of the group.
func (gg *GroupGraph) JointTables() Edges {
	return gg.inputTables
}

// LookupTables retuns all lookup table edges  of the group.
func (gg *GroupGraph) LookupTables() Edges {
	return gg.crossTables
}

// LoopStream returns the loopback edge of the group.
func (gg *GroupGraph) LoopStream() Edge {
	// only 1 loop stream is valid
	if len(gg.loopStream) > 0 {
		return gg.loopStream[0]
	}
	return nil
}

// GroupTable returns the group table edge of the group.
func (gg *GroupGraph) GroupTable() Edge {
	// only 1 group table is valid
	if len(gg.groupTable) > 0 {
		return gg.groupTable[0]
	}
	return nil
}

// OutputStreams returns the output stream edges of the group.
func (gg *GroupGraph) OutputStreams() Edges {
	return gg.outputStreams
}

// AllEdges returns a list of all edges for the group graph.
// This allows to modify a graph by cloning it's edges into a new one.
//
//  var existing Graph
//  edges := existiting.AllEdges()
//  // modify edges as required
//  // recreate the modifiedg raph
//  newGraph := DefineGroup(existing.Groug(), edges...)
func (gg *GroupGraph) AllEdges() Edges {
	return chainEdges(
		gg.inputTables,
		gg.crossTables,
		gg.inputStreams,
		gg.outputStreams,
		gg.loopStream,
		gg.groupTable,
		gg.visitors)
}

// returns whether the passed topic is a valid group output topic
func (gg *GroupGraph) isOutputTopic(topic Stream) bool {
	_, ok := gg.outputStreamTopics[topic]
	return ok
}

// inputs returns all input topics (tables and streams)
func (gg *GroupGraph) inputs() Edges {
	return chainEdges(gg.inputStreams, gg.inputTables, gg.crossTables)
}

// copartitioned returns all copartitioned topics (joint tables and input streams)
func (gg *GroupGraph) copartitioned() Edges {
	return chainEdges(gg.inputStreams, gg.inputTables)
}

func (gg *GroupGraph) codec(topic string) Codec {
	return gg.codecs[topic]
}

func (gg *GroupGraph) callback(topic string) ProcessCallback {
	return gg.callbacks[topic]
}

func (gg *GroupGraph) joint(topic string) bool {
	return gg.joinCheck[topic]
}

// DefineGroup creates a group graph with a given group name and a list of
// edges.
func DefineGroup(group Group, edges ...Edge) *GroupGraph {
	gg := GroupGraph{group: string(group),
		codecs:             make(map[string]Codec),
		callbacks:          make(map[string]ProcessCallback),
		joinCheck:          make(map[string]bool),
		outputStreamTopics: make(map[Stream]struct{}),
	}

	for _, e := range edges {
		switch e := e.(type) {
		case inputStreams:
			for _, input := range e {
				gg.validateInputTopic(input.Topic())
				inputStr := input.(*inputStream)
				gg.codecs[input.Topic()] = input.Codec()
				gg.callbacks[input.Topic()] = inputStr.cb
				gg.inputStreams = append(gg.inputStreams, inputStr)
			}
		case *inputStream:
			gg.validateInputTopic(e.Topic())
			gg.codecs[e.Topic()] = e.Codec()
			gg.callbacks[e.Topic()] = e.cb
			gg.inputStreams = append(gg.inputStreams, e)
		case *loopStream:
			e.setGroup(group)
			gg.codecs[e.Topic()] = e.Codec()
			gg.callbacks[e.Topic()] = e.cb
			gg.loopStream = append(gg.loopStream, e)
		case *outputStream:
			gg.codecs[e.Topic()] = e.Codec()
			gg.outputStreams = append(gg.outputStreams, e)
			gg.outputStreamTopics[Stream(e.Topic())] = struct{}{}
		case *inputTable:
			gg.codecs[e.Topic()] = e.Codec()
			gg.inputTables = append(gg.inputTables, e)
			gg.joinCheck[e.Topic()] = true
		case *crossTable:
			gg.codecs[e.Topic()] = e.Codec()
			gg.crossTables = append(gg.crossTables, e)
		case *groupTable:
			e.setGroup(group)
			gg.codecs[e.Topic()] = e.Codec()
			gg.groupTable = append(gg.groupTable, e)
		case *visitor:
			gg.visitors = append(gg.visitors, e)
		}
	}

	return &gg
}

func (gg *GroupGraph) validateInputTopic(topic string) {
	if topic == "" {
		panic("Input topic cannot be empty. This will not work.")
	}

	if _, exists := gg.callbacks[topic]; exists {
		panic(fmt.Errorf("Callback for topic %s already exists. It is illegal to consume a topic twice", topic))
	}
}

// Validate validates the group graph and returns an error if invalid.
// Main validation checks are:
// - at most one loopback stream edge is allowed
// - at most one group table edge is allowed
// - at least one input stream is required
// - table and loopback topics cannot be used in any other edge.
func (gg *GroupGraph) Validate() error {
	if len(gg.loopStream) > 1 {
		return errors.New("more than one loop stream in group graph")
	}
	if len(gg.groupTable) > 1 {
		return errors.New("more than one group table in group graph")
	}
	if len(gg.inputStreams) == 0 {
		return errors.New("no input stream in group graph")
	}
	for _, t := range chainEdges(gg.outputStreams, gg.inputStreams, gg.inputTables, gg.crossTables) {
		if t.Topic() == loopName(gg.Group()) {
			return errors.New("should not directly use loop stream")
		}
		if t.Topic() == tableName(gg.Group()) {
			return errors.New("should not directly use group table")
		}
	}
	if len(gg.visitors) > 0 && len(gg.groupTable) == 0 {
		return fmt.Errorf("visitors cannot be used in a stateless processor")
	}
	return nil
}

// Edge represents a topic in Kafka and the corresponding codec to encode and
// decode the messages of that topic.
type Edge interface {
	String() string
	Topic() string
	Codec() Codec
}

// Edges is a slice of edge objects.
type Edges []Edge

// chainEdges chains edges together to avoid error-prone
// append(edges, moreEdges...) constructs in the graph
func chainEdges(edgeList ...Edges) Edges {
	var sum int
	for _, edges := range edgeList {
		sum += len(edges)
	}
	chained := make(Edges, 0, sum)

	for _, edges := range edgeList {
		chained = append(chained, edges...)
	}
	return chained
}

// Topics returns the names of the topics of the edges.
func (e Edges) Topics() []string {
	var t []string
	for _, i := range e {
		t = append(t, i.Topic())
	}
	return t
}

type topicDef struct {
	name  string
	codec Codec
}

func (t *topicDef) Topic() string {
	return t.name
}

func (t *topicDef) String() string {
	return fmt.Sprintf("%s/%T", t.name, t.codec)
}

func (t *topicDef) Codec() Codec {
	return t.codec
}

type inputStream struct {
	*topicDef
	cb ProcessCallback
}

// Input represents an edge of an input stream topic. The edge
// specifies the topic name, its codec and the ProcessorCallback used to
// process it. The topic has to be copartitioned with any other input stream of
// the group and with the group table.
// The group starts reading the topic from the newest offset.
func Input(topic Stream, c Codec, cb ProcessCallback) Edge {
	return &inputStream{&topicDef{string(topic), c}, cb}
}

type inputStreams Edges

func (is inputStreams) String() string {
	if is == nil {
		return "empty input streams"
	}

	return fmt.Sprintf("input streams: %s/%T", is.Topic(), is.Codec())
}

func (is inputStreams) Topic() string {
	if is == nil {
		return ""
	}
	var topics []string

	for _, stream := range is {
		topics = append(topics, stream.Topic())
	}
	return strings.Join(topics, ",")
}

func (is inputStreams) Codec() Codec {
	if is == nil {
		return nil
	}
	return is[0].Codec()
}

// Inputs creates edges of multiple input streams sharing the same
// codec and callback.
func Inputs(topics Streams, c Codec, cb ProcessCallback) Edge {
	if len(topics) == 0 {
		return nil
	}
	var edges Edges
	for _, topic := range topics {
		edges = append(edges, Input(topic, c, cb))
	}
	return inputStreams(edges)
}

type visitor struct {
	name string
	cb   ProcessCallback
}

func (m *visitor) Topic() string {
	return m.name
}
func (m *visitor) Codec() Codec {
	return nil
}
func (m *visitor) String() string {
	return fmt.Sprintf("visitor %s", m.name)
}

// Visitor adds a visitor edge to the processor. This allows to iterate over the whole processor state
// while running. Note that this can block rebalance or processor shutdown.
// EXPERIMENTAL! This feature is not fully tested and might trigger unknown bugs. Be careful!
func Visitor(name string, cb ProcessCallback) Edge {
	return &visitor{
		name: name,
		cb:   cb,
	}
}

type loopStream inputStream

// Loop represents the edge of the loopback topic of the group. The edge
// specifies the codec of the messages in the topic and ProcesCallback to
// process the messages of the topic. Context.Loopback() is used to write
// messages into this topic from any callback of the group.
func Loop(c Codec, cb ProcessCallback) Edge {
	return &loopStream{&topicDef{codec: c}, cb}
}

func (s *loopStream) setGroup(group Group) {
	s.topicDef.name = loopName(group)
}

type inputTable struct {
	*topicDef
}

// Join represents an edge of a copartitioned, log-compacted table topic. The
// edge specifies the topic name and the codec of the messages of the topic.
// The group starts reading the topic from the oldest offset.
// The processing of input streams is blocked until all partitions of the table
// are recovered.
func Join(topic Table, c Codec) Edge {
	return &inputTable{&topicDef{string(topic), c}}
}

type crossTable struct {
	*topicDef
}

// Lookup represents an edge of a non-copartitioned, log-compacted table
// topic. The edge specifies the topic name and the codec of the messages of
// the topic.  The group starts reading the topic from the oldest offset.
// The processing of input streams is blocked until the table is fully
// recovered.
func Lookup(topic Table, c Codec) Edge {
	return &crossTable{&topicDef{string(topic), c}}
}

type groupTable struct {
	*topicDef
}

// Persist represents the edge of the group table, which is log-compacted and
// copartitioned with the input streams.
// Without Persist, calls to ctx.Value or ctx.SetValue in the consume callback will
// fail and lead to shutdown of the processor.
//
// This edge specifies the codec of the
// messages in the topic, ie, the codec of the values of the table.
// The processing of input streams is blocked until all partitions of the group
// table are recovered.
//
// The topic name is derived from the group name by appending "-table".
func Persist(c Codec) Edge {
	return &groupTable{&topicDef{codec: c}}
}

func (t *groupTable) setGroup(group Group) {
	t.topicDef.name = string(GroupTable(group))
}

type outputStream struct {
	*topicDef
}

// Output represents an edge of an output stream topic. The edge
// specifies the topic name and the codec of the messages of the topic.
// Context.Emit() only emits messages into Output edges defined in the group
// graph.
// The topic does not have to be copartitioned with the input streams.
func Output(topic Stream, c Codec) Edge {
	return &outputStream{&topicDef{string(topic), c}}
}

// GroupTable returns the name of the group table of group.
func GroupTable(group Group) Table {
	return Table(tableName(group))
}

func tableName(group Group) string {
	return string(group) + tableSuffix
}

// loopName returns the name of the loop topic of group.
func loopName(group Group) string {
	return string(group) + loopSuffix
}

// StringsToStreams is a simple cast/conversion functions that allows to pass a slice
// of strings as a slice of Stream (Streams)
// Avoids the boilerplate loop over the string array that would be necessary otherwise.
func StringsToStreams(strings ...string) Streams {
	streams := make(Streams, 0, len(strings))

	for _, str := range strings {
		streams = append(streams, Stream(str))
	}
	return streams
}
<|endoftext|>


<commit_before>// Copyright 2017 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package framework

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/golang/glog"

	"istio.io/istio/tests/e2e/util"
)

const (
	yamlSuffix         = ".yaml"
	istioInstallDir    = "install/kubernetes"
	istioAddonsDir     = "install/kubernetes/addons"
	nonAuthInstallFile = "istio.yaml"
	authInstallFile    = "istio-auth.yaml"
	istioSystem        = "istio-system"
)

var (
	namespace    = flag.String("namespace", "", "Namespace to use for testing (empty to create/delete temporary one)")
	mixerHub     = flag.String("mixer_hub", "", "Mixer hub, if different from istio.Version")
	mixerTag     = flag.String("mixer_tag", "", "Mixer tag, if different from istio.Version")
	pilotHub     = flag.String("pilot_hub", "", "pilot hub, if different from istio.Version")
	pilotTag     = flag.String("pilot_tag", "", "pilot tag, if different from istio.Version")
	caHub        = flag.String("ca_hub", "", "Ca hub")
	caTag        = flag.String("ca_tag", "", "Ca tag")
	authEnable   = flag.Bool("auth_enable", false, "Enable auth")
	rbacfile     = flag.String("rbac_path", "", "Rbac yaml file")
	localCluster = flag.Bool("use_local_cluster", false, "Whether the cluster is local or not")

	addons = []string{
		"prometheus",
		"zipkin",
	}
)

// KubeInfo gathers information for kubectl
type KubeInfo struct {
	Namespace string

	TmpDir  string
	yamlDir string

	Ingress string

	localCluster     bool
	namespaceCreated bool

	// Istioctl installation
	Istioctl *Istioctl
	// App Manager
	AppManager *AppManager
}

// newKubeInfo create a new KubeInfo by given temp dir and runID
func newKubeInfo(tmpDir, runID string) (*KubeInfo, error) {
	if *namespace == "" {
		*namespace = runID
	}
	yamlDir := filepath.Join(tmpDir, "yaml")
	i, err := NewIstioctl(yamlDir, *namespace, *namespace, *pilotHub, *pilotTag)
	if err != nil {
		return nil, err
	}
	a := NewAppManager(tmpDir, *namespace, i)

	return &KubeInfo{
		Namespace:        *namespace,
		namespaceCreated: false,
		TmpDir:           tmpDir,
		yamlDir:          yamlDir,
		localCluster:     *localCluster,
		Istioctl:         i,
		AppManager:       a,
	}, nil
}

// Setup set up Kubernetes prerequest for tests
func (k *KubeInfo) Setup() error {
	glog.Info("Setting up kubeInfo")
	var err error
	if err = os.Mkdir(k.yamlDir, os.ModeDir|os.ModePerm); err != nil {
		return err
	}

	if err = util.CreateNamespace(k.Namespace); err != nil {
		glog.Error("Failed to create namespace.")
		return err
	}
	k.namespaceCreated = true

	if err = k.deployIstio(); err != nil {
		glog.Error("Failed to deploy Istio.")
		return err
	}

	if err = k.deployAddons(); err != nil {
		glog.Error("Failed to deploy istio addons")
		return err
	}

	var in string
	if k.localCluster {
		in, err = util.GetIngressPod(k.Namespace)
	} else {
		in, err = util.GetIngress(k.Namespace)
	}
	if err != nil {
		return err
	}
	k.Ingress = in
	return nil
}

// Teardown clean up everything created by setup
func (k *KubeInfo) Teardown() error {
	glog.Info("Cleaning up kubeInfo")
	var err error

	if *rbacfile != "" {

		testRbacYaml := filepath.Join(k.TmpDir, "yaml", filepath.Base(*rbacfile))
		if _, err = os.Stat(testRbacYaml); os.IsNotExist(err) {
			glog.Errorf("%s File does not exist", testRbacYaml)
		} else if err = util.KubeDelete(k.Namespace, testRbacYaml); err != nil {
			glog.Errorf("Rbac deletion failed, please remove stale ClusterRoleBindings")
		}
	}

	if k.namespaceCreated {
		if err = util.DeleteNamespace(k.Namespace); err != nil {
			glog.Errorf("Failed to delete namespace %s", k.Namespace)
			return err
		}

		// confirm the namespace is deleted as it will cause future creation to fail
		maxAttempts := 15
		namespaceDeleted := false
		totalWait := 0
		for attempts := 1; attempts <= maxAttempts; attempts++ {
			namespaceDeleted, err = util.NamespaceDeleted(k.Namespace)
			if namespaceDeleted {
				break
			}
			totalWait += attempts
			time.Sleep(time.Duration(attempts) * time.Second)
		}

		if !namespaceDeleted {
			glog.Errorf("Failed to delete namespace %s after %v seconds", k.Namespace, totalWait)
			return err
		}
		k.namespaceCreated = false
		glog.Infof("Namespace %s deletion status: %v", k.Namespace, namespaceDeleted)
	}
	return err
}

func (k *KubeInfo) deployAddons() error {
	for _, addon := range addons {

		baseYamlFile := util.GetResourcePath(filepath.Join(istioAddonsDir, fmt.Sprintf("%s.yaml", addon)))

		content, err := ioutil.ReadFile(baseYamlFile)
		if err != nil {
			glog.Errorf("Cannot read file %s", baseYamlFile)
			return err
		}

		content = replacePattern(k, content, istioSystem, k.Namespace)

		yamlFile := filepath.Join(k.TmpDir, "yaml", addon+".yaml")
		err = ioutil.WriteFile(yamlFile, content, 0600)
		if err != nil {
			glog.Errorf("Cannot write into file %s", yamlFile)
		}

		if err := util.KubeApply(k.Namespace, yamlFile); err != nil {
			glog.Errorf("Kubectl apply %s failed", yamlFile)
			return err
		}
	}
	return nil
}

func (k *KubeInfo) deployIstio() error {
	istioYaml := nonAuthInstallFile
	if *authEnable {
		istioYaml = authInstallFile
	}
	baseIstioYaml := util.GetResourcePath(filepath.Join(istioInstallDir, istioYaml))
	testIstioYaml := filepath.Join(k.TmpDir, "yaml", istioYaml)

	if *rbacfile != "" {
		baseRbacYaml := util.GetResourcePath(*rbacfile)
		testRbacYaml := filepath.Join(k.TmpDir, "yaml", filepath.Base(*rbacfile))
		if err := k.generateRbac(baseRbacYaml, testRbacYaml); err != nil {
			glog.Errorf("Generating rbac yaml failed")
		}
		if err := util.KubeApply(k.Namespace, testRbacYaml); err != nil {
			glog.Errorf("Rbac deployment failed")
			return err
		}
	}

	if err := k.generateIstio(baseIstioYaml, testIstioYaml); err != nil {
		glog.Errorf("Generating yaml %s failed", testIstioYaml)
		return err
	}
	if err := util.KubeApply(k.Namespace, testIstioYaml); err != nil {
		glog.Errorf("Istio core %s deployment failed", testIstioYaml)
		return err
	}

	return nil
}

func (k *KubeInfo) generateRbac(src, dst string) error {
	content, err := ioutil.ReadFile(src)
	if err != nil {
		glog.Errorf("Cannot read original yaml file %s", src)
		return err
	}

	content = replacePattern(k, content, istioSystem, k.Namespace)

	content = replacePattern(k, content, "namespace: default",
		"namespace: "+k.Namespace)

	content = replacePattern(k, content, "istio-pilot-admin-role-binding",
		"istio-pilot-admin-role-binding-"+k.Namespace)

	content = replacePattern(k, content, "istio-mixer-admin-role-binding",
		"istio-mixer-admin-role-binding-"+k.Namespace)

	content = replacePattern(k, content, "istio-ca-role-binding",
		"istio-ca-role-binding-"+k.Namespace)

	content = replacePattern(k, content, "istio-ingress-admin-role-binding",
		"istio-ingress-admin-role-binding-"+k.Namespace)

	content = replacePattern(k, content, "istio-egress-admin-role-binding",
		"istio-egress-admin-role-binding-"+k.Namespace)

	content = replacePattern(k, content, "istio-sidecar-role-binding",
		"istio-sidecar-role-binding-"+k.Namespace)

	err = ioutil.WriteFile(dst, content, 0600)
	if err != nil {
		glog.Errorf("Cannot write into generate rbac file %s", dst)
	}
	return err
}

func replacePattern(k *KubeInfo, content []byte, src, dest string) []byte {
	r := []byte(dest)
	p := regexp.MustCompile(src)
	content = p.ReplaceAllLiteral(content, r)
	return content
}

func (k *KubeInfo) generateIstio(src, dst string) error {
	content, err := ioutil.ReadFile(src)
	if err != nil {
		glog.Errorf("Cannot read original yaml file %s", src)
		return err
	}

	content = replacePattern(k, content, istioSystem, k.Namespace)

	// Replace long refresh delays with short ones for the sake of tests.
	content = replacePattern(k, content, "rdsRefreshDelay: 30s", "rdsRefreshDelay: 1s")
	content = replacePattern(k, content, "discoveryRefreshDelay: 30s", "discoveryRefreshDelay: 1s")
	content = replacePattern(k, content, "connectTimeout: 10s", "connectTimeout: 1s")
	content = replacePattern(k, content, "drainDuration: 45s", "drainDuration: 2s")
	content = replacePattern(k, content, "parentShutdownDuration: 1m0s", "parentShutdownDuration: 3s")

	if *mixerHub != "" && *mixerTag != "" {
		content = updateIstioYaml("mixer", *mixerHub, *mixerTag, content)
	}
	if *pilotHub != "" && *pilotTag != "" {
		content = updateIstioYaml("pilot", *pilotHub, *pilotTag, content)
		//Need to be updated when the string "proxy_debug" is changed
		content = updateIstioYaml("proxy_debug", *pilotHub, *pilotTag, content)
	}
	if *caHub != "" && *caTag != "" {
		//Need to be updated when the string "istio-ca" is changed
		content = updateIstioYaml("istio-ca", *caHub, *caTag, content)
	}
	if *localCluster {
		content = []byte(strings.Replace(string(content), "LoadBalancer", "NodePort", 1))
	}

	content = []byte(strings.Replace(string(content), "args: [\"discovery\", \"-v\", \"2\"]",
		"args: [\"discovery\", \"-v\", \"2\", \"-a\", \""+k.Namespace+"\"]", -1))

	err = ioutil.WriteFile(dst, content, 0600)
	if err != nil {
		glog.Errorf("Cannot write into generated yaml file %s", dst)
	}
	return err
}

func updateIstioYaml(module, hub, tag string, content []byte) []byte {
	image := []byte(fmt.Sprintf("image: %s/%s:%s", hub, module, tag))
	r := regexp.MustCompile(fmt.Sprintf("image: .*(\\/%s):.*", module))
	return r.ReplaceAllLiteral(content, image)
}
<commit_msg>Add flag to skip namespace and istio setup, so only the test is run. (#790)<commit_after>// Copyright 2017 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package framework

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/golang/glog"

	"istio.io/istio/tests/e2e/util"
)

const (
	yamlSuffix         = ".yaml"
	istioInstallDir    = "install/kubernetes"
	istioAddonsDir     = "install/kubernetes/addons"
	nonAuthInstallFile = "istio.yaml"
	authInstallFile    = "istio-auth.yaml"
	istioSystem        = "istio-system"
)

var (
	namespace    = flag.String("namespace", "", "Namespace to use for testing (empty to create/delete temporary one)")
	mixerHub     = flag.String("mixer_hub", "", "Mixer hub, if different from istio.Version")
	mixerTag     = flag.String("mixer_tag", "", "Mixer tag, if different from istio.Version")
	pilotHub     = flag.String("pilot_hub", "", "pilot hub, if different from istio.Version")
	pilotTag     = flag.String("pilot_tag", "", "pilot tag, if different from istio.Version")
	caHub        = flag.String("ca_hub", "", "Ca hub")
	caTag        = flag.String("ca_tag", "", "Ca tag")
	authEnable   = flag.Bool("auth_enable", false, "Enable auth")
	rbacfile     = flag.String("rbac_path", "", "Rbac yaml file")
	localCluster = flag.Bool("use_local_cluster", false, "Whether the cluster is local or not")
	skipSetup    = flag.Bool("skip_setup", false, "Skip namespace creation and istio cluster setup")

	addons = []string{
		"prometheus",
		"zipkin",
	}
)

// KubeInfo gathers information for kubectl
type KubeInfo struct {
	Namespace string

	TmpDir  string
	yamlDir string

	Ingress string

	localCluster     bool
	namespaceCreated bool

	// Istioctl installation
	Istioctl *Istioctl
	// App Manager
	AppManager *AppManager
}

// newKubeInfo create a new KubeInfo by given temp dir and runID
func newKubeInfo(tmpDir, runID string) (*KubeInfo, error) {
	if *namespace == "" {
		*namespace = runID
	}
	yamlDir := filepath.Join(tmpDir, "yaml")
	i, err := NewIstioctl(yamlDir, *namespace, *namespace, *pilotHub, *pilotTag)
	if err != nil {
		return nil, err
	}
	a := NewAppManager(tmpDir, *namespace, i)

	return &KubeInfo{
		Namespace:        *namespace,
		namespaceCreated: false,
		TmpDir:           tmpDir,
		yamlDir:          yamlDir,
		localCluster:     *localCluster,
		Istioctl:         i,
		AppManager:       a,
	}, nil
}

// Setup set up Kubernetes prerequest for tests
func (k *KubeInfo) Setup() error {
	glog.Info("Setting up kubeInfo")
	var err error
	if err = os.Mkdir(k.yamlDir, os.ModeDir|os.ModePerm); err != nil {
		return err
	}

	if !*skipSetup {
		if err = util.CreateNamespace(k.Namespace); err != nil {
			glog.Error("Failed to create namespace.")
			return err
		}
		k.namespaceCreated = true

		if err = k.deployIstio(); err != nil {
			glog.Error("Failed to deploy Istio.")
			return err
		}

		if err = k.deployAddons(); err != nil {
			glog.Error("Failed to deploy istio addons")
			return err
		}
	}

	var in string
	if k.localCluster {
		in, err = util.GetIngressPod(k.Namespace)
	} else {
		in, err = util.GetIngress(k.Namespace)
	}
	if err != nil {
		return err
	}
	k.Ingress = in
	return nil
}

// Teardown clean up everything created by setup
func (k *KubeInfo) Teardown() error {
	glog.Info("Cleaning up kubeInfo")
	var err error

	if *rbacfile != "" {

		testRbacYaml := filepath.Join(k.TmpDir, "yaml", filepath.Base(*rbacfile))
		if _, err = os.Stat(testRbacYaml); os.IsNotExist(err) {
			glog.Errorf("%s File does not exist", testRbacYaml)
		} else if err = util.KubeDelete(k.Namespace, testRbacYaml); err != nil {
			glog.Errorf("Rbac deletion failed, please remove stale ClusterRoleBindings")
		}
	}

	if k.namespaceCreated {
		if err = util.DeleteNamespace(k.Namespace); err != nil {
			glog.Errorf("Failed to delete namespace %s", k.Namespace)
			return err
		}

		// confirm the namespace is deleted as it will cause future creation to fail
		maxAttempts := 15
		namespaceDeleted := false
		totalWait := 0
		for attempts := 1; attempts <= maxAttempts; attempts++ {
			namespaceDeleted, err = util.NamespaceDeleted(k.Namespace)
			if namespaceDeleted {
				break
			}
			totalWait += attempts
			time.Sleep(time.Duration(attempts) * time.Second)
		}

		if !namespaceDeleted {
			glog.Errorf("Failed to delete namespace %s after %v seconds", k.Namespace, totalWait)
			return err
		}
		k.namespaceCreated = false
		glog.Infof("Namespace %s deletion status: %v", k.Namespace, namespaceDeleted)
	}
	return err
}

func (k *KubeInfo) deployAddons() error {
	for _, addon := range addons {

		baseYamlFile := util.GetResourcePath(filepath.Join(istioAddonsDir, fmt.Sprintf("%s.yaml", addon)))

		content, err := ioutil.ReadFile(baseYamlFile)
		if err != nil {
			glog.Errorf("Cannot read file %s", baseYamlFile)
			return err
		}

		content = replacePattern(k, content, istioSystem, k.Namespace)

		yamlFile := filepath.Join(k.TmpDir, "yaml", addon+".yaml")
		err = ioutil.WriteFile(yamlFile, content, 0600)
		if err != nil {
			glog.Errorf("Cannot write into file %s", yamlFile)
		}

		if err := util.KubeApply(k.Namespace, yamlFile); err != nil {
			glog.Errorf("Kubectl apply %s failed", yamlFile)
			return err
		}
	}
	return nil
}

func (k *KubeInfo) deployIstio() error {
	istioYaml := nonAuthInstallFile
	if *authEnable {
		istioYaml = authInstallFile
	}
	baseIstioYaml := util.GetResourcePath(filepath.Join(istioInstallDir, istioYaml))
	testIstioYaml := filepath.Join(k.TmpDir, "yaml", istioYaml)

	if *rbacfile != "" {
		baseRbacYaml := util.GetResourcePath(*rbacfile)
		testRbacYaml := filepath.Join(k.TmpDir, "yaml", filepath.Base(*rbacfile))
		if err := k.generateRbac(baseRbacYaml, testRbacYaml); err != nil {
			glog.Errorf("Generating rbac yaml failed")
		}
		if err := util.KubeApply(k.Namespace, testRbacYaml); err != nil {
			glog.Errorf("Rbac deployment failed")
			return err
		}
	}

	if err := k.generateIstio(baseIstioYaml, testIstioYaml); err != nil {
		glog.Errorf("Generating yaml %s failed", testIstioYaml)
		return err
	}
	if err := util.KubeApply(k.Namespace, testIstioYaml); err != nil {
		glog.Errorf("Istio core %s deployment failed", testIstioYaml)
		return err
	}

	return nil
}

func (k *KubeInfo) generateRbac(src, dst string) error {
	content, err := ioutil.ReadFile(src)
	if err != nil {
		glog.Errorf("Cannot read original yaml file %s", src)
		return err
	}

	content = replacePattern(k, content, istioSystem, k.Namespace)

	content = replacePattern(k, content, "namespace: default",
		"namespace: "+k.Namespace)

	content = replacePattern(k, content, "istio-pilot-admin-role-binding",
		"istio-pilot-admin-role-binding-"+k.Namespace)

	content = replacePattern(k, content, "istio-mixer-admin-role-binding",
		"istio-mixer-admin-role-binding-"+k.Namespace)

	content = replacePattern(k, content, "istio-ca-role-binding",
		"istio-ca-role-binding-"+k.Namespace)

	content = replacePattern(k, content, "istio-ingress-admin-role-binding",
		"istio-ingress-admin-role-binding-"+k.Namespace)

	content = replacePattern(k, content, "istio-egress-admin-role-binding",
		"istio-egress-admin-role-binding-"+k.Namespace)

	content = replacePattern(k, content, "istio-sidecar-role-binding",
		"istio-sidecar-role-binding-"+k.Namespace)

	err = ioutil.WriteFile(dst, content, 0600)
	if err != nil {
		glog.Errorf("Cannot write into generate rbac file %s", dst)
	}
	return err
}

func replacePattern(k *KubeInfo, content []byte, src, dest string) []byte {
	r := []byte(dest)
	p := regexp.MustCompile(src)
	content = p.ReplaceAllLiteral(content, r)
	return content
}

func (k *KubeInfo) generateIstio(src, dst string) error {
	content, err := ioutil.ReadFile(src)
	if err != nil {
		glog.Errorf("Cannot read original yaml file %s", src)
		return err
	}

	content = replacePattern(k, content, istioSystem, k.Namespace)

	// Replace long refresh delays with short ones for the sake of tests.
	content = replacePattern(k, content, "rdsRefreshDelay: 30s", "rdsRefreshDelay: 1s")
	content = replacePattern(k, content, "discoveryRefreshDelay: 30s", "discoveryRefreshDelay: 1s")
	content = replacePattern(k, content, "connectTimeout: 10s", "connectTimeout: 1s")
	content = replacePattern(k, content, "drainDuration: 45s", "drainDuration: 2s")
	content = replacePattern(k, content, "parentShutdownDuration: 1m0s", "parentShutdownDuration: 3s")

	if *mixerHub != "" && *mixerTag != "" {
		content = updateIstioYaml("mixer", *mixerHub, *mixerTag, content)
	}
	if *pilotHub != "" && *pilotTag != "" {
		content = updateIstioYaml("pilot", *pilotHub, *pilotTag, content)
		//Need to be updated when the string "proxy_debug" is changed
		content = updateIstioYaml("proxy_debug", *pilotHub, *pilotTag, content)
	}
	if *caHub != "" && *caTag != "" {
		//Need to be updated when the string "istio-ca" is changed
		content = updateIstioYaml("istio-ca", *caHub, *caTag, content)
	}
	if *localCluster {
		content = []byte(strings.Replace(string(content), "LoadBalancer", "NodePort", 1))
	}

	content = []byte(strings.Replace(string(content), "args: [\"discovery\", \"-v\", \"2\"]",
		"args: [\"discovery\", \"-v\", \"2\", \"-a\", \""+k.Namespace+"\"]", -1))

	err = ioutil.WriteFile(dst, content, 0600)
	if err != nil {
		glog.Errorf("Cannot write into generated yaml file %s", dst)
	}
	return err
}

func updateIstioYaml(module, hub, tag string, content []byte) []byte {
	image := []byte(fmt.Sprintf("image: %s/%s:%s", hub, module, tag))
	r := regexp.MustCompile(fmt.Sprintf("image: .*(\\/%s):.*", module))
	return r.ReplaceAllLiteral(content, image)
}
<|endoftext|>


<commit_before>package bluemix

import (
	"encoding/json"
	"fmt"
	"os"
)

type vcapServices struct {
	Services map[string][]service
}

type service struct {
	Name        string
	Label       string
	Plan        string
	Credentials map[string]interface{}
}

var services = vcapServices{}

func loadCredentials() {
	file, _ := os.Open("VCAP_SERVICES.json")
	decoder := json.NewDecoder(file)
	err := decoder.Decode(&services)
	if err != nil {
		fmt.Println("error:", err)
	}
}

func GetCredentials(serviceName string) map[string]interface{} {
	return GetCredentialsOnIndex(serviceName, 0)
}

func GetCredentialsOnIndex(serviceName string, index int) map[string]interface{} {
	if services.Services == nil {
		loadCredentials()
		fmt.Println("load cred")
	}
	return services.Services[serviceName][index].Credentials
}
<commit_msg>restructure<commit_after><|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package etcd

import (
	"k8s.io/kubernetes/federation/apis/federation"
	"k8s.io/kubernetes/federation/registry/cluster"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/rest"
	"k8s.io/kubernetes/pkg/registry/generic"
	"k8s.io/kubernetes/pkg/registry/generic/registry"
	"k8s.io/kubernetes/pkg/runtime"
)

type REST struct {
	*registry.Store
}

type StatusREST struct {
	store *registry.Store
}

func (r *StatusREST) New() runtime.Object {
	return &federation.Cluster{}
}

// Update alters the status subset of an object.
func (r *StatusREST) Update(ctx api.Context, name string, objInfo rest.UpdatedObjectInfo) (runtime.Object, bool, error) {
	return r.store.Update(ctx, name, objInfo)
}

// NewREST returns a RESTStorage object that will work against clusters.
func NewREST(opts generic.RESTOptions) (*REST, *StatusREST) {
	prefix := "/clusters"

	newListFunc := func() runtime.Object { return &federation.ClusterList{} }
	storageInterface := opts.Decorator(
		opts.Storage, 100, &federation.Cluster{}, prefix, cluster.Strategy, newListFunc)

	store := &registry.Store{
		NewFunc:     func() runtime.Object { return &federation.Cluster{} },
		NewListFunc: newListFunc,
		KeyRootFunc: func(ctx api.Context) string {
			return prefix
		},
		KeyFunc: func(ctx api.Context, name string) (string, error) {
			return registry.NoNamespaceKeyFunc(ctx, prefix, name)
		},
		ObjectNameFunc: func(obj runtime.Object) (string, error) {
			return obj.(*federation.Cluster).Name, nil
		},
		PredicateFunc:           cluster.MatchCluster,
		QualifiedResource:       federation.Resource("clusters"),
		DeleteCollectionWorkers: opts.DeleteCollectionWorkers,

		CreateStrategy: cluster.Strategy,
		UpdateStrategy: cluster.Strategy,
		DeleteStrategy: cluster.Strategy,

		ReturnDeletedObject: true,

		Storage: storageInterface,
	}

	statusStore := *store
	statusStore.UpdateStrategy = cluster.StatusStrategy

	return &REST{store}, &StatusREST{store: &statusStore}
}
<commit_msg>Extend Filter interface with Trigger() and use it for pods and nodes<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package etcd

import (
	"k8s.io/kubernetes/federation/apis/federation"
	"k8s.io/kubernetes/federation/registry/cluster"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/rest"
	"k8s.io/kubernetes/pkg/registry/generic"
	"k8s.io/kubernetes/pkg/registry/generic/registry"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/storage"
)

type REST struct {
	*registry.Store
}

type StatusREST struct {
	store *registry.Store
}

func (r *StatusREST) New() runtime.Object {
	return &federation.Cluster{}
}

// Update alters the status subset of an object.
func (r *StatusREST) Update(ctx api.Context, name string, objInfo rest.UpdatedObjectInfo) (runtime.Object, bool, error) {
	return r.store.Update(ctx, name, objInfo)
}

// NewREST returns a RESTStorage object that will work against clusters.
func NewREST(opts generic.RESTOptions) (*REST, *StatusREST) {
	prefix := "/clusters"

	newListFunc := func() runtime.Object { return &federation.ClusterList{} }
	storageInterface := opts.Decorator(
		opts.Storage,
		100,
		&federation.Cluster{},
		prefix,
		cluster.Strategy,
		newListFunc,
		storage.NoTriggerPublisher,
	)

	store := &registry.Store{
		NewFunc:     func() runtime.Object { return &federation.Cluster{} },
		NewListFunc: newListFunc,
		KeyRootFunc: func(ctx api.Context) string {
			return prefix
		},
		KeyFunc: func(ctx api.Context, name string) (string, error) {
			return registry.NoNamespaceKeyFunc(ctx, prefix, name)
		},
		ObjectNameFunc: func(obj runtime.Object) (string, error) {
			return obj.(*federation.Cluster).Name, nil
		},
		PredicateFunc:           cluster.MatchCluster,
		QualifiedResource:       federation.Resource("clusters"),
		DeleteCollectionWorkers: opts.DeleteCollectionWorkers,

		CreateStrategy: cluster.Strategy,
		UpdateStrategy: cluster.Strategy,
		DeleteStrategy: cluster.Strategy,

		ReturnDeletedObject: true,

		Storage: storageInterface,
	}

	statusStore := *store
	statusStore.UpdateStrategy = cluster.StatusStrategy

	return &REST{store}, &StatusREST{store: &statusStore}
}
<|endoftext|>


<commit_before>// +build e2e

// Copyright 2017 The nats-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package e2e

import (
	"context"
	"testing"
	"time"

	natsv1alpha2 "github.com/nats-io/nats-operator/pkg/apis/nats/v1alpha2"
)

// TestCreateCluster creates a NatsCluster resource and waits for the full mesh to be formed.
func TestCreateCluster(t *testing.T) {
	var (
		size    = 3
		version = "1.3.0"
	)

	var (
		natsCluster *natsv1alpha2.NatsCluster
		err         error
	)

	// Create a NatsCluster resource with three members.
	if natsCluster, err = f.CreateCluster(f.Namespace, "test-nats-", size, version); err != nil {
		t.Fatal(err)
	}
	// Make sure we cleanup the NatsCluster resource after we're done testing.
	defer func() {
		if err = f.DeleteCluster(natsCluster); err != nil {
			t.Error(err)
		}
	}()

	// Wait until the full mesh is formed.
	ctx, fn := context.WithTimeout(context.Background(), waitTimeout)
	defer fn()
	if err = f.WaitUntilFullMeshWithVersion(ctx, natsCluster, size, version); err != nil {
		t.Fatal(err)
	}
}

// TestPauseControl creates a NatsCluster resource and waits for the full mesh to be formed.
// Then, it pauses control of the NatsCluster resource and scales it up to five nodes, expecting the operation to NOT be performed.
// Finally, it resumes control of the NatsCluster resource and waits for the full five-node mesh to be formed.
func TestPauseControl(t *testing.T) {
	var (
		initialSize = 3
		finalSize   = 5
		version     = "1.3.0"
	)

	var (
		natsCluster *natsv1alpha2.NatsCluster
		err         error
	)

	// Create a NatsCluster resource with three members.
	if natsCluster, err = f.CreateCluster(f.Namespace, "test-nats-", initialSize, version); err != nil {
		t.Fatal(err)
	}
	// Make sure we cleanup the NatsCluster resource after we're done testing.
	defer func() {
		if err = f.DeleteCluster(natsCluster); err != nil {
			t.Error(err)
		}
	}()

	// Wait until the full mesh is formed.
	ctx1, fn := context.WithTimeout(context.Background(), waitTimeout)
	defer fn()
	if err = f.WaitUntilFullMeshWithVersion(ctx1, natsCluster, initialSize, version); err != nil {
		t.Fatal(err)
	}

	// Pause control of the cluster.
	natsCluster.Spec.Paused = true
	if natsCluster, err = f.PatchCluster(natsCluster); err != nil {
		t.Fatal(err)
	}

	// Scale the cluster up to five members
	natsCluster.Spec.Size = finalSize
	if natsCluster, err = f.PatchCluster(natsCluster); err != nil {
		t.Fatal(err)
	}
	// Make sure that the full mesh is NOT formed with the current size (5) within the timeout period.
	ctx2, fn := context.WithTimeout(context.Background(), waitTimeout)
	defer fn()
	if err = f.WaitUntilFullMeshWithVersion(ctx2, natsCluster, finalSize, version); err == nil {
		t.Fatalf("the full mesh has formed while control is paused")
	}

	// Resume control of the cluster.
	natsCluster.Spec.Paused = false
	if natsCluster, err = f.PatchCluster(natsCluster); err != nil {
		t.Fatal(err)
	}
	// Make sure that the full mesh is formed with the current size, since control has been resumed.
	ctx3, fn := context.WithTimeout(context.Background(), waitTimeout)
	defer fn()
	if err = f.WaitUntilFullMeshWithVersion(ctx3, natsCluster, finalSize, version); err != nil {
		t.Fatal(err)
	}
}

// TestCreateClusterWithHostPort creates a NatsCluster resource using
// a host port with no advertise for clients.
func TestCreateClusterWithHostPort(t *testing.T) {
	var (
		size    = 1
		version = "1.3.0"
	)

	var (
		natsCluster *natsv1alpha2.NatsCluster
		err         error
	)

	// Create a NatsCluster resource with three members.
	natsCluster, err = f.CreateCluster("test-nats-", size, version, func(natsCluster *natsv1alpha2.NatsCluster) {
		natsCluster.Spec.Pod = &natsv1alpha2.PodPolicy{
			EnableClientsHostPort: true,
		}
		natsCluster.Spec.NoAdvertise = true
	})
	if err != nil {
		t.Fatal(err)
	}

	// Make sure we cleanup the NatsCluster resource after we're done testing.
	defer func() {
		if err = f.DeleteCluster(natsCluster); err != nil {
			t.Error(err)
		}
	}()

	// List pods belonging to the NATS cluster, and make
	// sure that every route is listed in the
	// configuration.
	var attempts int
	for range time.NewTicker(1 * time.Second).C {
		if attempts >= 30 {
			t.Fatalf("Timed out waiting for pods with host port")
		}
		attempts++

		pods, err := f.PodsForNatsCluster(natsCluster)
		if err != nil {
			continue
		}
		if len(pods) == 0 {
			continue
		}

		var foundNoAdvertise bool
		pod := pods[0]
		container := pod.Spec.Containers[0]
		for _, v := range container.Command {
			if v == "--no_advertise" {
				foundNoAdvertise = true
			}
		}
		if !foundNoAdvertise {
			t.Error("Container not configured with no advertise")
		}

		var foundHostPort bool
		for _, port := range container.Ports {
			if port.ContainerPort == int32(4222) && port.HostPort == int32(4222) {
				foundHostPort = true
			}
		}
		if !foundHostPort {
			t.Error("Container not configured with host port")
		}

		break
	}
}
<commit_msg>Fix host port test<commit_after>// +build e2e

// Copyright 2017 The nats-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package e2e

import (
	"context"
	"testing"
	"time"

	natsv1alpha2 "github.com/nats-io/nats-operator/pkg/apis/nats/v1alpha2"
)

// TestCreateCluster creates a NatsCluster resource and waits for the full mesh to be formed.
func TestCreateCluster(t *testing.T) {
	var (
		size    = 3
		version = "1.3.0"
	)

	var (
		natsCluster *natsv1alpha2.NatsCluster
		err         error
	)

	// Create a NatsCluster resource with three members.
	if natsCluster, err = f.CreateCluster(f.Namespace, "test-nats-", size, version); err != nil {
		t.Fatal(err)
	}
	// Make sure we cleanup the NatsCluster resource after we're done testing.
	defer func() {
		if err = f.DeleteCluster(natsCluster); err != nil {
			t.Error(err)
		}
	}()

	// Wait until the full mesh is formed.
	ctx, fn := context.WithTimeout(context.Background(), waitTimeout)
	defer fn()
	if err = f.WaitUntilFullMeshWithVersion(ctx, natsCluster, size, version); err != nil {
		t.Fatal(err)
	}
}

// TestPauseControl creates a NatsCluster resource and waits for the full mesh to be formed.
// Then, it pauses control of the NatsCluster resource and scales it up to five nodes, expecting the operation to NOT be performed.
// Finally, it resumes control of the NatsCluster resource and waits for the full five-node mesh to be formed.
func TestPauseControl(t *testing.T) {
	var (
		initialSize = 3
		finalSize   = 5
		version     = "1.3.0"
	)

	var (
		natsCluster *natsv1alpha2.NatsCluster
		err         error
	)

	// Create a NatsCluster resource with three members.
	if natsCluster, err = f.CreateCluster(f.Namespace, "test-nats-", initialSize, version); err != nil {
		t.Fatal(err)
	}
	// Make sure we cleanup the NatsCluster resource after we're done testing.
	defer func() {
		if err = f.DeleteCluster(natsCluster); err != nil {
			t.Error(err)
		}
	}()

	// Wait until the full mesh is formed.
	ctx1, fn := context.WithTimeout(context.Background(), waitTimeout)
	defer fn()
	if err = f.WaitUntilFullMeshWithVersion(ctx1, natsCluster, initialSize, version); err != nil {
		t.Fatal(err)
	}

	// Pause control of the cluster.
	natsCluster.Spec.Paused = true
	if natsCluster, err = f.PatchCluster(natsCluster); err != nil {
		t.Fatal(err)
	}

	// Scale the cluster up to five members
	natsCluster.Spec.Size = finalSize
	if natsCluster, err = f.PatchCluster(natsCluster); err != nil {
		t.Fatal(err)
	}
	// Make sure that the full mesh is NOT formed with the current size (5) within the timeout period.
	ctx2, fn := context.WithTimeout(context.Background(), waitTimeout)
	defer fn()
	if err = f.WaitUntilFullMeshWithVersion(ctx2, natsCluster, finalSize, version); err == nil {
		t.Fatalf("the full mesh has formed while control is paused")
	}

	// Resume control of the cluster.
	natsCluster.Spec.Paused = false
	if natsCluster, err = f.PatchCluster(natsCluster); err != nil {
		t.Fatal(err)
	}
	// Make sure that the full mesh is formed with the current size, since control has been resumed.
	ctx3, fn := context.WithTimeout(context.Background(), waitTimeout)
	defer fn()
	if err = f.WaitUntilFullMeshWithVersion(ctx3, natsCluster, finalSize, version); err != nil {
		t.Fatal(err)
	}
}

// TestCreateClusterWithHostPort creates a NatsCluster resource using
// a host port with no advertise for clients.
func TestCreateClusterWithHostPort(t *testing.T) {
	var (
		size    = 1
		version = "1.3.0"
	)

	var (
		natsCluster *natsv1alpha2.NatsCluster
		err         error
	)

	// Create a NatsCluster resource with three members.
	natsCluster, err = f.CreateCluster(f.Namespace, "test-nats-", size, version, func(natsCluster *natsv1alpha2.NatsCluster) {
		natsCluster.Spec.Pod = &natsv1alpha2.PodPolicy{
			EnableClientsHostPort: true,
		}
		natsCluster.Spec.NoAdvertise = true
	})
	if err != nil {
		t.Fatal(err)
	}

	// Make sure we cleanup the NatsCluster resource after we're done testing.
	defer func() {
		if err = f.DeleteCluster(natsCluster); err != nil {
			t.Error(err)
		}
	}()

	// List pods belonging to the NATS cluster, and make
	// sure that every route is listed in the
	// configuration.
	var attempts int
	for range time.NewTicker(1 * time.Second).C {
		if attempts >= 30 {
			t.Fatalf("Timed out waiting for pods with host port")
		}
		attempts++

		pods, err := f.PodsForNatsCluster(natsCluster)
		if err != nil {
			continue
		}
		if len(pods) == 0 {
			continue
		}

		var foundNoAdvertise bool
		pod := pods[0]
		container := pod.Spec.Containers[0]
		for _, v := range container.Command {
			if v == "--no_advertise" {
				foundNoAdvertise = true
			}
		}
		if !foundNoAdvertise {
			t.Error("Container not configured with no advertise")
		}

		var foundHostPort bool
		for _, port := range container.Ports {
			if port.ContainerPort == int32(4222) && port.HostPort == int32(4222) {
				foundHostPort = true
			}
		}
		if !foundHostPort {
			t.Error("Container not configured with host port")
		}

		break
	}
}
<|endoftext|>