<commit_before>package igb

import (
	"fmt"
	parserErr "github.com/goby-lang/goby/compiler/parser/errors"
	"io"
	"log"
	"math/rand"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/chzyer/readline"
	"github.com/goby-lang/goby/compiler/bytecode"
	"github.com/goby-lang/goby/compiler/lexer"
	"github.com/goby-lang/goby/compiler/parser"
	"github.com/goby-lang/goby/vm"
	"github.com/looplab/fsm"
	"github.com/mattn/go-colorable"
)

const (
	prmpt1    = "»"
	prmpt2    = "¤"
	prompt1   = "\033[32m" + prmpt1 + "\033[0m "
	prompt2   = "\033[31m" + prmpt2 + "\033[0m "
	pad       = "  "
	echo      = "\033[33m#»\033[0m"
	interrupt = "^C"
	semicolon = ";"
	help      = "help"
	reset     = "reset"

	readyToExec = "readyToExec"
	Waiting     = "waiting"
	waitEnded   = "waitEnded"
	waitExited  = "waitExited"

	emojis = "😀😁😂🤣😃😄😅😆😉😊😋😎😍😘😗😙😚🙂🤗🤔😐😑😶🙄😏😮😪😴😌😛😜😝🤤🙃🤑😲😭😳🤧😇🤠🤡🤥🤓😈👿👹👺💀👻👽🤖💩😺😸😹😻😼😽"
)

// iGb holds internal states of iGb.
type iGb struct {
	sm        *fsm.FSM
	rl        *readline.Instance
	completer *readline.PrefixCompleter
	lines     string
	cmds      []string
	indents   int
}

// iVM holds VM only for iGb.
type iVM struct {
	v *vm.VM
	p *parser.Parser
	g *bytecode.Generator
}

var out io.Writer

func init() {
	if runtime.GOOS == "windows" {
		out = colorable.NewColorableStderr()
	} else {
		out = os.Stderr
	}
}

func println(s ...string) {
	out.Write([]byte(strings.Join(s, " ") + "\n"))
}

// StartIgb starts goby's REPL.
func StartIgb(version string) {
reset:
	var err error
	igb := newIgb()

	igb.rl, err = readline.NewEx(&readline.Config{
		Prompt:              prompt1,
		HistoryFile:         filepath.Join(os.TempDir(), "readline_goby.tmp"),
		AutoComplete:        igb.completer,
		InterruptPrompt:     interrupt,
		HistorySearchFold:   true,
		FuncFilterInputRune: filterInput,
	})
	defer igb.rl.Close()

	if err != nil {
		fmt.Printf("iGb error: %s", err)
		return
	}

	log.SetOutput(igb.rl.Stderr())

	println("Goby", version, fortune(), fortune(), fortune())

	ivm, err := newIVM()

	if err != nil {
		fmt.Println(err.Error())
		return
	}

	for {
		igb, err = readIgb(igb, err)

		// Interruption handling
		if err != nil {
			switch {
			case err == io.EOF:
				println(igb.lines + "")
				return
			case err == readline.ErrInterrupt: // Pressing Ctrl-C
				if len(igb.lines) == 0 {
					if igb.cmds == nil {
						println("")
						println("Bye!")
						return
					}
				}
				// Erasing command buffer
				igb.indents = 0
				igb.rl.SetPrompt(prompt1)
				igb.sm.Event(waitExited)
				igb.cmds = nil
				println(" -- block cleared")
				continue
			}
		}

		// Command handling
		switch {
		case strings.HasPrefix(igb.lines, "#"):
			println(prompt(igb.indents) + indent(igb.indents) + igb.lines)
			continue
		case igb.lines == help:
			println(prompt(igb.indents) + igb.lines)
			usage(igb.rl.Stderr(), igb.completer)
			continue
		case igb.lines == reset:
			igb.rl = nil
			igb.cmds = nil
			println(prompt(igb.indents) + igb.lines)
			println("Restarting iGb...")
			goto reset
		case igb.lines == "":
			println(prompt(igb.indents) + indent(igb.indents) + igb.lines)
			continue
		}

		ivm.p.Lexer = lexer.New(igb.lines)
		program, perr := ivm.p.ParseProgram()

		// Parse error handling
		if perr != nil {
			switch {
			case perr.IsEOF():
				if !igb.sm.Is(Waiting) {
					igb.sm.Event(Waiting)
				}
				println(prompt(igb.indents) + indent(igb.indents) + igb.lines)
				igb.indents++
				igb.rl.SetPrompt(prompt(igb.indents) + indent(igb.indents))
				igb.cmds = append(igb.cmds, igb.lines)
				continue
			case perr.IsUnexpectedEnd() && len(igb.cmds) == 0:
				// If igb.cmds is empty, it means that user just typed 'end' without corresponding statement/expression
				println(prompt(igb.indents) + indent(igb.indents) + igb.lines)
				igb.indents = 0
				igb.rl.SetPrompt(prompt1)
				fmt.Println(perr.Message)
				igb.cmds = nil
				continue
			case perr.IsUnexpectedEnd():
				if igb.indents > 1 {
					igb.indents--
					println(prompt(igb.indents) + indent(igb.indents) + igb.lines)
					igb.sm.Event(Waiting)
					igb.rl.SetPrompt(prompt(igb.indents) + indent(igb.indents))
					igb.cmds = append(igb.cmds, igb.lines)
					continue
				}
				igb.indents = 0
				igb.sm.Event(waitEnded)
				igb.rl.SetPrompt(prompt(igb.indents) + indent(igb.indents))
				igb.cmds = append(igb.cmds, igb.lines)
			}
		}

		if igb.sm.Is(Waiting) && igb.indents > 0 {
			println(prompt(igb.indents) + indent(igb.indents) + igb.lines)
			igb.rl.SetPrompt(prompt(igb.indents) + indent(igb.indents))
			igb.cmds = append(igb.cmds, igb.lines)
			continue
		}

		if igb.sm.Is(waitEnded) {
			ivm.p.Lexer = lexer.New(string(strings.Join(igb.cmds, "\n")))

			// Test if current input can be properly parsed.
			program, perr = ivm.p.ParseProgram()

			if perr != nil {
				handleParserError(perr, igb)
				continue
			}

			// If everything goes well, reset state and statements buffer
			igb.rl.SetPrompt(prompt(igb.indents))
			igb.sm.Event(readyToExec)
		}

		if igb.sm.Is(readyToExec) {
			println(prompt(igb.indents) + igb.lines)

			if perr != nil {
				handleParserError(perr, igb)
				continue
			}

			instructions := ivm.g.GenerateInstructions(program.Statements)
			ivm.v.REPLExec(instructions)

			r := ivm.v.GetREPLResult()

			// Suppress echo back on trailing ';'
			if igb.cmds != nil {
				if t := igb.cmds[len(igb.cmds)-1]; string(t[len(t)-1]) != semicolon {
					println(echo, r)
				}
			} else {
				if string(igb.lines[len(igb.lines)-1]) != semicolon {
					println(echo, r)
				}
			}
			//}
			igb.cmds = nil
		}
	}
}

func handleParserError(e *parserErr.Error, igb *iGb) {
	if e != nil {
		if !e.IsEOF() {
			fmt.Println(e.Message)
		}
		println(prompt(igb.indents) + indent(igb.indents) + igb.lines)
	}
}

// Polymorphic helper functions --------------------------------------------

// Other helper functions --------------------------------------------------

// newIgb initializes iGb.
func newIgb() *iGb {
	return &iGb{
		cmds:    nil,
		indents: 0,
		sm: fsm.NewFSM(
			readyToExec,
			fsm.Events{
				{Name: Waiting, Src: []string{waitEnded, readyToExec}, Dst: Waiting},
				{Name: waitEnded, Src: []string{Waiting}, Dst: waitEnded},
				{Name: waitExited, Src: []string{Waiting, waitEnded}, Dst: readyToExec},
				{Name: readyToExec, Src: []string{waitEnded, readyToExec}, Dst: readyToExec},
			},
			fsm.Callbacks{},
		),
		completer: readline.NewPrefixCompleter(
			readline.PcItem(help),
			readline.PcItem(reset),
		),
	}
}

//
// newIVM initializes iVM.
func newIVM() (ivm iVM, err error) {
	ivm = iVM{}
	v, err := vm.New(os.Getenv("GOBY_ROOT"), []string{})

	if err != nil {
		return ivm, err
	}
	ivm.v = v
	ivm.v.SetClassISIndexTable("")
	ivm.v.SetMethodISIndexTable("")
	ivm.v.InitForREPL()
	// Initialize parser, lexer is not important here
	ivm.p = parser.New(lexer.New(""))
	ivm.p.Mode = parser.REPLMode
	program, _ := ivm.p.ParseProgram()
	// Initialize code generator, and it will behavior a little different in REPL mode.
	ivm.g = bytecode.NewGenerator()
	ivm.g.REPL = true
	ivm.g.InitTopLevelScope(program)
	return ivm, nil
}

// readIgb fetches one line from input, with helps of Readline lib.
func readIgb(igb *iGb, err error) (*iGb, error) {
	igb.rl.Config.UniqueEditLine = true // required to update the previous prompt
	igb.lines, err = igb.rl.Readline()
	igb.rl.Config.UniqueEditLine = false

	igb.lines = strings.TrimSpace(igb.lines)
	igb.lines = strings.TrimPrefix(igb.lines, prmpt1)
	igb.lines = strings.TrimPrefix(igb.lines, prmpt2)
	igb.lines = strings.TrimSpace(igb.lines)
	return igb, err
}

// filterInput just ignores Ctrl-z.
func filterInput(r rune) (rune, bool) {
	switch r {
	case readline.CharCtrlZ:
		return r, false
	}
	return r, true
}

// usage shows help lines.
func usage(w io.Writer, c *readline.PrefixCompleter) {
	io.WriteString(w, "commands:\n")
	io.WriteString(w, c.Tree("   "))
}

// indent performs indentation with space padding.
func indent(c int) string {
	var s string
	for i := 0; i < c; i++ {
		s = s + pad
	}
	return s
}

// prompt switches prompt sign.
func prompt(s int) string {
	if s > 0 {
		return prompt2
	}
	return prompt1
}

// fortune is just a fun item to show slot machine: receiving rep-digit would imply your fortune ;-)
func fortune() string {
	if runtime.GOOS == "windows" {
		return ""
	}
	var randSrc = rand.NewSource(time.Now().UnixNano())
	s := strings.Split(emojis, "")
	l := len(s)
	r := randSrc.Int63() % int64(l)
	return s[r]
}
<commit_msg>clear cmds<commit_after>package igb

import (
	"fmt"
	parserErr "github.com/goby-lang/goby/compiler/parser/errors"
	"io"
	"log"
	"math/rand"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/chzyer/readline"
	"github.com/goby-lang/goby/compiler/bytecode"
	"github.com/goby-lang/goby/compiler/lexer"
	"github.com/goby-lang/goby/compiler/parser"
	"github.com/goby-lang/goby/vm"
	"github.com/looplab/fsm"
	"github.com/mattn/go-colorable"
)

const (
	prmpt1    = "»"
	prmpt2    = "¤"
	prompt1   = "\033[32m" + prmpt1 + "\033[0m "
	prompt2   = "\033[31m" + prmpt2 + "\033[0m "
	pad       = "  "
	echo      = "\033[33m#»\033[0m"
	interrupt = "^C"
	semicolon = ";"
	help      = "help"
	reset     = "reset"

	readyToExec = "readyToExec"
	Waiting     = "waiting"
	waitEnded   = "waitEnded"
	waitExited  = "waitExited"

	emojis = "😀😁😂🤣😃😄😅😆😉😊😋😎😍😘😗😙😚🙂🤗🤔😐😑😶🙄😏😮😪😴😌😛😜😝🤤🙃🤑😲😭😳🤧😇🤠🤡🤥🤓😈👿👹👺💀👻👽🤖💩😺😸😹😻😼😽"
)

// iGb holds internal states of iGb.
type iGb struct {
	sm        *fsm.FSM
	rl        *readline.Instance
	completer *readline.PrefixCompleter
	lines     string
	cmds      []string
	indents   int
}

// iVM holds VM only for iGb.
type iVM struct {
	v *vm.VM
	p *parser.Parser
	g *bytecode.Generator
}

var out io.Writer

func init() {
	if runtime.GOOS == "windows" {
		out = colorable.NewColorableStderr()
	} else {
		out = os.Stderr
	}
}

func println(s ...string) {
	out.Write([]byte(strings.Join(s, " ") + "\n"))
}

// StartIgb starts goby's REPL.
func StartIgb(version string) {
reset:
	var err error
	igb := newIgb()

	igb.rl, err = readline.NewEx(&readline.Config{
		Prompt:              prompt1,
		HistoryFile:         filepath.Join(os.TempDir(), "readline_goby.tmp"),
		AutoComplete:        igb.completer,
		InterruptPrompt:     interrupt,
		HistorySearchFold:   true,
		FuncFilterInputRune: filterInput,
	})
	defer igb.rl.Close()

	if err != nil {
		fmt.Printf("iGb error: %s", err)
		return
	}

	log.SetOutput(igb.rl.Stderr())

	println("Goby", version, fortune(), fortune(), fortune())

	ivm, err := newIVM()

	if err != nil {
		fmt.Println(err.Error())
		return
	}

	for {
		igb, err = readIgb(igb, err)

		// Interruption handling
		if err != nil {
			switch {
			case err == io.EOF:
				println(igb.lines + "")
				return
			case err == readline.ErrInterrupt: // Pressing Ctrl-C
				if len(igb.lines) == 0 {
					if igb.cmds == nil {
						println("")
						println("Bye!")
						return
					}
				}
				// Erasing command buffer
				igb.indents = 0
				igb.rl.SetPrompt(prompt1)
				igb.sm.Event(waitExited)
				igb.cmds = nil
				println(" -- block cleared")
				continue
			}
		}

		// Command handling
		switch {
		case strings.HasPrefix(igb.lines, "#"):
			println(prompt(igb.indents) + indent(igb.indents) + igb.lines)
			continue
		case igb.lines == help:
			println(prompt(igb.indents) + igb.lines)
			usage(igb.rl.Stderr(), igb.completer)
			continue
		case igb.lines == reset:
			igb.rl = nil
			igb.cmds = nil
			println(prompt(igb.indents) + igb.lines)
			println("Restarting iGb...")
			goto reset
		case igb.lines == "":
			println(prompt(igb.indents) + indent(igb.indents) + igb.lines)
			continue
		}

		ivm.p.Lexer = lexer.New(igb.lines)
		program, perr := ivm.p.ParseProgram()

		// Parse error handling
		if perr != nil {
			switch {
			case perr.IsEOF():
				if !igb.sm.Is(Waiting) {
					igb.sm.Event(Waiting)
				}
				println(prompt(igb.indents) + indent(igb.indents) + igb.lines)
				igb.indents++
				igb.rl.SetPrompt(prompt(igb.indents) + indent(igb.indents))
				igb.cmds = append(igb.cmds, igb.lines)
				continue
			case perr.IsUnexpectedEnd() && len(igb.cmds) == 0:
				// If igb.cmds is empty, it means that user just typed 'end' without corresponding statement/expression
				println(prompt(igb.indents) + indent(igb.indents) + igb.lines)
				igb.indents = 0
				igb.rl.SetPrompt(prompt1)
				fmt.Println(perr.Message)
				igb.cmds = nil
				continue
			case perr.IsUnexpectedEnd():
				if igb.indents > 1 {
					igb.indents--
					println(prompt(igb.indents) + indent(igb.indents) + igb.lines)
					igb.sm.Event(Waiting)
					igb.rl.SetPrompt(prompt(igb.indents) + indent(igb.indents))
					igb.cmds = append(igb.cmds, igb.lines)
					continue
				}
				igb.indents = 0
				igb.sm.Event(waitEnded)
				igb.rl.SetPrompt(prompt(igb.indents) + indent(igb.indents))
				igb.cmds = append(igb.cmds, igb.lines)
			}
		}

		if igb.sm.Is(Waiting) && igb.indents > 0 {
			println(prompt(igb.indents) + indent(igb.indents) + igb.lines)
			igb.rl.SetPrompt(prompt(igb.indents) + indent(igb.indents))
			igb.cmds = append(igb.cmds, igb.lines)
			continue
		}

		if igb.sm.Is(waitEnded) {
			ivm.p.Lexer = lexer.New(string(strings.Join(igb.cmds, "\n")))

			// Test if current input can be properly parsed.
			program, perr = ivm.p.ParseProgram()

			if perr != nil {
				handleParserError(perr, igb)
				igb.cmds = nil
				igb.sm.Event(readyToExec)
				continue
			}

			// If everything goes well, reset state and statements buffer
			igb.rl.SetPrompt(prompt(igb.indents))
			igb.sm.Event(readyToExec)
		}

		if igb.sm.Is(readyToExec) {
			println(prompt(igb.indents) + igb.lines)

			if perr != nil {
				handleParserError(perr, igb)
				continue
			}

			instructions := ivm.g.GenerateInstructions(program.Statements)
			ivm.v.REPLExec(instructions)

			r := ivm.v.GetREPLResult()

			// Suppress echo back on trailing ';'
			if igb.cmds != nil {
				if t := igb.cmds[len(igb.cmds)-1]; string(t[len(t)-1]) != semicolon {
					println(echo, r)
				}
			} else {
				if string(igb.lines[len(igb.lines)-1]) != semicolon {
					println(echo, r)
				}
			}
			//}
			igb.cmds = nil
		}
	}
}

func handleParserError(e *parserErr.Error, igb *iGb) {
	if e != nil {
		if !e.IsEOF() {
			fmt.Println(e.Message)
		}
		println(prompt(igb.indents) + indent(igb.indents) + igb.lines)
	}
}

// Polymorphic helper functions --------------------------------------------

// Other helper functions --------------------------------------------------

// newIgb initializes iGb.
func newIgb() *iGb {
	return &iGb{
		cmds:    nil,
		indents: 0,
		sm: fsm.NewFSM(
			readyToExec,
			fsm.Events{
				{Name: Waiting, Src: []string{waitEnded, readyToExec}, Dst: Waiting},
				{Name: waitEnded, Src: []string{Waiting}, Dst: waitEnded},
				{Name: waitExited, Src: []string{Waiting, waitEnded}, Dst: readyToExec},
				{Name: readyToExec, Src: []string{waitEnded, readyToExec}, Dst: readyToExec},
			},
			fsm.Callbacks{},
		),
		completer: readline.NewPrefixCompleter(
			readline.PcItem(help),
			readline.PcItem(reset),
		),
	}
}

//
// newIVM initializes iVM.
func newIVM() (ivm iVM, err error) {
	ivm = iVM{}
	v, err := vm.New(os.Getenv("GOBY_ROOT"), []string{})

	if err != nil {
		return ivm, err
	}
	ivm.v = v
	ivm.v.SetClassISIndexTable("")
	ivm.v.SetMethodISIndexTable("")
	ivm.v.InitForREPL()
	// Initialize parser, lexer is not important here
	ivm.p = parser.New(lexer.New(""))
	ivm.p.Mode = parser.REPLMode
	program, _ := ivm.p.ParseProgram()
	// Initialize code generator, and it will behavior a little different in REPL mode.
	ivm.g = bytecode.NewGenerator()
	ivm.g.REPL = true
	ivm.g.InitTopLevelScope(program)
	return ivm, nil
}

// readIgb fetches one line from input, with helps of Readline lib.
func readIgb(igb *iGb, err error) (*iGb, error) {
	igb.rl.Config.UniqueEditLine = true // required to update the previous prompt
	igb.lines, err = igb.rl.Readline()
	igb.rl.Config.UniqueEditLine = false

	igb.lines = strings.TrimSpace(igb.lines)
	igb.lines = strings.TrimPrefix(igb.lines, prmpt1)
	igb.lines = strings.TrimPrefix(igb.lines, prmpt2)
	igb.lines = strings.TrimSpace(igb.lines)
	return igb, err
}

// filterInput just ignores Ctrl-z.
func filterInput(r rune) (rune, bool) {
	switch r {
	case readline.CharCtrlZ:
		return r, false
	}
	return r, true
}

// usage shows help lines.
func usage(w io.Writer, c *readline.PrefixCompleter) {
	io.WriteString(w, "commands:\n")
	io.WriteString(w, c.Tree("   "))
}

// indent performs indentation with space padding.
func indent(c int) string {
	var s string
	for i := 0; i < c; i++ {
		s = s + pad
	}
	return s
}

// prompt switches prompt sign.
func prompt(s int) string {
	if s > 0 {
		return prompt2
	}
	return prompt1
}

// fortune is just a fun item to show slot machine: receiving rep-digit would imply your fortune ;-)
func fortune() string {
	if runtime.GOOS == "windows" {
		return ""
	}
	var randSrc = rand.NewSource(time.Now().UnixNano())
	s := strings.Split(emojis, "")
	l := len(s)
	r := randSrc.Int63() % int64(l)
	return s[r]
}
<|endoftext|>


<commit_before>package logberry

import (
	"bytes"
	"fmt"
	"runtime"
)

// Error is used to report a fault, capturing a human-oriented message
// describing the problem, structured data providing identifying
// details, the source code file name and line number location at
// which this Error was generated, and if appropriate a preceding
// error that caused this higher level fault.
type Error struct {
	Message string
	Data    D

	File string
	Line int

	Cause error
}

func newerror(msg string, data []interface{}) *Error {
	e := &Error{
		Message: msg,
		Data:    DAggregate(data),
	}
	return e
}

func wraperror(msg string, err error, data []interface{}) *Error {
	e := newerror(msg, data)
	e.Cause = err
	return e
}

// NewError generates a new Error capturing the given human-oriented
// message and optionally structured data associated with this fault.
// The source code position to be reported by this Error is the point
// at which NewError was called.
func NewError(msg string, data ...interface{}) *Error {
	e := newerror(msg, data)
	e.Locate(1)
	return e
}

// WrapError generates a new Error capturing the given human-oriented
// message, a preceding error which caused this higher level fault,
// and optionally structured data associated with this fault.  The
// source code position to be reported by this Error is the point at
// which WrapError was called.
func WrapError(msg string, err error, data ...interface{}) *Error {
	e := wraperror(msg, err, data)
	e.Locate(1)
	return e
}

// Locate sets the source code position to be reported with this error
// as that point where the Locate call is made.  In general it should
// not be necessary to invoke this manually.
func (e *Error) Locate(skip int) {
	_, file, line, ok := runtime.Caller(skip + 1)
	if ok {
		e.File = file
		e.Line = line
	}
}

// Error implements the standard Go error interface, returning a
// human-oriented text string serialization of the Error.
func (e *Error) Error() string {

	var buffer = new(bytes.Buffer)

	buffer.WriteString(e.Message)

	if e.File != "" {
		fmt.Fprintf(buffer, " [%v:%v]", e.File, e.Line)
	}

	if len(e.Data) > 0 {
		fmt.Fprintf(buffer, " %v", e.Data.String())
	}

	if e.Cause != nil {
		fmt.Fprintf(buffer, ":%v", e.Error())
	}

	return buffer.String()
}

// String returns a human-oriented text string serialization of the
// Error.
func (e *Error) String() string {
	return e.Error()
}
<commit_msg>Infinite loop in error; thought this was already fixed?<commit_after>package logberry

import (
	"bytes"
	"fmt"
	"runtime"
)

// Error is used to report a fault, capturing a human-oriented message
// describing the problem, structured data providing identifying
// details, the source code file name and line number location at
// which this Error was generated, and if appropriate a preceding
// error that caused this higher level fault.
type Error struct {
	Message string
	Data    D

	File string
	Line int

	Cause error
}

func newerror(msg string, data []interface{}) *Error {
	e := &Error{
		Message: msg,
		Data:    DAggregate(data),
	}
	return e
}

func wraperror(msg string, err error, data []interface{}) *Error {
	e := newerror(msg, data)
	e.Cause = err
	return e
}

// NewError generates a new Error capturing the given human-oriented
// message and optionally structured data associated with this fault.
// The source code position to be reported by this Error is the point
// at which NewError was called.
func NewError(msg string, data ...interface{}) *Error {
	e := newerror(msg, data)
	e.Locate(1)
	return e
}

// WrapError generates a new Error capturing the given human-oriented
// message, a preceding error which caused this higher level fault,
// and optionally structured data associated with this fault.  The
// source code position to be reported by this Error is the point at
// which WrapError was called.
func WrapError(msg string, err error, data ...interface{}) *Error {
	e := wraperror(msg, err, data)
	e.Locate(1)
	return e
}

// Locate sets the source code position to be reported with this error
// as that point where the Locate call is made.  In general it should
// not be necessary to invoke this manually.
func (e *Error) Locate(skip int) {
	_, file, line, ok := runtime.Caller(skip + 1)
	if ok {
		e.File = file
		e.Line = line
	}
}

// Error implements the standard Go error interface, returning a
// human-oriented text string serialization of the Error.
func (e *Error) Error() string {

	var buffer = new(bytes.Buffer)

	buffer.WriteString(e.Message)

	if e.File != "" {
		fmt.Fprintf(buffer, " [%v:%v]", e.File, e.Line)
	}

	if len(e.Data) > 0 {
		fmt.Fprintf(buffer, " %v", e.Data.String())
	}

	if e.Cause != nil {
		fmt.Fprintf(buffer, ":%v", e.Cause.Error())
	}

	return buffer.String()
}

// String returns a human-oriented text string serialization of the
// Error.
func (e *Error) String() string {
	return e.Error()
}
<|endoftext|>


<commit_before>package payload

import (
	"DNA/common/serialization"
	"DNA/crypto"
	. "DNA/errors"
	"errors"
	"io"
)

type StateUpdate struct {
	Namespace []byte
	Key       []byte
	Value     []byte
	Updater   *crypto.PubKey
}

func (su *StateUpdate) Data() []byte {
	return []byte{0}
}

func (su *StateUpdate) Serialize(w io.Writer) error {
	err := serialization.WriteVarBytes(w, su.Namespace)
	if err != nil {
		return NewDetailErr(err, ErrNoCode, "[StateUpdate], Namespace serialize failed.")
	}

	err = serialization.WriteVarBytes(w, su.Key)
	if err != nil {
		return NewDetailErr(err, ErrNoCode, "[StateUpdate], key serialize failed.")
	}

	err = serialization.WriteVarBytes(w, su.Value)
	if err != nil {
		return NewDetailErr(err, ErrNoCode, "[StateUpdate], value serialize failed.")
	}

	su.Updater.Serialize(w)

	return nil
}

func (su *StateUpdate) Deserialize(r io.Reader) error {
	var err error

	su.Namespace, err = serialization.ReadVarBytes(r)
	if err != nil {
		return NewDetailErr(errors.New("[StateUpdate], Namespace deserialize failed."), ErrNoCode, "")
	}

	su.Key, err = serialization.ReadVarBytes(r)
	if err != nil {
		return NewDetailErr(errors.New("[StateUpdate], key deserialize failed."), ErrNoCode, "")
	}

	su.Value, err = serialization.ReadVarBytes(r)
	if err != nil {
		return NewDetailErr(errors.New("[StateUpdate], value deserialize failed."), ErrNoCode, "")
	}

	su.Updater = new(crypto.PubKey)
	err = su.Updater.DeSerialize(r)
	if err != nil {
		return NewDetailErr(err, ErrNoCode, "[StateUpdate], updater Deserialize failed.")
	}

	return nil
}
<commit_msg>StateUpdate payload add version byte<commit_after>package payload

import (
	"DNA/common/serialization"
	"DNA/crypto"
	. "DNA/errors"
	"errors"
	"io"
)

type StateUpdate struct {
	Namespace []byte
	Key       []byte
	Value     []byte
	Updater   *crypto.PubKey
}

func (su *StateUpdate) Data(version byte) []byte {
	return []byte{0}
}

func (su *StateUpdate) Serialize(w io.Writer, version byte) error {
	err := serialization.WriteVarBytes(w, su.Namespace)
	if err != nil {
		return NewDetailErr(err, ErrNoCode, "[StateUpdate], Namespace serialize failed.")
	}

	err = serialization.WriteVarBytes(w, su.Key)
	if err != nil {
		return NewDetailErr(err, ErrNoCode, "[StateUpdate], key serialize failed.")
	}

	err = serialization.WriteVarBytes(w, su.Value)
	if err != nil {
		return NewDetailErr(err, ErrNoCode, "[StateUpdate], value serialize failed.")
	}

	su.Updater.Serialize(w)

	return nil
}

func (su *StateUpdate) Deserialize(r io.Reader, version byte) error {
	var err error

	su.Namespace, err = serialization.ReadVarBytes(r)
	if err != nil {
		return NewDetailErr(errors.New("[StateUpdate], Namespace deserialize failed."), ErrNoCode, "")
	}

	su.Key, err = serialization.ReadVarBytes(r)
	if err != nil {
		return NewDetailErr(errors.New("[StateUpdate], key deserialize failed."), ErrNoCode, "")
	}

	su.Value, err = serialization.ReadVarBytes(r)
	if err != nil {
		return NewDetailErr(errors.New("[StateUpdate], value deserialize failed."), ErrNoCode, "")
	}

	su.Updater = new(crypto.PubKey)
	err = su.Updater.DeSerialize(r)
	if err != nil {
		return NewDetailErr(err, ErrNoCode, "[StateUpdate], updater Deserialize failed.")
	}

	return nil
}
<|endoftext|>


<commit_before>package tar

import (
	"archive/tar"
	"bytes"
	"io"
	"io/ioutil"
)

func Fuzz(data []byte) int {
	t := tar.NewReader(bytes.NewReader(data))
	score := 0
	for {
		_, err := t.Next()
		if err != nil {
			return score
		}
		io.Copy(ioutil.Discard, t)
		score++
	}
}
<commit_msg>extend tar test<commit_after>package tar

import (
	"archive/tar"
	"bytes"
	"io"
	"io/ioutil"

	"github.com/dvyukov/go-fuzz/examples/fuzz"
)

func Fuzz(data []byte) int {
	t := tar.NewReader(bytes.NewReader(data))
	var headers []*tar.Header
	var contents [][]byte
	for {
		hdr, err := t.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return 0
		}
		if len(hdr.Name) > 1e6 ||
			len(hdr.Linkname) > 1e6 ||
			len(hdr.Uname) > 1e6 ||
			len(hdr.Gname) > 1e6 ||
			len(hdr.Xattrs) > 1e6 {
			panic("huge header data")
		}
		if hdr.Size > 1e6 {
			panic("huge claimed file size")
		}
		fdata, err := ioutil.ReadAll(t)
		if err != nil {
			return 0
		}
		if int64(len(fdata)) > hdr.Size {
			panic("long read")
		}
		if int64(len(fdata)) < hdr.Size {
			panic("short read")
		}
		hdr1 := *hdr // make a copy to be safe
		headers = append(headers, &hdr1)
		contents = append(contents, fdata)
	}
	buf := new(bytes.Buffer)
	w := tar.NewWriter(buf)
	for i, hdr := range headers {
		err := w.WriteHeader(hdr)
		if err != nil {
			panic(err)
		}
		n, err := w.Write(contents[i])
		if err != nil {
			panic(err)
		}
		if n != len(contents[i]) {
			panic("short write")
		}
	}
	err := w.Close()
	if err != nil {
		panic(err)
	}
	t1 := tar.NewReader(buf)
	for i := 0; ; i++ {
		hdr, err := t1.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			panic(err)
		}
		fdata, err := ioutil.ReadAll(t)
		if err != nil {
			panic(err)
		}
		if !fuzz.DeepEqual(hdr, headers[i]) {
			panic("headers diffs")
		}
		if !bytes.Equal(fdata, contents[i]) {
			panic("data diffs")
		}
	}
	return 1
}
<|endoftext|>


<commit_before>package gen

import (
	"fmt"
	"html/template"

	"github.com/robertkrimen/otto"
)

type Template struct {
	vm      *otto.Otto
	jsFuncs []string
	funcs   template.FuncMap
	tpl     *template.Template
}

func (t *Template) funcMap() template.FuncMap {
	rst := make(template.FuncMap)
	for _, name := range t.jsFuncs {
		rst[name] = t.jsTplFunc(name)
	}
	for k, v := range t.funcs {
		rst[k] = v
	}
	return rst
}

func (t *Template) jsTplFunc(name string) func(interface{}) string {
	return func(arg interface{}) string {
		call := fmt.Sprintf("Tpl.funcs.%s", name)
		rst, err := t.vm.Call(call, nil, arg)
		if err != nil {
			Panic(err.Error())
		}
		if !rst.IsString() {
			Panic("non string retrun value from " + name + " template func")
		}
		s, _ := rst.ToString()
		return s
	}
}

func (t *Template) New() *Template {
	tpl := template.New("base").Funcs(t.funcMap())
	return &Template{
		jsFuncs: t.jsFuncs,
		vm:      t.vm,
		funcs:   t.funcs,
		tpl:     tpl,
	}
}
<commit_msg>Make *Template inherit *template.Template<commit_after>package gen

import (
	"fmt"
	"html/template"

	"github.com/robertkrimen/otto"
)

type Template struct {
	vm      *otto.Otto
	jsFuncs []string
	funcs   template.FuncMap
	*template.Template
}

func (t *Template) funcMap() template.FuncMap {
	rst := make(template.FuncMap)
	for _, name := range t.jsFuncs {
		rst[name] = t.jsTplFunc(name)
	}
	for k, v := range t.funcs {
		rst[k] = v
	}
	return rst
}

func (t *Template) jsTplFunc(name string) func(interface{}) string {
	return func(arg interface{}) string {
		call := fmt.Sprintf("Tpl.funcs.%s", name)
		rst, err := t.vm.Call(call, nil, arg)
		if err != nil {
			Panic(err.Error())
		}
		if !rst.IsString() {
			Panic("non string retrun value from " + name + " template func")
		}
		s, _ := rst.ToString()
		return s
	}
}

func (t *Template) New() *Template {
	tpl := template.New("base").Funcs(t.funcMap())
	return &Template{
		jsFuncs:  t.jsFuncs,
		vm:       t.vm,
		funcs:    t.funcs,
		Template: tpl,
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 Jesse Allen. All rights reserved
// Released under the MIT license found in the LICENSE file.

package main

import (
	"flag"
	"html/template"
)

var (
	LayoutTemplateGlob = flag.String("layouts", "templates/layouts/*.html", "Pattern for layout templates")
	HelperTemplateGlob = flag.String("helpers", "templates/helpers/*.html", "Pattern for helper templates")
)

/*
func init() {
	var err error

	if !flag.Parsed() {
		flag.Parse()
	}
}
*/

// Load base templates and templates from the provided pattern
// TODO: if performance becomes an issue, we can start caching the base templates, and cloning
func LoadTemplates(patterns ...string) (*template.Template, error) {
	b, err := template.ParseGlob(*LayoutTemplateGlob)
	if err != nil {
		return nil, err
	}
	for _, p := range append([]string{*HelperTemplateGlob}, patterns...) {
		_, err = b.ParseGlob(p)
		if err != nil {
			return nil, err
		}
	}
	return b, nil
}
<commit_msg>Add support for Markdown Processing<commit_after>// Copyright 2013 Jesse Allen. All rights reserved
// Released under the MIT license found in the LICENSE file.

package main

import (
	"flag"
	"github.com/russross/blackfriday"
	"html/template"
)

var (
	LayoutTemplateGlob = flag.String("layouts", "templates/layouts/*.html", "Pattern for layout templates")
	HelperTemplateGlob = flag.String("helpers", "templates/helpers/*.html", "Pattern for helper templates")
)

/*
func init() {
	var err error

	if !flag.Parsed() {
		flag.Parse()
	}
}
*/

// Load base templates and templates from the provided pattern
// TODO: if performance becomes an issue, we can start caching the base templates, and cloning
func LoadTemplates(patterns ...string) (*template.Template, error) {
	var err error
	// add some key helper functions to the templates
	b := template.New("base").Funcs(template.FuncMap{
		"markdownCommon": func(raw string) template.HTML {
			return template.HTML(blackfriday.MarkdownCommon([]byte(raw)))
		},
		"markdownBasic": func(raw string) template.HTML {
			return template.HTML(blackfriday.MarkdownBasic([]byte(raw)))
		},
	})
	b, err = b.ParseGlob(*LayoutTemplateGlob)
	if err != nil {
		return nil, err
	}
	for _, p := range append([]string{*HelperTemplateGlob}, patterns...) {
		_, err = b.ParseGlob(p)
		if err != nil {
			return nil, err
		}
	}
	return b, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"strings"
	"text/template"

	"github.com/hashicorp/consul-template/util"
)

type Template struct {
	//
	Path string

	//
	dependencies []util.Dependency
}

//
func NewTemplate(path string) (*Template, error) {
	template := &Template{Path: path}
	if err := template.init(); err != nil {
		return nil, err
	}

	return template, nil
}

// HashCode returns the map value for this Template
func (t *Template) HashCode() string {
	return fmt.Sprintf("Template|%s", t.Path)
}

// Dependencies returns the dependencies that this template has.
func (t *Template) Dependencies() []util.Dependency {
	return t.dependencies
}

// Decodestring calls jsonpath.DecodeString, which returns a structure for valid json
func DecodeString(s string) (interface{}, error) {
	// Empty string returns an empty interface
	if len(s) < 2 {
		return map[string]interface{}{}, nil
	}

	var data map[string]interface{}
	if err := json.Unmarshal([]byte(s), &data); err != nil {
		return nil, err
	}

	return data, nil
}

// ServiceByTag is a template func that takes the provided services and
// produces a map based on service tags.
//
// The map key is a string representing the service tag. The map value is a
// slice of Services which have the tag assigned.
func ServiceByTag(in []*util.Service) map[string][]*util.Service {
	m := make(map[string][]*util.Service)
	for _, s := range in {
		for _, t := range s.Tags {
			m[t] = append(m[t], s)
		}
	}
	return m
}

// Execute takes the given template context and processes the template.
//
// If the TemplateContext is nil, an error will be returned.
//
// If the TemplateContext does not have all required Dependencies, an error will
// be returned.
func (t *Template) Execute(c *TemplateContext) ([]byte, error) {
	if c == nil {
		return nil, errors.New("templateContext must be given")
	}

	// Make sure the context contains everything we need
	if err := t.validateDependencies(c); err != nil {
		return nil, err
	}

	// Render the template
	contents, err := ioutil.ReadFile(t.Path)
	if err != nil {
		return nil, err
	}

	tmpl, err := template.New("out").Funcs(template.FuncMap{
		// API functions
		"file": func(s string) string {
			return c.File[s]
		},
		"keyPrefix": func(s string) []*util.KeyPair {
			log.Printf("[WARN] DEPRECATED: Please use `ls` or `tree` instead of `keyPrefix`")
			return c.KeyPrefixes[s]
		},
		"key": func(s string) string {
			return c.Keys[s]
		},
		"ls": func(s string) []*util.KeyPair {
			result := make([](*util.KeyPair), 0)
			// Only return non-empty top-level keys
			for _, pair := range c.KeyPrefixes[s] {
				if pair.Key != "" && !strings.Contains(pair.Key, "/") {
					result = append(result, pair)
				}
			}
			return result
		},
		"service": func(s ...string) []*util.Service {
			// We should not get any errors here as the same arguments will
			// have been processed in the template pre process stage.
			d, err := util.ParseServiceDependency(s...)
			if err != nil {
				panic(err)
			}
			return c.Services[d.Key()]
		},
		"tree": func(s string) []*util.KeyPair {
			return c.KeyPrefixes[s]
		},

		// Helper functions
		"byTag":      c.groupByTag,
		"parseJSON":  c.decodeJSON,
		"replaceAll": c.replaceAll,
		"toLower":    c.toLower,
		"toTitle":    c.toTitle,
		"toUpper":    c.toUpper,
	}).Parse(string(contents))

	if err != nil {
		return nil, err
	}

	var buff = new(bytes.Buffer)
	err = tmpl.Execute(buff, c)
	if err != nil {
		return nil, err
	}

	return buff.Bytes(), nil
}

// init reads the template file and parses all the required dependencies into a
// dependencies slice which is then added onto the Template.
func (t *Template) init() error {
	contents, err := ioutil.ReadFile(t.Path)
	if err != nil {
		return err
	}

	depsMap := make(map[string]util.Dependency)

	tmpl, err := template.New("out").Funcs(template.FuncMap{
		// API functions
		"file":      t.dependencyAcc(depsMap, DependencyTypeFile),
		"keyPrefix": t.dependencyAcc(depsMap, DependencyTypeKeyPrefix),
		"key":       t.dependencyAcc(depsMap, DependencyTypeKey),
		"ls":        t.dependencyAcc(depsMap, DependencyTypeKeyPrefix),
		"service":   t.dependencyAccVariadic(depsMap, DependencyTypeService),
		"tree":      t.dependencyAcc(depsMap, DependencyTypeKeyPrefix),

		// Helper functions
		"byTag":      t.noop,
		"parseJSON":  t.noop,
		"replaceAll": t.noop,
		"toLower":    t.noop,
		"toTitle":    t.noop,
		"toUpper":    t.noop,
	}).Parse(string(contents))

	if err != nil {
		return err
	}

	err = tmpl.Execute(ioutil.Discard, nil)
	if err != nil {
		return err
	}

	dependencies := make([]util.Dependency, 0, len(depsMap))
	for _, dep := range depsMap {
		dependencies = append(dependencies, dep)
	}
	depsMap = nil

	t.dependencies = dependencies

	return nil
}

// Helper function that is used by the dependency collecting.
func (t *Template) dependencyAcc(depsMap map[string]util.Dependency, dt DependencyType) func(string) (interface{}, error) {
	return func(s string) (interface{}, error) {
		switch dt {
		case DependencyTypeFile:
			d, err := util.ParseFileDependency(s)
			if err != nil {
				return nil, err
			}
			if _, ok := depsMap[d.HashCode()]; !ok {
				depsMap[d.HashCode()] = d
			}

			return "", nil
		case DependencyTypeKeyPrefix:
			d, err := util.ParseKeyPrefixDependency(s)
			if err != nil {
				return nil, err
			}
			if _, ok := depsMap[d.HashCode()]; !ok {
				depsMap[d.HashCode()] = d
			}

			return []*util.KeyPair{}, nil
		case DependencyTypeKey:
			d, err := util.ParseKeyDependency(s)
			if err != nil {
				return nil, err
			}
			if _, ok := depsMap[d.HashCode()]; !ok {
				depsMap[d.HashCode()] = d
			}

			return "", nil
		default:
			return nil, fmt.Errorf("unknown DependencyType %#v", dt)
		}
	}
}

// Helper function that is used by the dependency collecting fot remplate functions that accept variadic arguments.
func (t *Template) dependencyAccVariadic(depsMap map[string]util.Dependency, dt DependencyType) func(...string) (interface{}, error) {
	return func(s ...string) (interface{}, error) {
		switch dt {
		case DependencyTypeService:
			d, err := util.ParseServiceDependency(s...)
			if err != nil {
				return nil, err
			}
			if _, ok := depsMap[d.HashCode()]; !ok {
				depsMap[d.HashCode()] = d
			}

			return []*util.Service{}, nil
		default:
			return nil, fmt.Errorf("unknown variadic DependencyType %#v", dt)
		}
	}
}

// Validates that all required dependencies in t are defined in c.
func (t *Template) validateDependencies(c *TemplateContext) error {
	for _, dep := range t.Dependencies() {
		switch d := dep.(type) {
		case *util.FileDependency:
			if _, ok := c.File[d.Key()]; !ok {
				return fmt.Errorf("templateContext missing file `%s'", d.Key())
			}
		case *util.KeyPrefixDependency:
			if _, ok := c.KeyPrefixes[d.Key()]; !ok {
				return fmt.Errorf("templateContext missing keyPrefix `%s'", d.Key())
			}
		case *util.KeyDependency:
			if _, ok := c.Keys[d.Key()]; !ok {
				return fmt.Errorf("templateContext missing key `%s'", d.Key())
			}
		case *util.ServiceDependency:
			if _, ok := c.Services[d.Key()]; !ok {
				return fmt.Errorf("templateContext missing service `%s'", d.Key())
			}
		default:
			return fmt.Errorf("unknown dependency type in template e %#v", d)
		}
	}

	return nil
}

// noop is a special function that returns itself. This is used during the
// dependency accumulation to allow the template to be processed once.
func (t *Template) noop(thing ...interface{}) (interface{}, error) {
	return thing[len(thing)-1], nil
}

// TemplateContext is what Template uses to determine the values that are
// available for template parsing.
type TemplateContext struct {
	Services    map[string][]*util.Service
	Keys        map[string]string
	KeyPrefixes map[string][]*util.KeyPair
	File        map[string]string
}

// decodeJSON returns a structure for valid JSON
func (c *TemplateContext) decodeJSON(s string) (interface{}, error) {
	var data map[string]interface{}
	if err := json.Unmarshal([]byte(s), &data); err != nil {
		return nil, err
	}

	return data, nil
}

// groupByTag is a template func that takes the provided services and
// produces a map based on Service tags.
//
// The map key is a string representing the service tag. The map value is a
// slice of Services which have the tag assigned.
func (c *TemplateContext) groupByTag(in []*util.Service) map[string][]*util.Service {
	m := make(map[string][]*util.Service)
	for _, s := range in {
		for _, t := range s.Tags {
			m[t] = append(m[t], s)
		}
	}
	return m
}

// toLower converts the given string (usually by a pipe) to lowercase.
func (c *TemplateContext) toLower(s string) (string, error) {
	return strings.ToLower(s), nil
}

// toTitle converts the given string (usually by a pipe) to titlecase.
func (c *TemplateContext) toTitle(s string) (string, error) {
	return strings.Title(s), nil
}

// toUpper converts the given string (usually by a pipe) to uppercase.
func (c *TemplateContext) toUpper(s string) (string, error) {
	return strings.ToUpper(s), nil
}

// replaceAll replaces all occurrences of a value in a string with the given
// replacement value.
func (c *TemplateContext) replaceAll(f, t, s string) (string, error) {
	return strings.Replace(s, f, t, -1), nil
}

// DependencyType is an enum type that says the kind of the dependency.
type DependencyType byte

const (
	DependencyTypeInvalid DependencyType = iota
	DependencyTypeService
	DependencyTypeKey
	DependencyTypeKeyPrefix
	DependencyTypeFile
)
<commit_msg>Remove panic call from template func.<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"strings"
	"text/template"

	"github.com/hashicorp/consul-template/util"
)

type Template struct {
	//
	Path string

	//
	dependencies []util.Dependency
}

//
func NewTemplate(path string) (*Template, error) {
	template := &Template{Path: path}
	if err := template.init(); err != nil {
		return nil, err
	}

	return template, nil
}

// HashCode returns the map value for this Template
func (t *Template) HashCode() string {
	return fmt.Sprintf("Template|%s", t.Path)
}

// Dependencies returns the dependencies that this template has.
func (t *Template) Dependencies() []util.Dependency {
	return t.dependencies
}

// Decodestring calls jsonpath.DecodeString, which returns a structure for valid json
func DecodeString(s string) (interface{}, error) {
	// Empty string returns an empty interface
	if len(s) < 2 {
		return map[string]interface{}{}, nil
	}

	var data map[string]interface{}
	if err := json.Unmarshal([]byte(s), &data); err != nil {
		return nil, err
	}

	return data, nil
}

// ServiceByTag is a template func that takes the provided services and
// produces a map based on service tags.
//
// The map key is a string representing the service tag. The map value is a
// slice of Services which have the tag assigned.
func ServiceByTag(in []*util.Service) map[string][]*util.Service {
	m := make(map[string][]*util.Service)
	for _, s := range in {
		for _, t := range s.Tags {
			m[t] = append(m[t], s)
		}
	}
	return m
}

// Execute takes the given template context and processes the template.
//
// If the TemplateContext is nil, an error will be returned.
//
// If the TemplateContext does not have all required Dependencies, an error will
// be returned.
func (t *Template) Execute(c *TemplateContext) ([]byte, error) {
	if c == nil {
		return nil, errors.New("templateContext must be given")
	}

	// Make sure the context contains everything we need
	if err := t.validateDependencies(c); err != nil {
		return nil, err
	}

	// Render the template
	contents, err := ioutil.ReadFile(t.Path)
	if err != nil {
		return nil, err
	}

	tmpl, err := template.New("out").Funcs(template.FuncMap{
		// API functions
		"file": func(s string) string {
			return c.File[s]
		},
		"keyPrefix": func(s string) []*util.KeyPair {
			log.Printf("[WARN] DEPRECATED: Please use `ls` or `tree` instead of `keyPrefix`")
			return c.KeyPrefixes[s]
		},
		"key": func(s string) string {
			return c.Keys[s]
		},
		"ls": func(s string) []*util.KeyPair {
			result := make([](*util.KeyPair), 0)
			// Only return non-empty top-level keys
			for _, pair := range c.KeyPrefixes[s] {
				if pair.Key != "" && !strings.Contains(pair.Key, "/") {
					result = append(result, pair)
				}
			}
			return result
		},
		"service": func(s ...string) ([]*util.Service, error) {
			// We should not get any errors here as the same arguments will
			// have been processed in the template pre process stage.
			d, err := util.ParseServiceDependency(s...)
			if err != nil {
				return nil, err
			}
			return c.Services[d.Key()], nil
		},
		"tree": func(s string) []*util.KeyPair {
			return c.KeyPrefixes[s]
		},

		// Helper functions
		"byTag":      c.groupByTag,
		"parseJSON":  c.decodeJSON,
		"replaceAll": c.replaceAll,
		"toLower":    c.toLower,
		"toTitle":    c.toTitle,
		"toUpper":    c.toUpper,
	}).Parse(string(contents))

	if err != nil {
		return nil, err
	}

	var buff = new(bytes.Buffer)
	err = tmpl.Execute(buff, c)
	if err != nil {
		return nil, err
	}

	return buff.Bytes(), nil
}

// init reads the template file and parses all the required dependencies into a
// dependencies slice which is then added onto the Template.
func (t *Template) init() error {
	contents, err := ioutil.ReadFile(t.Path)
	if err != nil {
		return err
	}

	depsMap := make(map[string]util.Dependency)

	tmpl, err := template.New("out").Funcs(template.FuncMap{
		// API functions
		"file":      t.dependencyAcc(depsMap, DependencyTypeFile),
		"keyPrefix": t.dependencyAcc(depsMap, DependencyTypeKeyPrefix),
		"key":       t.dependencyAcc(depsMap, DependencyTypeKey),
		"ls":        t.dependencyAcc(depsMap, DependencyTypeKeyPrefix),
		"service":   t.dependencyAccVariadic(depsMap, DependencyTypeService),
		"tree":      t.dependencyAcc(depsMap, DependencyTypeKeyPrefix),

		// Helper functions
		"byTag":      t.noop,
		"parseJSON":  t.noop,
		"replaceAll": t.noop,
		"toLower":    t.noop,
		"toTitle":    t.noop,
		"toUpper":    t.noop,
	}).Parse(string(contents))

	if err != nil {
		return err
	}

	err = tmpl.Execute(ioutil.Discard, nil)
	if err != nil {
		return err
	}

	dependencies := make([]util.Dependency, 0, len(depsMap))
	for _, dep := range depsMap {
		dependencies = append(dependencies, dep)
	}
	depsMap = nil

	t.dependencies = dependencies

	return nil
}

// Helper function that is used by the dependency collecting.
func (t *Template) dependencyAcc(depsMap map[string]util.Dependency, dt DependencyType) func(string) (interface{}, error) {
	return func(s string) (interface{}, error) {
		switch dt {
		case DependencyTypeFile:
			d, err := util.ParseFileDependency(s)
			if err != nil {
				return nil, err
			}
			if _, ok := depsMap[d.HashCode()]; !ok {
				depsMap[d.HashCode()] = d
			}

			return "", nil
		case DependencyTypeKeyPrefix:
			d, err := util.ParseKeyPrefixDependency(s)
			if err != nil {
				return nil, err
			}
			if _, ok := depsMap[d.HashCode()]; !ok {
				depsMap[d.HashCode()] = d
			}

			return []*util.KeyPair{}, nil
		case DependencyTypeKey:
			d, err := util.ParseKeyDependency(s)
			if err != nil {
				return nil, err
			}
			if _, ok := depsMap[d.HashCode()]; !ok {
				depsMap[d.HashCode()] = d
			}

			return "", nil
		default:
			return nil, fmt.Errorf("unknown DependencyType %#v", dt)
		}
	}
}

// Helper function that is used by the dependency collecting fot remplate functions that accept variadic arguments.
func (t *Template) dependencyAccVariadic(depsMap map[string]util.Dependency, dt DependencyType) func(...string) (interface{}, error) {
	return func(s ...string) (interface{}, error) {
		switch dt {
		case DependencyTypeService:
			d, err := util.ParseServiceDependency(s...)
			if err != nil {
				return nil, err
			}
			if _, ok := depsMap[d.HashCode()]; !ok {
				depsMap[d.HashCode()] = d
			}

			return []*util.Service{}, nil
		default:
			return nil, fmt.Errorf("unknown variadic DependencyType %#v", dt)
		}
	}
}

// Validates that all required dependencies in t are defined in c.
func (t *Template) validateDependencies(c *TemplateContext) error {
	for _, dep := range t.Dependencies() {
		switch d := dep.(type) {
		case *util.FileDependency:
			if _, ok := c.File[d.Key()]; !ok {
				return fmt.Errorf("templateContext missing file `%s'", d.Key())
			}
		case *util.KeyPrefixDependency:
			if _, ok := c.KeyPrefixes[d.Key()]; !ok {
				return fmt.Errorf("templateContext missing keyPrefix `%s'", d.Key())
			}
		case *util.KeyDependency:
			if _, ok := c.Keys[d.Key()]; !ok {
				return fmt.Errorf("templateContext missing key `%s'", d.Key())
			}
		case *util.ServiceDependency:
			if _, ok := c.Services[d.Key()]; !ok {
				return fmt.Errorf("templateContext missing service `%s'", d.Key())
			}
		default:
			return fmt.Errorf("unknown dependency type in template e %#v", d)
		}
	}

	return nil
}

// noop is a special function that returns itself. This is used during the
// dependency accumulation to allow the template to be processed once.
func (t *Template) noop(thing ...interface{}) (interface{}, error) {
	return thing[len(thing)-1], nil
}

// TemplateContext is what Template uses to determine the values that are
// available for template parsing.
type TemplateContext struct {
	Services    map[string][]*util.Service
	Keys        map[string]string
	KeyPrefixes map[string][]*util.KeyPair
	File        map[string]string
}

// decodeJSON returns a structure for valid JSON
func (c *TemplateContext) decodeJSON(s string) (interface{}, error) {
	var data map[string]interface{}
	if err := json.Unmarshal([]byte(s), &data); err != nil {
		return nil, err
	}

	return data, nil
}

// groupByTag is a template func that takes the provided services and
// produces a map based on Service tags.
//
// The map key is a string representing the service tag. The map value is a
// slice of Services which have the tag assigned.
func (c *TemplateContext) groupByTag(in []*util.Service) map[string][]*util.Service {
	m := make(map[string][]*util.Service)
	for _, s := range in {
		for _, t := range s.Tags {
			m[t] = append(m[t], s)
		}
	}
	return m
}

// toLower converts the given string (usually by a pipe) to lowercase.
func (c *TemplateContext) toLower(s string) (string, error) {
	return strings.ToLower(s), nil
}

// toTitle converts the given string (usually by a pipe) to titlecase.
func (c *TemplateContext) toTitle(s string) (string, error) {
	return strings.Title(s), nil
}

// toUpper converts the given string (usually by a pipe) to uppercase.
func (c *TemplateContext) toUpper(s string) (string, error) {
	return strings.ToUpper(s), nil
}

// replaceAll replaces all occurrences of a value in a string with the given
// replacement value.
func (c *TemplateContext) replaceAll(f, t, s string) (string, error) {
	return strings.Replace(s, f, t, -1), nil
}

// DependencyType is an enum type that says the kind of the dependency.
type DependencyType byte

const (
	DependencyTypeInvalid DependencyType = iota
	DependencyTypeService
	DependencyTypeKey
	DependencyTypeKeyPrefix
	DependencyTypeFile
)
<|endoftext|>


<commit_before>package sms

import (
	"errors"
	"fmt"
	"regexp"
	"sync"
)

var templates = map[string]*SMSTemplate{}
var templatesRWM sync.RWMutex

func RegisterTemplate(id string, t *SMSTemplate) {
	templatesRWM.Lock()
	_, exist := templates[id]
	if exist && t == nil { // 删除
		delete(templates, id)
	} else if t != nil { // 替换或新增
		templates[id] = t
	}
	templatesRWM.Unlock()
}

func FindTemplate(id string) *SMSTemplate {
	templatesRWM.RLock()
	t := templates[id]
	templatesRWM.RUnlock()
	return t
}

type SMSTemplate struct {
	TempID        string
	Temp          string
	NumArgs       int
	valueCheckers []ValueChecker
}

func (t SMSTemplate) SMSContent(args []string) (content string, err error) {
	if t.NumArgs != len(args) {
		err = fmt.Errorf("template need %d args, provide %d", t.NumArgs, len(args))
		return
	}

	for i, v := range args {
		c := t.valueCheckers[i]
		if c == nil {
			continue
		}
		err = c.Check(v)
		if err != nil {
			return
		}
	}

	argTemp := make([]interface{}, len(args))
	for i, a := range args {
		argTemp[i] = a
	}
	content = fmt.Sprintf(t.Temp, argTemp...)
	return
}

type ValueChecker interface {
	Check(v string) error
}

type RegexpChecker struct {
	reg *regexp.Regexp
}

func NewRegexpChecker(expr string) (*RegexpChecker, error) {
	reg, err := regexp.Compile(expr)
	if err != nil {
		return nil, err
	}
	return &RegexpChecker{
		reg: reg,
	}, nil
}

func (rc *RegexpChecker) Check(v string) error {
	matched := rc.reg.MatchString(v)
	if !matched {
		return errors.New("invalid arg:" + v)
	}
	return nil
}
<commit_msg>add int checker<commit_after>package sms

import (
	"errors"
	"fmt"
	"regexp"
	"strconv"
	"sync"
)

var templates = map[string]*SMSTemplate{}
var templatesRWM sync.RWMutex

func RegisterTemplate(id string, t *SMSTemplate) {
	templatesRWM.Lock()
	_, exist := templates[id]
	if exist && t == nil { // 删除
		delete(templates, id)
	} else if t != nil { // 替换或新增
		templates[id] = t
	}
	templatesRWM.Unlock()
}

func FindTemplate(id string) *SMSTemplate {
	templatesRWM.RLock()
	t := templates[id]
	templatesRWM.RUnlock()
	return t
}

type SMSTemplate struct {
	TempID        string
	Temp          string
	NumArgs       int
	valueCheckers []ValueChecker
}

func (t SMSTemplate) SMSContent(args []string) (content string, err error) {
	if t.NumArgs != len(args) {
		err = fmt.Errorf("template need %d args, provide %d", t.NumArgs, len(args))
		return
	}

	for i, v := range args {
		c := t.valueCheckers[i]
		if c == nil {
			continue
		}
		err = c.Check(v)
		if err != nil {
			return
		}
	}

	argTemp := make([]interface{}, len(args))
	for i, a := range args {
		argTemp[i] = a
	}
	content = fmt.Sprintf(t.Temp, argTemp...)
	return
}

type ValueChecker interface {
	Check(v string) error
}

type RegexpChecker struct {
	reg *regexp.Regexp
}

func NewRegexpChecker(expr string) (*RegexpChecker, error) {
	reg, err := regexp.Compile(expr)
	if err != nil {
		return nil, err
	}
	return &RegexpChecker{
		reg: reg,
	}, nil
}

func (rc *RegexpChecker) Check(v string) error {
	matched := rc.reg.MatchString(v)
	if !matched {
		return errors.New("invalid arg:" + v)
	}
	return nil
}

type IntChecker struct{}

func (ic *IntChecker) Check(v string) error {
	_, err := strconv.ParseInt(v, 10, 64)
	return err
}
<|endoftext|>


<commit_before>package iniflags

import (
	"flag"
	"io/ioutil"
	"log"
	"os"
	"path"
	"regexp"
	"strings"
)

type Arg struct {
	key, value string
}

var (
	config = flag.String("config", "dev.ini", "Path to config.")
)

var (
	LINES_REGEXP = regexp.MustCompile("[\\r\\n]")
	KV_REGEXP    = regexp.MustCompile("\\s*=\\s*")
)

func Parse() {
	flag.Parse()
	configPath := *config
	if configPath[0] != '/' {
		configPath = path.Join(path.Dir(os.Args[0]), configPath)
	}
	parsedArgs := getArgsFromConfig(configPath)
	missingFlags := getMissingFlags()
	for _, arg := range parsedArgs {
		if _, found := missingFlags[arg.key]; found {
			flag.Set(arg.key, arg.value)
		}
	}
}

func getArgsFromConfig(configPath string) []Arg {
	file, err := os.Open(configPath)
	if err != nil {
		log.Fatalf("cannot open config file at [%s]: [%s]\n", configPath, err)
	}
	defer file.Close()
	data, err := ioutil.ReadAll(file)
	if err != nil {
		log.Fatalf("Error when reading config file [%s]: [%s]\n", configPath, err)
	}

	var args []Arg
	for _, line := range LINES_REGEXP.Split(string(data), -1) {
		if line == "" || line[0] == ';' || line[0] == '#' || line[0] == '[' {
			continue
		}
		parts := KV_REGEXP.Split(line, 2)
		if len(parts) != 2 {
			log.Fatalf("Cannot split line=[%s] into key and value in config file [%s]", line, configPath)
		}
		key := parts[0]
		value := unquoteValue(parts[1])
		args = append(args, Arg{key: key, value: value})
	}
	return args
}

func getMissingFlags() map[string]bool {
	missingFlags := make(map[string]bool, 0)
	flag.VisitAll(func(f *flag.Flag) {
		missingFlags[f.Name] = true
	})
	flag.Visit(func(f *flag.Flag) {
		delete(missingFlags, f.Name)
	})
	return missingFlags
}

func unquoteValue(v string) string {
	if v[0] != '"' {
		return v
	}
	n := strings.LastIndex(v, "\"")
	if n == -1 {
		return v
	}
	v = v[1:n]
	v = strings.Replace(v, "\\\"", "\"", -1)
	return strings.Replace(v, "\\n", "\n", -1)
}
<commit_msg>handle empty config<commit_after>package iniflags

import (
	"flag"
	"io/ioutil"
	"log"
	"os"
	"path"
	"regexp"
	"strings"
)

type Arg struct {
	key, value string
}

var (
	config = flag.String("config", "dev.ini", "Path to config.")
)

var (
	LINES_REGEXP = regexp.MustCompile("[\\r\\n]")
	KV_REGEXP    = regexp.MustCompile("\\s*=\\s*")
)

func Parse() {
	flag.Parse()
	configPath := *config
	if configPath == "" {
		return
	}
	if configPath[0] != '/' {
		configPath = path.Join(path.Dir(os.Args[0]), configPath)
	}
	parsedArgs := getArgsFromConfig(configPath)
	missingFlags := getMissingFlags()
	for _, arg := range parsedArgs {
		if _, found := missingFlags[arg.key]; found {
			flag.Set(arg.key, arg.value)
		}
	}
}

func getArgsFromConfig(configPath string) []Arg {
	file, err := os.Open(configPath)
	if err != nil {
		log.Fatalf("cannot open config file at [%s]: [%s]\n", configPath, err)
	}
	defer file.Close()
	data, err := ioutil.ReadAll(file)
	if err != nil {
		log.Fatalf("Error when reading config file [%s]: [%s]\n", configPath, err)
	}

	var args []Arg
	for _, line := range LINES_REGEXP.Split(string(data), -1) {
		if line == "" || line[0] == ';' || line[0] == '#' || line[0] == '[' {
			continue
		}
		parts := KV_REGEXP.Split(line, 2)
		if len(parts) != 2 {
			log.Fatalf("Cannot split line=[%s] into key and value in config file [%s]", line, configPath)
		}
		key := parts[0]
		value := unquoteValue(parts[1])
		args = append(args, Arg{key: key, value: value})
	}
	return args
}

func getMissingFlags() map[string]bool {
	missingFlags := make(map[string]bool, 0)
	flag.VisitAll(func(f *flag.Flag) {
		missingFlags[f.Name] = true
	})
	flag.Visit(func(f *flag.Flag) {
		delete(missingFlags, f.Name)
	})
	return missingFlags
}

func unquoteValue(v string) string {
	if v[0] != '"' {
		return v
	}
	n := strings.LastIndex(v, "\"")
	if n == -1 {
		return v
	}
	v = v[1:n]
	v = strings.Replace(v, "\\\"", "\"", -1)
	return strings.Replace(v, "\\n", "\n", -1)
}
<|endoftext|>


<commit_before>package mgostasher

import (
  "context"
  "time"
  "google.golang.org/api/blogger/v3"
  "gopkg.in/mgo.v2"
  "gopkg.in/mgo.v2/bson"
)

const (
  DefaultBlogPageCollection = "blog_page"
  DefaultBlogPostCollection = "blog_post"
  DefaultBlogCollection = "blog"
)

type BlogStasher struct {
  BlogPageCollection *mgo.Collection
  BlogPostCollection *mgo.Collection
  BlogCollection *mgo.Collection
}

type BlogPost struct {
  Id *bson.ObjectId `bson:"_id,omitempty"`
  BlogPost *blogger.Post
  Updated *time.Time
}

type BlogPage struct {
  Id *bson.ObjectId `bson:"_id,omitempty"`
  BlogPage *blogger.Page
  Updated *time.Time
}

type Blog struct {
  Id *bson.ObjectId `bson:"_id,omitempty"`
  Blog *blogger.Blog `bson:",omitempty"`
  Updated *time.Time `bson:",omitempty"`
  PostListEtag *string `bson:",omitempty"`
  PostListUpdated *time.Time `bson:",omitempty"`
  PageListEtag *string `bson:",omitempty"`
  PageListUpdated *time.Time `bson:",omitempty"`
}

func DefaultStasher(db *mgo.Database) (m *BlogStasher) {
  m = new(BlogStasher)
  m.BlogPageCollection = db.C(DefaultBlogPageCollection)
  m.BlogPostCollection = db.C(DefaultBlogPostCollection)
  m.BlogCollection = db.C(DefaultBlogCollection)
  return
}

func(m *BlogStasher) GetPageListEtag(ctx context.Context, blogId string) (etag *string) {
  dbBlog := new(Blog)
  err := m.BlogCollection.Find(bson.M{"blog.id": blogId}).Select(bson.M{"pageListEtag": 1}).One(dbBlog)
  if err != nil {
    if err == mgo.ErrNotFound {
      return nil
    }
    panic(err)
  }
  return dbBlog.PageListEtag
}
func(m *BlogStasher) HasPage(ctx context.Context, pageId string, etag string) bool {
  n, err := m.BlogPageCollection.Find(bson.M{"blogPage.id": pageId, "blogPage.etag": etag}).Count()
  if err != nil { panic(err) }
  if n > 0 { return true }
  return false
}
func WrapBlogPage(page *blogger.Page) *BlogPage {
  dbPage := &BlogPage{
    BlogPage: page,
  }
  pageUpdated, err := time.Parse(time.RFC3339, page.Updated)
  if err != nil { panic(err) }
  dbPage.Updated = &pageUpdated
  return dbPage
}
func(m *BlogStasher) StashPage(ctx context.Context, page *blogger.Page) {
  dbPage := WrapBlogPage(page)
  _, err := m.BlogPageCollection.Upsert(bson.M{"blogPage.id": page.Id, "updated": bson.M{"$lt": dbPage.Updated}}, &dbPage);
  if err != nil { panic(err) }
}
func(m *BlogStasher) StashPageEtags(ctx context.Context, blogId string, listEtag string, pageEtags map[string]string, newUpdated *time.Time) {
  iter := m.BlogPageCollection.Find(nil).Select(bson.M{"dbPage.id": 1, "dbPage.etag": 1}).Select(bson.M{"_id": 1}).Iter()
  defer iter.Close()
  dbPage := new(BlogPage)
  for iter.Next(dbPage) {
    if pageEtags[dbPage.BlogPage.Id] != dbPage.BlogPage.Etag {
      if err := m.BlogPageCollection.RemoveId(dbPage.Id); err != nil {
        panic(err)
      }
    }
  }
  if err := iter.Err(); err != nil {
    panic(err)
  }

  _, err := m.BlogCollection.Upsert(bson.M{"blog.id": blogId, "pageListUpdated": bson.M{"$lt": newUpdated}}, bson.M{"$set": &Blog{PageListEtag: &listEtag, PageListUpdated: newUpdated}})
  if (err != nil) { panic(err) }
}

func(m *BlogStasher) GetPostListEtag(ctx context.Context, blogId string) (etag *string) {
  dbBlog := new(Blog)
  err := m.BlogCollection.Find(bson.M{"blog.id": blogId}).Select(bson.M{"postListEtag": 1}).One(dbBlog)
  if err != nil {
    if err == mgo.ErrNotFound {
      return nil
    }
    panic(err)
  }
  return dbBlog.PostListEtag
}
func(m *BlogStasher) HasPost(ctx context.Context, postId string, etag string) bool {
  n, err := m.BlogPostCollection.Find(bson.M{"blogPost.id": postId, "blogPost.etag": etag}).Count()
  if err != nil { panic(err) }
  if n > 0 { return true }
  return false
}

func WrapBlogPost(post *blogger.Post) *BlogPost {
  dbPost := &BlogPost{
    BlogPost: post,
  }
  postUpdated, err := time.Parse(time.RFC3339, post.Updated)
  if err != nil { panic(err) }
  dbPost.Updated = &postUpdated
  return dbPost
}
func(m *BlogStasher) StashPost(ctx context.Context, post *blogger.Post) {
  dbPost := WrapBlogPost(post)
  _, err := m.BlogPostCollection.Upsert(bson.M{"blogPost.id": post.Id, "updated": bson.M{"$lt": dbPost.Updated}}, dbPost);
  if err != nil { panic(err) }
}
func(m *BlogStasher) StashPostEtags(ctx context.Context, blogId string, listEtag string, postEtags map[string]string, newUpdated *time.Time) {
  iter := m.BlogPostCollection.Find(nil).Select(bson.M{"dbPost.id": 1, "dbPost.etag": 1}).Select(bson.M{"_id": 1}).Iter()
  defer iter.Close()
  dbPost := new(BlogPost)
  for iter.Next(dbPost) {
    if postEtags[dbPost.BlogPost.Id] != dbPost.BlogPost.Etag {
      if err := m.BlogPostCollection.RemoveId(dbPost.Id); err != nil {
        panic(err)
      }
    }
  }
  if err := iter.Err(); err != nil {
    panic(err)
  }

  _, err := m.BlogCollection.Upsert(bson.M{"blog.id": blogId, "postListUpdated": bson.M{"$lt": newUpdated}}, bson.M{"$set": &Blog{PostListEtag: &listEtag, PostListUpdated: newUpdated}})
  if (err != nil) { panic(err) }
}

func(m *BlogStasher) StashBlog(ctx context.Context, blog *blogger.Blog) {
  dbBlog := Blog{
    Blog: blog,
  }
  var err error
  *dbBlog.Updated, err = time.Parse(time.RFC3339, blog.Updated)
  if err != nil { panic(err) }
  _, err = m.BlogCollection.Upsert(bson.M{"blog.id": blog.Id, "updated": bson.M{"$lt": dbBlog.Updated}}, bson.M{"$set": &dbBlog})
  if (err != nil) { panic(err) }
}
<commit_msg>API for stashing tweets in mgo<commit_after>package mgostasher

import (
  "context"
  "time"
  "google.golang.org/api/blogger/v3"
  "github.com/dghubble/go-twitter/twitter"
  "gopkg.in/mgo.v2"
  "gopkg.in/mgo.v2/bson"
)

const (
  DefaultBlogPageCollection = "blog_page"
  DefaultBlogPostCollection = "blog_post"
  DefaultBlogCollection = "blog"
  DefaultTweetCollection = "tweet"
  DefaultTwitterUserCollection = "twitterer"
)

type BlogStasher struct {
  BlogPageCollection *mgo.Collection
  BlogPostCollection *mgo.Collection
  BlogCollection *mgo.Collection
}

type TwitterStasher struct {
  TweetCollection *mgo.Collection
  TwitterUserCollection *mgo.Collection
}

type BlogPost struct {
  Id *bson.ObjectId `bson:"_id,omitempty"`
  BlogPost *blogger.Post
  Updated *time.Time
}

type BlogPage struct {
  Id *bson.ObjectId `bson:"_id,omitempty"`
  BlogPage *blogger.Page
  Updated *time.Time
}

type Blog struct {
  Id *bson.ObjectId `bson:"_id,omitempty"`
  Blog *blogger.Blog `bson:",omitempty"`
  Updated *time.Time `bson:",omitempty"`
  PostListEtag *string `bson:",omitempty"`
  PostListUpdated *time.Time `bson:",omitempty"`
  PageListEtag *string `bson:",omitempty"`
  PageListUpdated *time.Time `bson:",omitempty"`
}

type Tweet struct {
  Id *bson.ObjectId `bson:"_id,omitempty"`
  Tweet *twitter.Tweet
  OEmbed *twitter.OEmbedTweet
}

type TwitterUser struct {
  Id *bson.ObjectId `bson:"_id,omitempty"`
  TwitterUser *twitter.User
}

func DefaultBlogStasher(db *mgo.Database) (m *BlogStasher) {
  m = new(BlogStasher)
  m.BlogPageCollection = db.C(DefaultBlogPageCollection)
  m.BlogPostCollection = db.C(DefaultBlogPostCollection)
  m.BlogCollection = db.C(DefaultBlogCollection)
  return
}

func DefaultTwitterStasher(db *mgo.Database) (m *TwitterStasher) {
  m = new(TwitterStasher)
  m.TweetCollection = db.C(DefaultTweetCollection)
  m.TwitterUserCollection = db.C(DefaultTwitterUserCollection)
  return
}

func(m *BlogStasher) GetPageListEtag(ctx context.Context, blogId string) (etag *string) {
  dbBlog := new(Blog)
  err := m.BlogCollection.Find(bson.M{"blog.id": blogId}).Select(bson.M{"pageListEtag": 1}).One(dbBlog)
  if err != nil {
    if err == mgo.ErrNotFound {
      return nil
    }
    panic(err)
  }
  return dbBlog.PageListEtag
}
func(m *BlogStasher) HasPage(ctx context.Context, pageId string, etag string) bool {
  n, err := m.BlogPageCollection.Find(bson.M{"blogPage.id": pageId, "blogPage.etag": etag}).Count()
  if err != nil { panic(err) }
  if n > 0 { return true }
  return false
}
func WrapBlogPage(page *blogger.Page) *BlogPage {
  dbPage := &BlogPage{
    BlogPage: page,
  }
  pageUpdated, err := time.Parse(time.RFC3339, page.Updated)
  if err != nil { panic(err) }
  dbPage.Updated = &pageUpdated
  return dbPage
}
func(m *BlogStasher) StashPage(ctx context.Context, page *blogger.Page) {
  dbPage := WrapBlogPage(page)
  _, err := m.BlogPageCollection.Upsert(bson.M{"blogPage.id": page.Id, "updated": bson.M{"$lt": dbPage.Updated}}, &dbPage);
  if err != nil { panic(err) }
}
func(m *BlogStasher) StashPageEtags(ctx context.Context, blogId string, listEtag string, pageEtags map[string]string, newUpdated *time.Time) {
  iter := m.BlogPageCollection.Find(nil).Select(bson.M{"dbPage.id": 1, "dbPage.etag": 1}).Select(bson.M{"_id": 1}).Iter()
  defer iter.Close()
  dbPage := new(BlogPage)
  for iter.Next(dbPage) {
    if pageEtags[dbPage.BlogPage.Id] != dbPage.BlogPage.Etag {
      if err := m.BlogPageCollection.RemoveId(dbPage.Id); err != nil {
        panic(err)
      }
    }
  }
  if err := iter.Err(); err != nil {
    panic(err)
  }

  _, err := m.BlogCollection.Upsert(bson.M{"blog.id": blogId, "pageListUpdated": bson.M{"$lt": newUpdated}}, bson.M{"$set": &Blog{PageListEtag: &listEtag, PageListUpdated: newUpdated}})
  if (err != nil) { panic(err) }
}

func(m *BlogStasher) GetPostListEtag(ctx context.Context, blogId string) (etag *string) {
  dbBlog := new(Blog)
  err := m.BlogCollection.Find(bson.M{"blog.id": blogId}).Select(bson.M{"postListEtag": 1}).One(dbBlog)
  if err != nil {
    if err == mgo.ErrNotFound {
      return nil
    }
    panic(err)
  }
  return dbBlog.PostListEtag
}
func(m *BlogStasher) HasPost(ctx context.Context, postId string, etag string) bool {
  n, err := m.BlogPostCollection.Find(bson.M{"blogPost.id": postId, "blogPost.etag": etag}).Count()
  if err != nil { panic(err) }
  if n > 0 { return true }
  return false
}

func WrapBlogPost(post *blogger.Post) *BlogPost {
  dbPost := &BlogPost{
    BlogPost: post,
  }
  postUpdated, err := time.Parse(time.RFC3339, post.Updated)
  if err != nil { panic(err) }
  dbPost.Updated = &postUpdated
  return dbPost
}
func(m *BlogStasher) StashPost(ctx context.Context, post *blogger.Post) {
  dbPost := WrapBlogPost(post)
  _, err := m.BlogPostCollection.Upsert(bson.M{"blogPost.id": post.Id, "updated": bson.M{"$lt": dbPost.Updated}}, dbPost);
  if err != nil { panic(err) }
}
func(m *BlogStasher) StashPostEtags(ctx context.Context, blogId string, listEtag string, postEtags map[string]string, newUpdated *time.Time) {
  iter := m.BlogPostCollection.Find(nil).Select(bson.M{"dbPost.id": 1, "dbPost.etag": 1}).Select(bson.M{"_id": 1}).Iter()
  defer iter.Close()
  dbPost := new(BlogPost)
  for iter.Next(dbPost) {
    if postEtags[dbPost.BlogPost.Id] != dbPost.BlogPost.Etag {
      if err := m.BlogPostCollection.RemoveId(dbPost.Id); err != nil {
        panic(err)
      }
    }
  }
  if err := iter.Err(); err != nil {
    panic(err)
  }

  _, err := m.BlogCollection.Upsert(bson.M{"blog.id": blogId, "postListUpdated": bson.M{"$lt": newUpdated}}, bson.M{"$set": &Blog{PostListEtag: &listEtag, PostListUpdated: newUpdated}})
  if (err != nil) { panic(err) }
}

func(m *BlogStasher) StashBlog(ctx context.Context, blog *blogger.Blog) {
  dbBlog := Blog{
    Blog: blog,
  }
  var err error
  *dbBlog.Updated, err = time.Parse(time.RFC3339, blog.Updated)
  if err != nil { panic(err) }
  _, err = m.BlogCollection.Upsert(bson.M{"blog.id": blog.Id, "updated": bson.M{"$lt": dbBlog.Updated}}, bson.M{"$set": &dbBlog})
  if (err != nil) { panic(err) }
}

func(m *TwitterStasher) GetLastTweetId(ctx context.Context, userId int64) int64 {
  tweet := new(Tweet)
  err := m.TweetCollection.Find(bson.M{"tweet.user.id": userId}).Select(bson.M{"tweet.Id": 1}).One(tweet)
  if err != nil {
    if err == mgo.ErrNotFound {
      return 0
    }
    panic(err)
  }
  return tweet.Tweet.ID
}

func(m *TwitterStasher) StashUser(ctx context.Context, user *twitter.User) {
  dbUser := &TwitterUser{TwitterUser: user}
  _, err := m.TwitterUserCollection.Upsert(bson.M{"user.id": user.ID}, &dbUser);
  if err != nil { panic(err) }
}

func(m *TwitterStasher) StashTweet(ctx context.Context, tweet *twitter.Tweet, oembed *twitter.OEmbedTweet) {
  dbTweet := &Tweet{Tweet: tweet, OEmbed: oembed}
  _, err := m.TweetCollection.Upsert(bson.M{"tweet.id": tweet.ID}, &dbTweet);
  if err != nil { panic(err) }
}
<|endoftext|>


<commit_before>package acsengine

import (
	"bytes"
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"errors"
	"fmt"
	"math/big"
	"net"
	"time"

	log "github.com/sirupsen/logrus"
)

const (
	// ValidityDuration specifies the duration an TLS certificate is valid
	ValidityDuration = time.Hour * 24 * 365 * 2
	// PkiKeySize is the size in bytes of the PKI key
	PkiKeySize = 4096
)

// PkiKeyCertPair represents an PKI public and private cert pair
type PkiKeyCertPair struct {
	CertificatePem string
	PrivateKeyPem  string
}

// CreatePki creates PKI certificates
func CreatePki(extraFQDNs []string, extraIPs []net.IP, clusterDomain string, caPair *PkiKeyCertPair, masterCount int) (*PkiKeyCertPair, *PkiKeyCertPair, *PkiKeyCertPair, *PkiKeyCertPair, *PkiKeyCertPair, []*PkiKeyCertPair, error) {
	start := time.Now()
	defer func(s time.Time) {
		log.Debugf("pki: PKI asset creation took %s", time.Since(s))
	}(start)
	extraFQDNs = append(extraFQDNs, fmt.Sprintf("kubernetes"))
	extraFQDNs = append(extraFQDNs, fmt.Sprintf("kubernetes.default"))
	extraFQDNs = append(extraFQDNs, fmt.Sprintf("kubernetes.default.svc"))
	extraFQDNs = append(extraFQDNs, fmt.Sprintf("kubernetes.default.svc.%s", clusterDomain))
	extraFQDNs = append(extraFQDNs, fmt.Sprintf("kubernetes.kube-system"))
	extraFQDNs = append(extraFQDNs, fmt.Sprintf("kubernetes.kube-system.svc"))
	extraFQDNs = append(extraFQDNs, fmt.Sprintf("kubernetes.kube-system.svc.%s", clusterDomain))

	var (
		caCertificate         *x509.Certificate
		caPrivateKey          *rsa.PrivateKey
		apiServerCertificate  *x509.Certificate
		apiServerPrivateKey   *rsa.PrivateKey
		clientCertificate     *x509.Certificate
		clientPrivateKey      *rsa.PrivateKey
		kubeConfigCertificate *x509.Certificate
		kubeConfigPrivateKey  *rsa.PrivateKey
		etcdServerCertificate *x509.Certificate
		etcdServerPrivateKey  *rsa.PrivateKey
		etcdClientCertificate *x509.Certificate
		etcdClientPrivateKey  *rsa.PrivateKey
		etcdPeerCertPairs     []*PkiKeyCertPair
	)
	errors := make(chan error)

	var err error
	caCertificate, err = pemToCertificate(caPair.CertificatePem)
	if err != nil {
		return nil, nil, nil, nil, nil, nil, err
	}
	caPrivateKey, err = pemToKey(caPair.PrivateKeyPem)
	if err != nil {
		return nil, nil, nil, nil, nil, nil, err
	}

	go func() {
		var err error
		apiServerCertificate, apiServerPrivateKey, err = createCertificate("apiserver", caCertificate, caPrivateKey, false, true, extraFQDNs, extraIPs, nil)
		errors <- err
	}()

	go func() {
		var err error
		organization := make([]string, 1)
		organization[0] = "system:masters"
		clientCertificate, clientPrivateKey, err = createCertificate("client", caCertificate, caPrivateKey, false, false, nil, nil, organization)
		errors <- err
	}()

	go func() {
		var err error
		organization := make([]string, 1)
		organization[0] = "system:masters"
		kubeConfigCertificate, kubeConfigPrivateKey, err = createCertificate("client", caCertificate, caPrivateKey, false, false, nil, nil, organization)
		errors <- err
	}()

	go func() {
		var err error
		var organization []string
		ip := net.ParseIP("127.0.0.1").To4()
		peerIPs := append(extraIPs, ip)
		etcdServerCertificate, etcdServerPrivateKey, err = createCertificate("etcdserver", caCertificate, caPrivateKey, true, true, nil, peerIPs, organization)
		errors <- err
	}()

	go func() {
		var err error
		var organization []string
		ip := net.ParseIP("127.0.0.1").To4()
		peerIPs := append(extraIPs, ip)
		etcdClientCertificate, etcdClientPrivateKey, err = createCertificate("etcdclient", caCertificate, caPrivateKey, true, false, nil, peerIPs, organization)
		errors <- err
	}()

	etcdPeerCertPairs = make([]*PkiKeyCertPair, masterCount)
	for i := 0; i < masterCount; i++ {
		go func(i int) {
			var err error
			var organization []string
			ip := net.ParseIP("127.0.0.1").To4()
			peerIPs := append(extraIPs, ip)
			etcdPeerCertificate := new(x509.Certificate)
			etcdPeerPrivateKey := new(rsa.PrivateKey)
			etcdPeerCertificate, etcdPeerPrivateKey, err = createCertificate("etcdpeer", caCertificate, caPrivateKey, true, false, nil, peerIPs, organization)
			etcdPeerCertPairs[i] = &PkiKeyCertPair{CertificatePem: string(certificateToPem(etcdPeerCertificate.Raw)), PrivateKeyPem: string(privateKeyToPem(etcdPeerPrivateKey))}
			errors <- err
		}(i)
	}

	e := make([]error, (masterCount + 5))
	for i := 0; i < len(e); i++ {
		e[i] = <-errors
		if e[i] != nil {
			return nil, nil, nil, nil, nil, nil, e[i]
		}
	}

	return &PkiKeyCertPair{CertificatePem: string(certificateToPem(apiServerCertificate.Raw)), PrivateKeyPem: string(privateKeyToPem(apiServerPrivateKey))},
		&PkiKeyCertPair{CertificatePem: string(certificateToPem(clientCertificate.Raw)), PrivateKeyPem: string(privateKeyToPem(clientPrivateKey))},
		&PkiKeyCertPair{CertificatePem: string(certificateToPem(kubeConfigCertificate.Raw)), PrivateKeyPem: string(privateKeyToPem(kubeConfigPrivateKey))},
		&PkiKeyCertPair{CertificatePem: string(certificateToPem(etcdServerCertificate.Raw)), PrivateKeyPem: string(privateKeyToPem(etcdServerPrivateKey))},
		&PkiKeyCertPair{CertificatePem: string(certificateToPem(etcdClientCertificate.Raw)), PrivateKeyPem: string(privateKeyToPem(etcdClientPrivateKey))},
		etcdPeerCertPairs,
		nil
}

func createCertificate(commonName string, caCertificate *x509.Certificate, caPrivateKey *rsa.PrivateKey, isEtcd bool, isServer bool, extraFQDNs []string, extraIPs []net.IP, organization []string) (*x509.Certificate, *rsa.PrivateKey, error) {
	var err error

	isCA := (caCertificate == nil)

	now := time.Now()

	template := x509.Certificate{
		Subject:   pkix.Name{CommonName: commonName},
		NotBefore: now,
		NotAfter:  now.Add(ValidityDuration),

		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		BasicConstraintsValid: true,
	}

	if organization != nil {
		template.Subject.Organization = organization
	}

	if isCA {
		template.KeyUsage |= x509.KeyUsageCertSign
		template.IsCA = isCA
	} else if isEtcd {
		if commonName == "etcdServer" {
			template.IPAddresses = extraIPs
			template.ExtKeyUsage = append(template.ExtKeyUsage, x509.ExtKeyUsageServerAuth)
		} else if commonName == "etcdClient" {
			template.IPAddresses = extraIPs
			template.ExtKeyUsage = append(template.ExtKeyUsage, x509.ExtKeyUsageClientAuth)
		} else {
			template.IPAddresses = extraIPs
			template.ExtKeyUsage = append(template.ExtKeyUsage, x509.ExtKeyUsageServerAuth)
			template.ExtKeyUsage = append(template.ExtKeyUsage, x509.ExtKeyUsageClientAuth)
		}
	} else if isServer {
		template.DNSNames = extraFQDNs
		template.IPAddresses = extraIPs
		template.ExtKeyUsage = append(template.ExtKeyUsage, x509.ExtKeyUsageServerAuth)
	} else {
		template.ExtKeyUsage = append(template.ExtKeyUsage, x509.ExtKeyUsageClientAuth)
	}

	snMax := new(big.Int).Lsh(big.NewInt(1), 128)
	template.SerialNumber, err = rand.Int(rand.Reader, snMax)
	if err != nil {
		return nil, nil, err
	}

	privateKey, _ := rsa.GenerateKey(rand.Reader, PkiKeySize)

	var privateKeyToUse *rsa.PrivateKey
	var certificateToUse *x509.Certificate
	if !isCA {
		privateKeyToUse = caPrivateKey
		certificateToUse = caCertificate
	} else {
		privateKeyToUse = privateKey
		certificateToUse = &template
	}

	certDerBytes, err := x509.CreateCertificate(rand.Reader, &template, certificateToUse, &privateKey.PublicKey, privateKeyToUse)
	if err != nil {
		return nil, nil, err
	}

	certificate, err := x509.ParseCertificate(certDerBytes)
	if err != nil {
		return nil, nil, err
	}

	return certificate, privateKey, nil
}

func certificateToPem(derBytes []byte) []byte {
	pemBlock := &pem.Block{
		Type:  "CERTIFICATE",
		Bytes: derBytes,
	}
	pemBuffer := bytes.Buffer{}
	pem.Encode(&pemBuffer, pemBlock)

	return pemBuffer.Bytes()
}

func privateKeyToPem(privateKey *rsa.PrivateKey) []byte {
	pemBlock := &pem.Block{
		Type:  "RSA PRIVATE KEY",
		Bytes: x509.MarshalPKCS1PrivateKey(privateKey),
	}
	pemBuffer := bytes.Buffer{}
	pem.Encode(&pemBuffer, pemBlock)

	return pemBuffer.Bytes()
}

func pemToCertificate(raw string) (*x509.Certificate, error) {
	cpb, _ := pem.Decode([]byte(raw))
	if cpb == nil {
		return nil, errors.New("The raw pem is not a valid PEM formatted block")
	}
	return x509.ParseCertificate(cpb.Bytes)
}

func pemToKey(raw string) (*rsa.PrivateKey, error) {
	kpb, _ := pem.Decode([]byte(raw))
	if kpb == nil {
		return nil, errors.New("The raw pem is not a valid PEM formatted block")
	}
	return x509.ParsePKCS1PrivateKey(kpb.Bytes)
}
<commit_msg>Etcd certs OU (#2005)<commit_after>package acsengine

import (
	"bytes"
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"errors"
	"fmt"
	"math/big"
	"net"
	"time"

	log "github.com/sirupsen/logrus"
)

const (
	// ValidityDuration specifies the duration an TLS certificate is valid
	ValidityDuration = time.Hour * 24 * 365 * 2
	// PkiKeySize is the size in bytes of the PKI key
	PkiKeySize = 4096
)

// PkiKeyCertPair represents an PKI public and private cert pair
type PkiKeyCertPair struct {
	CertificatePem string
	PrivateKeyPem  string
}

// CreatePki creates PKI certificates
func CreatePki(extraFQDNs []string, extraIPs []net.IP, clusterDomain string, caPair *PkiKeyCertPair, masterCount int) (*PkiKeyCertPair, *PkiKeyCertPair, *PkiKeyCertPair, *PkiKeyCertPair, *PkiKeyCertPair, []*PkiKeyCertPair, error) {
	start := time.Now()
	defer func(s time.Time) {
		log.Debugf("pki: PKI asset creation took %s", time.Since(s))
	}(start)
	extraFQDNs = append(extraFQDNs, fmt.Sprintf("kubernetes"))
	extraFQDNs = append(extraFQDNs, fmt.Sprintf("kubernetes.default"))
	extraFQDNs = append(extraFQDNs, fmt.Sprintf("kubernetes.default.svc"))
	extraFQDNs = append(extraFQDNs, fmt.Sprintf("kubernetes.default.svc.%s", clusterDomain))
	extraFQDNs = append(extraFQDNs, fmt.Sprintf("kubernetes.kube-system"))
	extraFQDNs = append(extraFQDNs, fmt.Sprintf("kubernetes.kube-system.svc"))
	extraFQDNs = append(extraFQDNs, fmt.Sprintf("kubernetes.kube-system.svc.%s", clusterDomain))

	var (
		caCertificate         *x509.Certificate
		caPrivateKey          *rsa.PrivateKey
		apiServerCertificate  *x509.Certificate
		apiServerPrivateKey   *rsa.PrivateKey
		clientCertificate     *x509.Certificate
		clientPrivateKey      *rsa.PrivateKey
		kubeConfigCertificate *x509.Certificate
		kubeConfigPrivateKey  *rsa.PrivateKey
		etcdServerCertificate *x509.Certificate
		etcdServerPrivateKey  *rsa.PrivateKey
		etcdClientCertificate *x509.Certificate
		etcdClientPrivateKey  *rsa.PrivateKey
		etcdPeerCertPairs     []*PkiKeyCertPair
	)
	errors := make(chan error)

	var err error
	caCertificate, err = pemToCertificate(caPair.CertificatePem)
	if err != nil {
		return nil, nil, nil, nil, nil, nil, err
	}
	caPrivateKey, err = pemToKey(caPair.PrivateKeyPem)
	if err != nil {
		return nil, nil, nil, nil, nil, nil, err
	}

	go func() {
		var err error
		apiServerCertificate, apiServerPrivateKey, err = createCertificate("apiserver", caCertificate, caPrivateKey, false, true, extraFQDNs, extraIPs, nil)
		errors <- err
	}()

	go func() {
		var err error
		organization := make([]string, 1)
		organization[0] = "system:masters"
		clientCertificate, clientPrivateKey, err = createCertificate("client", caCertificate, caPrivateKey, false, false, nil, nil, organization)
		errors <- err
	}()

	go func() {
		var err error
		organization := make([]string, 1)
		organization[0] = "system:masters"
		kubeConfigCertificate, kubeConfigPrivateKey, err = createCertificate("client", caCertificate, caPrivateKey, false, false, nil, nil, organization)
		errors <- err
	}()

	go func() {
		var err error
		ip := net.ParseIP("127.0.0.1").To4()
		peerIPs := append(extraIPs, ip)
		etcdServerCertificate, etcdServerPrivateKey, err = createCertificate("etcdserver", caCertificate, caPrivateKey, true, true, nil, peerIPs, nil)
		errors <- err
	}()

	go func() {
		var err error
		ip := net.ParseIP("127.0.0.1").To4()
		peerIPs := append(extraIPs, ip)
		etcdClientCertificate, etcdClientPrivateKey, err = createCertificate("etcdclient", caCertificate, caPrivateKey, true, false, nil, peerIPs, nil)
		errors <- err
	}()

	etcdPeerCertPairs = make([]*PkiKeyCertPair, masterCount)
	for i := 0; i < masterCount; i++ {
		go func(i int) {
			var err error
			ip := net.ParseIP("127.0.0.1").To4()
			peerIPs := append(extraIPs, ip)
			etcdPeerCertificate := new(x509.Certificate)
			etcdPeerPrivateKey := new(rsa.PrivateKey)
			etcdPeerCertificate, etcdPeerPrivateKey, err = createCertificate("etcdpeer", caCertificate, caPrivateKey, true, false, nil, peerIPs, nil)
			etcdPeerCertPairs[i] = &PkiKeyCertPair{CertificatePem: string(certificateToPem(etcdPeerCertificate.Raw)), PrivateKeyPem: string(privateKeyToPem(etcdPeerPrivateKey))}
			errors <- err
		}(i)
	}

	e := make([]error, (masterCount + 5))
	for i := 0; i < len(e); i++ {
		e[i] = <-errors
		if e[i] != nil {
			return nil, nil, nil, nil, nil, nil, e[i]
		}
	}

	return &PkiKeyCertPair{CertificatePem: string(certificateToPem(apiServerCertificate.Raw)), PrivateKeyPem: string(privateKeyToPem(apiServerPrivateKey))},
		&PkiKeyCertPair{CertificatePem: string(certificateToPem(clientCertificate.Raw)), PrivateKeyPem: string(privateKeyToPem(clientPrivateKey))},
		&PkiKeyCertPair{CertificatePem: string(certificateToPem(kubeConfigCertificate.Raw)), PrivateKeyPem: string(privateKeyToPem(kubeConfigPrivateKey))},
		&PkiKeyCertPair{CertificatePem: string(certificateToPem(etcdServerCertificate.Raw)), PrivateKeyPem: string(privateKeyToPem(etcdServerPrivateKey))},
		&PkiKeyCertPair{CertificatePem: string(certificateToPem(etcdClientCertificate.Raw)), PrivateKeyPem: string(privateKeyToPem(etcdClientPrivateKey))},
		etcdPeerCertPairs,
		nil
}

func createCertificate(commonName string, caCertificate *x509.Certificate, caPrivateKey *rsa.PrivateKey, isEtcd bool, isServer bool, extraFQDNs []string, extraIPs []net.IP, organization []string) (*x509.Certificate, *rsa.PrivateKey, error) {
	var err error

	isCA := (caCertificate == nil)

	now := time.Now()

	template := x509.Certificate{
		Subject:   pkix.Name{CommonName: commonName},
		NotBefore: now,
		NotAfter:  now.Add(ValidityDuration),

		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		BasicConstraintsValid: true,
	}

	if organization != nil {
		template.Subject.Organization = organization
	}

	if isCA {
		template.KeyUsage |= x509.KeyUsageCertSign
		template.IsCA = isCA
	} else if isEtcd {
		if commonName == "etcdServer" {
			template.IPAddresses = extraIPs
			template.ExtKeyUsage = append(template.ExtKeyUsage, x509.ExtKeyUsageServerAuth)
		} else if commonName == "etcdClient" {
			template.IPAddresses = extraIPs
			template.ExtKeyUsage = append(template.ExtKeyUsage, x509.ExtKeyUsageClientAuth)
		} else {
			template.IPAddresses = extraIPs
			template.ExtKeyUsage = append(template.ExtKeyUsage, x509.ExtKeyUsageServerAuth)
			template.ExtKeyUsage = append(template.ExtKeyUsage, x509.ExtKeyUsageClientAuth)
		}
	} else if isServer {
		template.DNSNames = extraFQDNs
		template.IPAddresses = extraIPs
		template.ExtKeyUsage = append(template.ExtKeyUsage, x509.ExtKeyUsageServerAuth)
	} else {
		template.ExtKeyUsage = append(template.ExtKeyUsage, x509.ExtKeyUsageClientAuth)
	}

	snMax := new(big.Int).Lsh(big.NewInt(1), 128)
	template.SerialNumber, err = rand.Int(rand.Reader, snMax)
	if err != nil {
		return nil, nil, err
	}

	privateKey, _ := rsa.GenerateKey(rand.Reader, PkiKeySize)

	var privateKeyToUse *rsa.PrivateKey
	var certificateToUse *x509.Certificate
	if !isCA {
		privateKeyToUse = caPrivateKey
		certificateToUse = caCertificate
	} else {
		privateKeyToUse = privateKey
		certificateToUse = &template
	}

	certDerBytes, err := x509.CreateCertificate(rand.Reader, &template, certificateToUse, &privateKey.PublicKey, privateKeyToUse)
	if err != nil {
		return nil, nil, err
	}

	certificate, err := x509.ParseCertificate(certDerBytes)
	if err != nil {
		return nil, nil, err
	}

	return certificate, privateKey, nil
}

func certificateToPem(derBytes []byte) []byte {
	pemBlock := &pem.Block{
		Type:  "CERTIFICATE",
		Bytes: derBytes,
	}
	pemBuffer := bytes.Buffer{}
	pem.Encode(&pemBuffer, pemBlock)

	return pemBuffer.Bytes()
}

func privateKeyToPem(privateKey *rsa.PrivateKey) []byte {
	pemBlock := &pem.Block{
		Type:  "RSA PRIVATE KEY",
		Bytes: x509.MarshalPKCS1PrivateKey(privateKey),
	}
	pemBuffer := bytes.Buffer{}
	pem.Encode(&pemBuffer, pemBlock)

	return pemBuffer.Bytes()
}

func pemToCertificate(raw string) (*x509.Certificate, error) {
	cpb, _ := pem.Decode([]byte(raw))
	if cpb == nil {
		return nil, errors.New("The raw pem is not a valid PEM formatted block")
	}
	return x509.ParseCertificate(cpb.Bytes)
}

func pemToKey(raw string) (*rsa.PrivateKey, error) {
	kpb, _ := pem.Decode([]byte(raw))
	if kpb == nil {
		return nil, errors.New("The raw pem is not a valid PEM formatted block")
	}
	return x509.ParsePKCS1PrivateKey(kpb.Bytes)
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package audit

import (
	"bytes"
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/golang/glog"
	"github.com/pborman/uuid"

	"reflect"

	"k8s.io/apimachinery/pkg/apis/meta/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/types"
	utilnet "k8s.io/apimachinery/pkg/util/net"
	"k8s.io/apiserver/pkg/apis/audit"
	auditinternal "k8s.io/apiserver/pkg/apis/audit"
	"k8s.io/apiserver/pkg/authorization/authorizer"
	authenticationv1 "k8s.io/client-go/pkg/apis/authentication/v1"
)

func NewEventFromRequest(req *http.Request, level auditinternal.Level, attribs authorizer.Attributes) (*auditinternal.Event, error) {
	ev := &auditinternal.Event{
		Timestamp:  metav1.NewTime(time.Now()),
		Verb:       attribs.GetVerb(),
		RequestURI: req.URL.RequestURI(),
	}

	ev.Level = level

	// prefer the id from the headers. If not available, create a new one.
	// TODO(audit): do we want to forbid the header for non-front-proxy users?
	ids := req.Header[auditinternal.HeaderAuditID]
	if len(ids) > 0 {
		ev.AuditID = types.UID(ids[0])
	} else {
		ev.AuditID = types.UID(uuid.NewRandom().String())
	}

	ips := utilnet.SourceIPs(req)
	ev.SourceIPs = make([]string, len(ips))
	for i := range ips {
		ev.SourceIPs[i] = ips[i].String()
	}

	if user := attribs.GetUser(); user != nil {
		ev.User.Username = user.GetName()
		ev.User.Extra = map[string]auditinternal.ExtraValue{}
		for k, v := range user.GetExtra() {
			ev.User.Extra[k] = auditinternal.ExtraValue(v)
		}
		ev.User.Groups = user.GetGroups()
		ev.User.UID = user.GetUID()
	}

	if asuser := req.Header.Get(authenticationv1.ImpersonateUserHeader); len(asuser) > 0 {
		ev.ImpersonatedUser = &auditinternal.UserInfo{
			Username: asuser,
		}
		if requestedGroups := req.Header[authenticationv1.ImpersonateGroupHeader]; len(requestedGroups) > 0 {
			ev.ImpersonatedUser.Groups = requestedGroups
		}

		ev.ImpersonatedUser.Extra = map[string]auditinternal.ExtraValue{}
		for k, v := range req.Header {
			if !strings.HasPrefix(k, authenticationv1.ImpersonateUserExtraHeaderPrefix) {
				continue
			}
			k = k[len(authenticationv1.ImpersonateUserExtraHeaderPrefix):]
			ev.ImpersonatedUser.Extra[k] = auditinternal.ExtraValue(v)
		}
	}

	if attribs.IsResourceRequest() {
		ev.ObjectRef = &auditinternal.ObjectReference{
			Namespace:  attribs.GetNamespace(),
			Name:       attribs.GetName(),
			Resource:   attribs.GetResource(),
			APIVersion: attribs.GetAPIGroup() + "/" + attribs.GetAPIVersion(),
		}
	}

	return ev, nil
}

// LogRequestObject fills in the request object into an audit event. The passed runtime.Object
// will be converted to the given gv.
func LogRequestObject(ae *audit.Event, obj runtime.Object, gv schema.GroupVersion, s runtime.NegotiatedSerializer) {
	if ae == nil || ae.Level.Less(audit.LevelRequest) {
		return
	}

	// TODO(audit): hook into the serializer to avoid double conversion
	var err error
	ae.RequestObject, err = encodeObject(obj, gv, s)
	if err != nil {
		// TODO(audit): add error slice to audit event struct
		glog.Warningf("Auditing failed of %v request: %v", reflect.TypeOf(obj).Name(), err)
		return
	}

	// complete ObjectRef
	if ae.ObjectRef == nil {
		ae.ObjectRef = &audit.ObjectReference{}
	}
	if acc, ok := obj.(v1.ObjectMetaAccessor); ok {
		meta := acc.GetObjectMeta()
		if len(ae.ObjectRef.Namespace) == 0 {
			ae.ObjectRef.Namespace = meta.GetNamespace()
		}
		if len(ae.ObjectRef.Name) == 0 {
			ae.ObjectRef.Name = meta.GetName()
		}
		if len(ae.ObjectRef.UID) == 0 {
			ae.ObjectRef.UID = meta.GetUID()
		}
		if len(ae.ObjectRef.ResourceVersion) == 0 {
			ae.ObjectRef.ResourceVersion = meta.GetResourceVersion()
		}
	}
}

// LogRquestPatch fills in the given patch as the request object into an audit event.
func LogRequestPatch(ae *audit.Event, patch []byte) {
	if ae == nil || ae.Level.Less(audit.LevelRequest) {
		return
	}

	ae.RequestObject = &runtime.Unknown{
		Raw:         patch,
		ContentType: runtime.ContentTypeJSON,
	}
}

// LogResponseObject fills in the response object into an audit event. The passed runtime.Object
// will be converted to the given gv.
func LogResponseObject(ae *audit.Event, obj runtime.Object, gv schema.GroupVersion, s runtime.NegotiatedSerializer) {
	if ae == nil || ae.Level.Less(audit.LevelRequestResponse) {
		return
	}

	if status, ok := obj.(*metav1.Status); ok {
		ae.ResponseStatus = status
	}

	// TODO(audit): hook into the serializer to avoid double conversion
	var err error
	ae.ResponseObject, err = encodeObject(obj, gv, s)
	if err != nil {
		glog.Warningf("Audit failed for %q response: %v", reflect.TypeOf(obj).Name(), err)
	}
}

func encodeObject(obj runtime.Object, gv schema.GroupVersion, serializer runtime.NegotiatedSerializer) (*runtime.Unknown, error) {
	supported := serializer.SupportedMediaTypes()
	for i := range supported {
		if supported[i].MediaType == "application/json" {
			enc := serializer.EncoderForVersion(supported[i].Serializer, gv)
			var buf bytes.Buffer
			if err := enc.Encode(obj, &buf); err != nil {
				return nil, fmt.Errorf("encoding failed: %v", err)
			}

			return &runtime.Unknown{
				Raw:         buf.Bytes(),
				ContentType: runtime.ContentTypeJSON,
			}, nil
		}
	}
	return nil, fmt.Errorf("no json encoder found")
}
<commit_msg>audit: fill in sub-resource<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package audit

import (
	"bytes"
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/golang/glog"
	"github.com/pborman/uuid"

	"reflect"

	"k8s.io/apimachinery/pkg/apis/meta/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/types"
	utilnet "k8s.io/apimachinery/pkg/util/net"
	"k8s.io/apiserver/pkg/apis/audit"
	auditinternal "k8s.io/apiserver/pkg/apis/audit"
	"k8s.io/apiserver/pkg/authorization/authorizer"
	authenticationv1 "k8s.io/client-go/pkg/apis/authentication/v1"
)

func NewEventFromRequest(req *http.Request, level auditinternal.Level, attribs authorizer.Attributes) (*auditinternal.Event, error) {
	ev := &auditinternal.Event{
		Timestamp:  metav1.NewTime(time.Now()),
		Verb:       attribs.GetVerb(),
		RequestURI: req.URL.RequestURI(),
	}

	ev.Level = level

	// prefer the id from the headers. If not available, create a new one.
	// TODO(audit): do we want to forbid the header for non-front-proxy users?
	ids := req.Header[auditinternal.HeaderAuditID]
	if len(ids) > 0 {
		ev.AuditID = types.UID(ids[0])
	} else {
		ev.AuditID = types.UID(uuid.NewRandom().String())
	}

	ips := utilnet.SourceIPs(req)
	ev.SourceIPs = make([]string, len(ips))
	for i := range ips {
		ev.SourceIPs[i] = ips[i].String()
	}

	if user := attribs.GetUser(); user != nil {
		ev.User.Username = user.GetName()
		ev.User.Extra = map[string]auditinternal.ExtraValue{}
		for k, v := range user.GetExtra() {
			ev.User.Extra[k] = auditinternal.ExtraValue(v)
		}
		ev.User.Groups = user.GetGroups()
		ev.User.UID = user.GetUID()
	}

	if asuser := req.Header.Get(authenticationv1.ImpersonateUserHeader); len(asuser) > 0 {
		ev.ImpersonatedUser = &auditinternal.UserInfo{
			Username: asuser,
		}
		if requestedGroups := req.Header[authenticationv1.ImpersonateGroupHeader]; len(requestedGroups) > 0 {
			ev.ImpersonatedUser.Groups = requestedGroups
		}

		ev.ImpersonatedUser.Extra = map[string]auditinternal.ExtraValue{}
		for k, v := range req.Header {
			if !strings.HasPrefix(k, authenticationv1.ImpersonateUserExtraHeaderPrefix) {
				continue
			}
			k = k[len(authenticationv1.ImpersonateUserExtraHeaderPrefix):]
			ev.ImpersonatedUser.Extra[k] = auditinternal.ExtraValue(v)
		}
	}

	if attribs.IsResourceRequest() {
		ev.ObjectRef = &auditinternal.ObjectReference{
			Namespace:   attribs.GetNamespace(),
			Name:        attribs.GetName(),
			Resource:    attribs.GetResource(),
			Subresource: attribs.GetSubresource(),
			APIVersion:  attribs.GetAPIGroup() + "/" + attribs.GetAPIVersion(),
		}
	}

	return ev, nil
}

// LogRequestObject fills in the request object into an audit event. The passed runtime.Object
// will be converted to the given gv.
func LogRequestObject(ae *audit.Event, obj runtime.Object, gv schema.GroupVersion, s runtime.NegotiatedSerializer) {
	if ae == nil || ae.Level.Less(audit.LevelRequest) {
		return
	}

	// TODO(audit): hook into the serializer to avoid double conversion
	var err error
	ae.RequestObject, err = encodeObject(obj, gv, s)
	if err != nil {
		// TODO(audit): add error slice to audit event struct
		glog.Warningf("Auditing failed of %v request: %v", reflect.TypeOf(obj).Name(), err)
		return
	}

	// complete ObjectRef
	if ae.ObjectRef == nil {
		ae.ObjectRef = &audit.ObjectReference{}
	}
	if acc, ok := obj.(v1.ObjectMetaAccessor); ok {
		meta := acc.GetObjectMeta()
		if len(ae.ObjectRef.Namespace) == 0 {
			ae.ObjectRef.Namespace = meta.GetNamespace()
		}
		if len(ae.ObjectRef.Name) == 0 {
			ae.ObjectRef.Name = meta.GetName()
		}
		if len(ae.ObjectRef.UID) == 0 {
			ae.ObjectRef.UID = meta.GetUID()
		}
		if len(ae.ObjectRef.ResourceVersion) == 0 {
			ae.ObjectRef.ResourceVersion = meta.GetResourceVersion()
		}
	}
}

// LogRquestPatch fills in the given patch as the request object into an audit event.
func LogRequestPatch(ae *audit.Event, patch []byte) {
	if ae == nil || ae.Level.Less(audit.LevelRequest) {
		return
	}

	ae.RequestObject = &runtime.Unknown{
		Raw:         patch,
		ContentType: runtime.ContentTypeJSON,
	}
}

// LogResponseObject fills in the response object into an audit event. The passed runtime.Object
// will be converted to the given gv.
func LogResponseObject(ae *audit.Event, obj runtime.Object, gv schema.GroupVersion, s runtime.NegotiatedSerializer) {
	if ae == nil || ae.Level.Less(audit.LevelRequestResponse) {
		return
	}

	if status, ok := obj.(*metav1.Status); ok {
		ae.ResponseStatus = status
	}

	// TODO(audit): hook into the serializer to avoid double conversion
	var err error
	ae.ResponseObject, err = encodeObject(obj, gv, s)
	if err != nil {
		glog.Warningf("Audit failed for %q response: %v", reflect.TypeOf(obj).Name(), err)
	}
}

func encodeObject(obj runtime.Object, gv schema.GroupVersion, serializer runtime.NegotiatedSerializer) (*runtime.Unknown, error) {
	supported := serializer.SupportedMediaTypes()
	for i := range supported {
		if supported[i].MediaType == "application/json" {
			enc := serializer.EncoderForVersion(supported[i].Serializer, gv)
			var buf bytes.Buffer
			if err := enc.Encode(obj, &buf); err != nil {
				return nil, fmt.Errorf("encoding failed: %v", err)
			}

			return &runtime.Unknown{
				Raw:         buf.Bytes(),
				ContentType: runtime.ContentTypeJSON,
			}, nil
		}
	}
	return nil, fmt.Errorf("no json encoder found")
}
<|endoftext|>